Merge branch 'open_gj' of http://gitlab.fangjia.com/dev0/main into open_gj
This commit is contained in:
@@ -19,6 +19,7 @@
|
||||
<modules>
|
||||
<module>../../projects/fangjia-core</module>
|
||||
<module>../../projects/fangjia-dal</module>
|
||||
<module>../../projects/chafangtong-dal</module>
|
||||
<module>../../projects/fangjia-service</module>
|
||||
<module>../../projects/geluke-dal</module>
|
||||
<module>../../projects/geluke-service</module>
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: geluke
|
||||
namespace: fangjia
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
revisionHistoryLimit: 5
|
||||
selector:
|
||||
matchLabels:
|
||||
project: geluke
|
||||
app: fangjia
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
project: geluke
|
||||
app: fangjia
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: nexus-registry-secret
|
||||
nodeSelector:
|
||||
app: fangjia
|
||||
containers:
|
||||
- name: geluke
|
||||
image: 192.168.0.19:9012/geluke:latest
|
||||
imagePullPolicy: Always
|
||||
resources:
|
||||
limits:
|
||||
memory: "3Gi"
|
||||
requests:
|
||||
memory: "2Gi"
|
||||
env:
|
||||
- name: JAVA_OPTS
|
||||
value: "-Xmx3072m -XX:MaxPermSize=256m
|
||||
-Dlog4j.configuration=file:/home/goojia/webapps/ROOT/WEB-INF/classes/log4j_linux.xml
|
||||
-Dfangjia.overridePropertiesLocation=/home/goojia/goojiaconfig/geluke.properties"
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: web
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: host-time
|
||||
mountPath: /etc/localtime
|
||||
- name: goojiaconfig
|
||||
mountPath: /home/goojia/goojiaconfig
|
||||
|
||||
volumes:
|
||||
- name: host-time
|
||||
hostPath:
|
||||
path: /etc/localtime
|
||||
- name: goojiaconfig
|
||||
cephfs:
|
||||
monitors:
|
||||
- 192.168.0.208:6789
|
||||
- 192.168.0.209:6789
|
||||
- 192.168.0.210:6789
|
||||
user: admin
|
||||
path: /k8s/goojiaconfig
|
||||
secretRef:
|
||||
name: ceph-secret
|
||||
readOnly: false
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: geluke
|
||||
namespace: fangjia
|
||||
labels:
|
||||
project: geluke
|
||||
app: fangjia
|
||||
spec:
|
||||
type: NodePort
|
||||
selector:
|
||||
project: geluke
|
||||
app: fangjia
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
nodePort: 30010
|
||||
@@ -1,63 +0,0 @@
|
||||
/**
|
||||
* Created by Administrator on 2017/5/9.
|
||||
*/
|
||||
import com.sun.org.apache.xerces.internal.impl.io.UTF8Reader;
|
||||
import com.sun.xml.internal.txw2.Document;
|
||||
import sun.nio.cs.ext.GB18030;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
public class Test {
|
||||
String fixture_Date = null;
|
||||
String area = null;
|
||||
String unit_Price = null;
|
||||
String transaction_Price = null;
|
||||
static String sendGet(String url) {
|
||||
String result = "";
|
||||
// 定义一个缓冲字符输入流
|
||||
BufferedReader in = null;
|
||||
try {
|
||||
// 将string转成url对象
|
||||
URL realUrl = new URL(url);
|
||||
// 初始化一个链接到那个url的连接
|
||||
URLConnection connection = realUrl.openConnection();
|
||||
// 开始实际的连接
|
||||
connection.connect();
|
||||
// 初始化 BufferedReader输入流来读取URL的响应
|
||||
in = new BufferedReader(new InputStreamReader(connection.getInputStream(),"gbk"));
|
||||
// 用来临时存储抓取到的每一行的数据
|
||||
String line;
|
||||
while ((line = in.readLine()) != null)
|
||||
{ // 遍历抓取到的每一行并将其存储到result里面
|
||||
result += line;
|
||||
}
|
||||
Document document = Jsoup.parse(result, url);
|
||||
|
||||
parseListPage(document);
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
System.out.println("发送GET请求出现异常!" + e);
|
||||
e.printStackTrace(); }
|
||||
// 使用finally来关闭输入流
|
||||
finally {
|
||||
try {
|
||||
if (in != null) {
|
||||
in.close(); }
|
||||
} catch (Exception e2) {
|
||||
e2.printStackTrace();
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
public static void main(String[] args) {
|
||||
// 定义即将访问的链接
|
||||
String url = "http://dl.goufang.com/xq/1062/6";
|
||||
// 访问链接并获取页面内容
|
||||
String result = sendGet(url);
|
||||
System.out.println(result);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-core</artifactId>
|
||||
<version>0.2.0-SNAPSHOT</version>
|
||||
<version>0.2.0</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
@@ -154,7 +154,7 @@
|
||||
<dependency>
|
||||
<groupId>com.poseidon</groupId>
|
||||
<artifactId>poseidon-message</artifactId>
|
||||
<version>0.0.1-SNAPSHOT</version>
|
||||
<version>0.0.1</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>org.mongodb</groupId>
|
||||
|
||||
@@ -99,7 +99,7 @@
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-web</artifactId>
|
||||
<version>0.2.0-SNAPSHOT</version>
|
||||
<version>0.2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
|
||||
+9
@@ -12,6 +12,7 @@ import com.fangjia.service.mongo.fangjia.SeawaterService;
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.sothis.core.util.Pager;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.Arrays;
|
||||
@@ -116,6 +117,14 @@ public class SeawaterController {
|
||||
return ret;
|
||||
}
|
||||
|
||||
public Object batchDeleteAction(@Param(name="ids") String ids) throws UnsupportedEncodingException {
|
||||
JsonModel ret = new JsonModel();
|
||||
if (StringUtils.isNotBlank(ids)){
|
||||
seawaterService.deleteByIds(ids);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
public void setSeawaterService(SeawaterService seawaterService) {
|
||||
this.seawaterService = seawaterService;
|
||||
}
|
||||
|
||||
@@ -118,9 +118,13 @@
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
|
||||
<div>
|
||||
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
|
||||
<input class ="btn btn-primary" id="batchDelete" type="submit" value="批量删除" style="float: right" />
|
||||
</div>
|
||||
<table id="dataSheet" class="table table-hover table-condensed" name="reginTable">
|
||||
<tr>
|
||||
<th style="width:30px;"><input type="checkbox" id="checkAll"/>全选</th>
|
||||
<th style="width:60px;">类型</th>
|
||||
<th style="width:50px;">区域</th>
|
||||
<th style="width:130px;">小区名</th>
|
||||
@@ -142,6 +146,7 @@
|
||||
<#if seawaterList??>
|
||||
<#list seawaterList as it>
|
||||
<tr>
|
||||
<td><input type="checkbox" name="houseCheck" value="${it.id}"/></td>
|
||||
<td>
|
||||
<#if it.category! == 'sell'>
|
||||
二手房
|
||||
@@ -249,7 +254,7 @@
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
changeCity('${cityDisplay!}');
|
||||
|
||||
$(".hrefClick").click(function(){
|
||||
@@ -336,8 +341,48 @@
|
||||
$("#roomSpan").hide();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// 全选
|
||||
$("#checkAll").click(function(){
|
||||
$('input[name="houseCheck"]').each(function() {
|
||||
var _this = this
|
||||
if (_this.checked == true) {
|
||||
_this.checked = false
|
||||
} else {
|
||||
_this.checked = true
|
||||
}
|
||||
|
||||
})
|
||||
})
|
||||
|
||||
$("#batchDelete").click(function(){
|
||||
if(confirm('确认要删除所选的数据吗?')) {
|
||||
var ids = "";
|
||||
$('input:checkbox[name=houseCheck]:checked').each(function() {
|
||||
var _this = $(this);
|
||||
ids += _this.val() + ","
|
||||
});
|
||||
if (ids == "") {
|
||||
alert("请选择数据")
|
||||
return false;
|
||||
}
|
||||
jQuery.ajax({
|
||||
type: 'POST',
|
||||
url: '../seawater/batchDelete',
|
||||
data:{ids:ids},
|
||||
dataType: 'json',
|
||||
timeout: 100000,
|
||||
success: function(model) {
|
||||
alert("批量删除操作成功");
|
||||
location.reload();
|
||||
},
|
||||
error : function() {
|
||||
//alert("操作失败");
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
</script>
|
||||
</html>
|
||||
</#compress>
|
||||
@@ -277,6 +277,39 @@ public class BaiDuApiUtils {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param coords
|
||||
* @param from
|
||||
* @return
|
||||
*/
|
||||
public static JSONObject geoconv(String coords,int from){
|
||||
Map<String,String> params = new HashMap<String, String>();
|
||||
params.put("output", "json");
|
||||
params.put("ak", AK);
|
||||
params.put("coords", coords);
|
||||
params.put("from", String.valueOf(from));
|
||||
HttpRequest request = new HttpRequest();
|
||||
try{
|
||||
String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(result);
|
||||
while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){
|
||||
changeAK();
|
||||
result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
|
||||
json = JSONObject.fromObject(result);
|
||||
Thread.sleep(1000 * 60);
|
||||
}
|
||||
if(json.getInt("status") >= 300){
|
||||
changeAK();
|
||||
json = geoconv(coords, from);
|
||||
}
|
||||
return json.getJSONArray("result").getJSONObject(0);
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static JSONObject geoconv(String coords, int from, int to){
|
||||
Map<String,String> params = new HashMap<String, String>();
|
||||
|
||||
@@ -312,25 +312,6 @@ public class DataUtil {
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* 加密图片url
|
||||
*
|
||||
* @param u
|
||||
* @param s
|
||||
* @param t
|
||||
* @return
|
||||
*/
|
||||
public static String encodePicUrl(String u, String s, String t) {
|
||||
if (StringUtils.isBlank(u)) {
|
||||
return StringUtils.EMPTY;
|
||||
}
|
||||
try {
|
||||
return SecurityUtil.encode("u=" + URLEncoder.encode(u, "UTF-8") + "&s=" + s + "&t=" + t);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return SecurityUtil.encode("u=" + u + "&s=" + s + "&t=" + t);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 把字符串以16进制数值显示出来
|
||||
*
|
||||
@@ -1805,6 +1786,25 @@ public class DataUtil {
|
||||
return b2.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* 加密图片url
|
||||
*
|
||||
* @param u
|
||||
* @param s
|
||||
* @param t
|
||||
* @return
|
||||
*/
|
||||
public static String encodePicUrl(String u, String s, String t) {
|
||||
if (StringUtils.isBlank(u)) {
|
||||
return StringUtils.EMPTY;
|
||||
}
|
||||
try {
|
||||
return SecurityUtil.encode("u=" + URLEncoder.encode(u, "UTF-8") + "&s=" + s + "&t=" + t);
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
return SecurityUtil.encode("u=" + u + "&s=" + s + "&t=" + t);
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
System.out.println(getFirstChar("文裕花园"));
|
||||
System.out.println(getFirstChar("文怡花园"));
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -205,7 +205,6 @@ public abstract class ElasticDaoImpl<E extends MongoEntity> extends AbstractJpaC
|
||||
|
||||
@Override
|
||||
public void deleteByIds(List<String> idList) {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@@ -14,10 +14,8 @@ public interface SeawaterDao extends com.fangjia.dal.SeawaterDao<Seawater> {
|
||||
|
||||
Seawater findByIdFromEs(String id,String table);
|
||||
|
||||
|
||||
List<Seawater> findSeawatersFromEs(String category, String city, String region, String name, List<String> alias, List<String> regionAlias, Integer startDate, Integer endDate,
|
||||
Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root, String sort, boolean order, int start, int limit , String table);
|
||||
|
||||
List<Seawater> findHouses(String city, String region, List<String> alias, Integer minPrice, Integer maxPrice, Integer limit);
|
||||
|
||||
}
|
||||
|
||||
+47
-1
@@ -96,6 +96,44 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
return new ArrayList<Seawater>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteById(String id) {
|
||||
deleteByIds(Arrays.asList(id));
|
||||
}
|
||||
|
||||
private Map<String, String> buildHeader() {
|
||||
Base64 token = new Base64();
|
||||
String authenticationEncoding = token.encodeAsString(new String("" + ES_USER + ":" + ES_PASSWORD + "").getBytes());
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Basic " + authenticationEncoding + "");
|
||||
return headers;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteByIds(List<String> idList) {
|
||||
HttpRequest request = new HttpRequest();
|
||||
|
||||
JSONObject reqBody = new JSONObject();
|
||||
JSONObject query = new JSONObject();
|
||||
JSONObject bool = new JSONObject();
|
||||
JSONArray must = new JSONArray();
|
||||
must.add(addTerms("_id", idList));
|
||||
bool.put("must", must);
|
||||
query.put("bool", bool);
|
||||
reqBody.put("query", query);
|
||||
|
||||
JSONObject script = new JSONObject();
|
||||
script.put("source", "ctx._source.status=1");
|
||||
reqBody.put("script", script);
|
||||
|
||||
try {
|
||||
request.postWithBody("http://es.fangjia.com:9200/seawater/_update_by_query", reqBody, this.buildHeader(), "UTF-8");
|
||||
} catch (HttpRequestException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
private JSONObject addMatch(String name, String value){
|
||||
JSONObject term = new JSONObject();
|
||||
JSONObject match = new JSONObject();
|
||||
@@ -339,6 +377,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
if(limit >= 0){
|
||||
reqBody.put("size", limit);
|
||||
}
|
||||
|
||||
// ²»Õ¹Ê¾É¾³ý
|
||||
JSONObject param = new JSONObject();
|
||||
param.put("status", 0);
|
||||
JSONObject temp = new JSONObject();
|
||||
temp.put("term", param);
|
||||
jsonList.add(temp);
|
||||
|
||||
query.put("must", jsonList);
|
||||
q.put("bool", query);
|
||||
reqBody.put("query", q);
|
||||
@@ -350,7 +396,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
headers.put("Content-Type", "application/json");
|
||||
headers.put("Authorization", "Basic "+authenticationEncoding+"");
|
||||
try {
|
||||
String response = request.postWithBody("http://es.fangjia.com:9200/"+table+"/_search", reqBody, headers, "UTF-8");
|
||||
String response = request.postWithBody("http://es.fangjia.com:9200/"+table+"/_search?rest_total_hits_as_int=true", reqBody, headers, "UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(response);
|
||||
long time2 = System.currentTimeMillis();
|
||||
System.out.println(time2-time1);
|
||||
|
||||
@@ -147,5 +147,28 @@
|
||||
<version>1.0.0</version>
|
||||
<classifier>jdk18</classifier>
|
||||
</dependency>
|
||||
<!-- html转pdf -->
|
||||
<dependency>
|
||||
<groupId>org.xhtmlrenderer</groupId>
|
||||
<artifactId>core-renderer</artifactId>
|
||||
<version>R8</version>
|
||||
</dependency>
|
||||
<!-- 绘制折线图等 -->
|
||||
<dependency>
|
||||
<groupId>org.jfree</groupId>
|
||||
<artifactId>jfreechart</artifactId>
|
||||
<version>1.5.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.xmlgraphics</groupId>
|
||||
<artifactId>batik-codec</artifactId>
|
||||
<version>1.11</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-core</artifactId>
|
||||
<version>0.1.0-SNAPSHOT</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+22
-1
@@ -1705,6 +1705,25 @@ public class AddressController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object addressSimilarityAction(AddesssMatchModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空"));
|
||||
return mav;
|
||||
}
|
||||
model.setAddress(model.getAddress_1());
|
||||
String result1 = addressMatchProxyService.standard(convertModelToMap(model));
|
||||
model.setAddress(model.getAddress_2());
|
||||
String result2 = addressMatchProxyService.standard(convertModelToMap(model));
|
||||
if (result1 != null && result2 != null) {
|
||||
Map<String, Object> resultMap = addressService.addressSimilarity(result1, result2);
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(resultMap));
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object parseCityAction(AddressModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getProvince())) {
|
||||
@@ -1765,13 +1784,14 @@ public class AddressController {
|
||||
dto.append("township", address.getTownship());
|
||||
dto.append("street", address.getStreet());
|
||||
dto.append("streetNumber", address.getStreetNumber());
|
||||
dto.append("name", building.getSuggestName());
|
||||
if(StringUtils.isBlank(building.getSuggestName())){
|
||||
if(StringUtils.isNotBlank(address.getParsedPropertyName())){
|
||||
dto.append("name", address.getParsedPropertyName());
|
||||
}else if(StringUtils.isNotBlank(address.getOrgKeywords())){
|
||||
dto.append("name", address.getOrgKeywords());
|
||||
}
|
||||
}else{
|
||||
dto.append("name", building.getSuggestName());
|
||||
}
|
||||
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
|
||||
String addr = building.getAddress();
|
||||
@@ -1789,6 +1809,7 @@ public class AddressController {
|
||||
}
|
||||
}
|
||||
dto.append("buildingNumber", address.getBuildingNumber());
|
||||
dto.append("floor", address.getFloor());
|
||||
dto.append("unitNumber", address.getUnitNumber());
|
||||
dto.append("roomNumber", address.getRoomNumber());
|
||||
dto.append("formattedAddress", address.getFormattedAddress());
|
||||
|
||||
+4
-11
@@ -131,7 +131,7 @@ import com.google.common.collect.Sets;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.DBObject;
|
||||
|
||||
//@Authority(role = Role.USER)
|
||||
@Authority(role = Role.USER)
|
||||
public class PropertyController {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
|
||||
@@ -4966,6 +4966,7 @@ public class PropertyController {
|
||||
* @throws NumberFormatException
|
||||
* @throws IllegalAccessException
|
||||
*/
|
||||
@Authority(role = Role.NONE)
|
||||
public Object rentReportAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
@@ -5103,8 +5104,8 @@ public class PropertyController {
|
||||
|
||||
//周边相似小区租金
|
||||
Integer room = null;
|
||||
if(model.getHouseType()!=null){
|
||||
room = model.getHouseType();
|
||||
if(model.getRoom()!=0){
|
||||
room = model.getRoom();
|
||||
}else{
|
||||
room = houseService.getDistrictRoomType((float)model.getSize());
|
||||
}
|
||||
@@ -5164,7 +5165,6 @@ public class PropertyController {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public Object evaluateRentAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
@@ -5255,13 +5255,6 @@ public class PropertyController {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public void setHouseLoudongService(HouseLoudongService houseLoudongService) {
|
||||
this.houseLoudongService = houseLoudongService;
|
||||
}
|
||||
|
||||
+533
-11
@@ -1,32 +1,57 @@
|
||||
package com.fangjia.open.controllers.weixin;
|
||||
|
||||
import com.fangjia.core.util.BeanUtils;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.GeocodeUtils;
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.SdQuery;
|
||||
import com.fangjia.dal.SrQuery;
|
||||
import com.fangjia.dal.mongo.model.DistrictRoomSqmetreRent;
|
||||
import com.fangjia.dal.mongo.model.Evaluation;
|
||||
import com.fangjia.dal.mongo.model.RentDealPrice;
|
||||
import com.fangjia.dal.seawater.model.Seawater;
|
||||
import com.fangjia.dal.seaweed.model.District;
|
||||
import com.fangjia.open.APIConstant;
|
||||
import com.fangjia.open.compatible.model.JsonModel;
|
||||
import com.fangjia.open.controllers.PropertyController;
|
||||
import com.fangjia.open.dto.DistrictHouseDtao;
|
||||
import com.fangjia.open.interceptors.AuthenticationBean;
|
||||
import com.fangjia.open.model.BaseModel;
|
||||
import com.fangjia.open.model.weixin.EvaluationModel;
|
||||
import com.fangjia.open.model.weixin.PropertyEvaluationModel;
|
||||
import com.fangjia.open.util.LineChartsUtils;
|
||||
import com.fangjia.service.api.AutoCompleteBo;
|
||||
import com.fangjia.service.datastore.DistrictService;
|
||||
import com.fangjia.service.datastore.EvaluationService;
|
||||
import com.fangjia.service.datastore.HouseService;
|
||||
import com.fangjia.service.district.DistrictRoomSqmetreRentService;
|
||||
import com.fangjia.service.district.RentDealPriceService;
|
||||
import com.fangjia.service.mongo.fangjia.FileService;
|
||||
import com.fangjia.service.mongo.fangjia.SeawaterService;
|
||||
import com.fangjia.service.util.DistrictScoreUtil;
|
||||
import com.lowagie.text.pdf.BaseFont;
|
||||
import org.apache.batik.transcoder.Transcoder;
|
||||
import org.apache.batik.transcoder.TranscoderException;
|
||||
import org.apache.batik.transcoder.TranscoderInput;
|
||||
import org.apache.batik.transcoder.TranscoderOutput;
|
||||
import org.apache.batik.transcoder.image.PNGTranscoder;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.sothis.core.util.CollectionUtils;
|
||||
import org.sothis.web.mvc.ActionContext;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.xhtmlrenderer.pdf.ITextFontResolver;
|
||||
import org.xhtmlrenderer.pdf.ITextRenderer;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.*;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.text.ParseException;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* 建设银行租金控制类
|
||||
@@ -36,7 +61,15 @@ import java.util.Map;
|
||||
*/
|
||||
public class CcbRentController {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(CcbRentController.class);
|
||||
|
||||
private DistrictService districtService;
|
||||
private SeawaterService seawaterService;
|
||||
private DistrictRoomSqmetreRentService districtRoomSqmetreRentService;
|
||||
private EvaluationService evaluationService;
|
||||
private HouseService houseService;
|
||||
private RentDealPriceService rentDealPriceService;
|
||||
private FileService fileService;
|
||||
|
||||
public Object indexAction(BaseModel result) {
|
||||
return result;
|
||||
@@ -44,7 +77,9 @@ public class CcbRentController {
|
||||
|
||||
public Object reportAction(EvaluationModel result, @Param(name = "address") String address, @Param(name = "totalPrice") String totalPrice) {
|
||||
result.setLender(address);
|
||||
result.setDealPrice(Double.valueOf(totalPrice));
|
||||
if (StringUtils.isNotBlank(totalPrice)) {
|
||||
result.setDealPrice(Double.valueOf(totalPrice));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -52,6 +87,434 @@ public class CcbRentController {
|
||||
return result;
|
||||
}
|
||||
|
||||
public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
String result = null;
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
try {
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
// &__refresh_cache=1 刷新缓存
|
||||
String url = "http://open.fangjia.com/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN;
|
||||
Map<String, String> params = org.apache.commons.beanutils.BeanUtils.describe(model);
|
||||
result = httpRequest.post(url, params, headers,"UTF-8");
|
||||
} catch (NoSuchMethodException e) {
|
||||
e.printStackTrace();
|
||||
} catch (HttpRequestException e) {
|
||||
e.printStackTrace();
|
||||
} catch (InvocationTargetException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
|
||||
@Param(name = "name") String name, @Param(name = "address") String address,
|
||||
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
|
||||
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) {
|
||||
com.fangjia.open.model.PropertyEvaluationModel model = new com.fangjia.open.model.PropertyEvaluationModel();
|
||||
model.setCity(city);
|
||||
model.setDistrict(districtStr);
|
||||
model.setName(name);
|
||||
model.setAddress(address);
|
||||
model.setSize(size);
|
||||
model.setFloor(floor);
|
||||
model.setHeight(height);
|
||||
if (room != null) {
|
||||
model.setRoom(room);
|
||||
}
|
||||
if (hall != null) {
|
||||
model.setHall(hall);
|
||||
}
|
||||
if (toilet != null) {
|
||||
model.setToilet(toilet);
|
||||
}
|
||||
model.setDirection(direction);
|
||||
model.setFitment(fitment);
|
||||
|
||||
if (model.getCity().endsWith("市")) {
|
||||
model.setCity(model.getCity().replaceAll("市$", ""));
|
||||
}
|
||||
if (StringUtils.isNotBlank(request.getParameter("calDate"))) {
|
||||
Date calDate = null;
|
||||
try {
|
||||
calDate = org.apache.commons.lang.time.DateUtils.parseDate(request.getParameter("calDate"), new String[] { "yyyy-MM-dd" });
|
||||
} catch (ParseException e) {
|
||||
|
||||
}
|
||||
model.setCalDate(calDate);
|
||||
}
|
||||
/* 先到CPDB库中查找小区 */
|
||||
District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(),
|
||||
"district");
|
||||
model.setName(district.getName());
|
||||
|
||||
|
||||
model.setRegion(district.getRegion());
|
||||
model.setDistrictName(district.getName());
|
||||
if (StringUtils.isBlank(model.getPropertyType())) {
|
||||
if (district.getEstateType2() != null && district.getEstateType2().contains("别墅")) {
|
||||
model.setPropertyType("1");
|
||||
}
|
||||
}
|
||||
|
||||
if (model.getCalDate() != null) {
|
||||
if (new Date().before(model.getCalDate())) {
|
||||
model.setCalDate(DateUtil.getFirstDayOfWeek(new Date()));
|
||||
} else {
|
||||
model.setCalDate(DateUtil.getFirstDayOfWeek(model.getCalDate()));
|
||||
}
|
||||
} else {
|
||||
model.setCalDate(DateUtil.getFirstDayOfWeek(new Date()));
|
||||
}
|
||||
|
||||
Evaluation evaluation = buildEvaluation(model, request);
|
||||
evaluation.setSource("api_property_evaluate_rent");
|
||||
evaluation = evaluationService.evaluateRent(district, evaluation, model.getCalDate());
|
||||
|
||||
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0) {
|
||||
evaluation.setAvgPrice((int)(evaluation.getTotalPrice()/evaluation.getArea()));
|
||||
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
|
||||
districtPriceMap.put("city", district.getCity());
|
||||
districtPriceMap.put("district", district.getRegion());
|
||||
districtPriceMap.put("name", district.getName());
|
||||
districtPriceMap.put("address", district.getAddress());
|
||||
districtPriceMap.put("totalPrice", evaluation.getTotalPrice());
|
||||
districtPriceMap.put("totalPriceUpper", CNNMFilter.convert(String.valueOf(evaluation.getTotalPrice())));
|
||||
districtPriceMap.put("cal_date", DateUtil.formatByPattern(new Date(), "yyyy/MM/dd"));
|
||||
districtPriceMap.put("direction", evaluation.getDirection());
|
||||
districtPriceMap.put("size", evaluation.getArea());
|
||||
districtPriceMap.put("floor", model.getFloor());
|
||||
districtPriceMap.put("height", model.getHeight());
|
||||
districtPriceMap.put("fitment", evaluation.getFitment());
|
||||
|
||||
Integer startPrice = (int)(evaluation.getTotalPrice() - evaluation.getTotalPrice()*0.2);
|
||||
Integer endPrice = (int)(evaluation.getTotalPrice() + evaluation.getTotalPrice()*0.2);
|
||||
|
||||
//周边相似小区
|
||||
if (null == model.getRadius()) {// 设置默认值
|
||||
model.setRadius(1000);
|
||||
}
|
||||
String[] fields = { "city", "region", "name", "estateType2", "id","finishBuildingDate"};
|
||||
List<District> allDistricts = districtService.findNearestDistrict(district, model.getRadius(), fields, Integer.MAX_VALUE);
|
||||
//按条件过滤相似小区
|
||||
Integer buildYear = null;
|
||||
List<District> districts = null;
|
||||
if(StringUtils.isNotBlank(district.getFinishBuildingDate())){
|
||||
buildYear = DistrictScoreUtil.parseStringToDate(district.getFinishBuildingDate());
|
||||
if(buildYear!=null){
|
||||
districts = new ArrayList<District>();
|
||||
int startYear = buildYear -7;
|
||||
int endYear = buildYear + 7;
|
||||
for(District dist : allDistricts){
|
||||
if(StringUtils.isNotBlank(dist.getFinishBuildingDate())){
|
||||
int simlarDistYear = DistrictScoreUtil.parseStringToDate(district.getFinishBuildingDate());
|
||||
if(simlarDistYear>=startYear && simlarDistYear<=endYear && StringUtils.isNotBlank(district.getEstateType2()) && StringUtils.isNotBlank(dist.getEstateType2()) && district.getEstateType2().equals(dist.getEstateType2())){
|
||||
districts.add(dist);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//相似房源
|
||||
Double startArea = model.getSize() - model.getSize()*0.1;
|
||||
Double endArea = model.getSize() + model.getSize()*0.1;
|
||||
Integer startDate = Integer.parseInt(org.apache.tools.ant.util.DateUtils.format(DateUtil.addMonthToDate(new Date(), -12), "yyyyMMdd"));
|
||||
List<Seawater> houses = seawaterService.findSeawatersFromEs2("rent", district.getCity(), district.getRegion(), district.getName(), null, null, startDate, null,
|
||||
startArea, endArea, startPrice, endPrice, null, null, "p_date", false, 1, 5 , null,"seawater-20*/_doc");
|
||||
//无评估小区房源用周边相似小区的房源
|
||||
if(null == houses || houses.size()==0){
|
||||
for (District dist : districts) {
|
||||
houses = seawaterService.findSeawatersFromEs2("rent", dist.getCity(), dist.getRegion(), dist.getName(), null, null, startDate, null,
|
||||
startArea, endArea, startPrice, endPrice, null, null, "p_date", false, 1, 5 , null,"seawater-20*/_doc");
|
||||
if(houses!=null && houses.size()>0){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(houses)) {
|
||||
for (Seawater seawater: houses) {
|
||||
if (seawater.getpDate() != null) {
|
||||
seawater.setRoot(DateUtil.formatByPattern(DateUtil.parseDate(String.valueOf(seawater.getpDate()), "yyyyMMdd"), "yyyy-MM-dd"));
|
||||
}
|
||||
}
|
||||
}
|
||||
districtPriceMap.put("houses", houses);
|
||||
|
||||
//周边相似小区租金
|
||||
if(model.getRoom() != 0){
|
||||
room = model.getRoom();
|
||||
}else{
|
||||
room = houseService.getDistrictRoomType((float)model.getSize());
|
||||
}
|
||||
if (null != districts) {
|
||||
if (null != room) {
|
||||
List<DistrictHouseDtao> dtos = new ArrayList<DistrictHouseDtao>();
|
||||
Date weekend = DateUtil.getFirstDayOfWeek(new Date());
|
||||
for (District dist : districts) {
|
||||
|
||||
List<DistrictRoomSqmetreRent> districtRoomSqmetreRents = districtRoomSqmetreRentService.findDistrictRoomSqmetreRents(dist, 1, room);
|
||||
|
||||
///List<DistrictStatRent> districtStatRents = districtStatRentService.findLatestDistrictStatRent(dist, 1, room);
|
||||
if (null != districtRoomSqmetreRents && districtRoomSqmetreRents.size() > 0) {
|
||||
DistrictRoomSqmetreRent districtRoomSqmetreRent = districtRoomSqmetreRents.get(0);
|
||||
if (districtRoomSqmetreRent.getAvgPrice() > 0) {
|
||||
|
||||
int startSqPrice = (int)(evaluation.getAvgPrice() - evaluation.getAvgPrice()*0.1);
|
||||
int endSqPrice = (int)(evaluation.getAvgPrice() + evaluation.getAvgPrice()*0.1);
|
||||
// SqmetreRent sqmetreRent = sqmetreRentService.findNearestSqmetreRent(dist.getCity(), dist.getRegion(), dist.getName(), DateUtil.formatToYYYYMMDD(weekend), "district");
|
||||
if(districtRoomSqmetreRent.getAvgPrice()>=startSqPrice && districtRoomSqmetreRent.getAvgPrice()<=endSqPrice){
|
||||
DistrictHouseDtao dto = new DistrictHouseDtao();
|
||||
dto.setCity(dist.getCity());
|
||||
dto.setRegion(dist.getRegion());
|
||||
dto.setDistrictName(dist.getName());
|
||||
dto.setSqAvgPrice(districtRoomSqmetreRent.getAvgPrice());
|
||||
dto.setRoom(room);
|
||||
dtos.add(dto);
|
||||
if(dtos.size()>=5){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
districtPriceMap.put("districts", dtos);
|
||||
}
|
||||
}
|
||||
|
||||
//近一年户型租金走势
|
||||
Map<String, Object> trendsMap = districtRoomSqmetreRentService.getRentTrends(district.getCity(), district.getRegion(), district.getName(), room, -12);
|
||||
districtPriceMap.put("trends", trendsMap);
|
||||
|
||||
// 生成走势图图片,一遍转pdf
|
||||
//String tradesImgUrl = buildTrendImage(trendsMap, room);
|
||||
String tradesImgUrl = chartImgUrl;
|
||||
|
||||
//成交的租房房源
|
||||
List<RentDealPrice> rentTrades = rentDealPriceService.getRentDealPrices(district.getCity(), district.getRegion(), district.getName(), 5);
|
||||
if (CollectionUtils.isNotEmpty(rentTrades)) {
|
||||
for (RentDealPrice price : rentTrades) {
|
||||
// 返回成交日期到前端,直接取日期转pdf报错!!!!!!!!!!!!!!!!!
|
||||
if (price.getTradeDate() != null) {
|
||||
price.setSource(DateUtil.formatByPattern(price.getTradeDate(), "yyyy-MM-dd"));
|
||||
}
|
||||
}
|
||||
}
|
||||
districtPriceMap.put("trades", rentTrades);
|
||||
districtPriceMap.put("tradesImgUrl", tradesImgUrl);
|
||||
|
||||
result.setResult(districtPriceMap);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String buildTrendImage(Map<String, Object> trendsMap, Integer room) {
|
||||
List<String> times = (List<String>) trendsMap.get("times");
|
||||
Map<String, Object> distMap = (Map<String, Object>) ((Map<String, Object>) trendsMap.get("distMap")).get(String.valueOf(room));
|
||||
if (distMap != null) {
|
||||
List<Integer> values = new ArrayList<Integer>();
|
||||
for (int i = 0; i < times.size(); i++) {
|
||||
if (distMap.get(times.get(i)) != null) {
|
||||
values.add((Integer) distMap.get(times.get(i)));
|
||||
} else {
|
||||
values.add(null);
|
||||
}
|
||||
}
|
||||
// 保存到本地测试
|
||||
//String url = LineChartsUtils.createChartAndSave2Png("d://test.png",room + "居室", times, values);
|
||||
String url = LineChartsUtils.createChartAndUpload(fileService,room + "居室", times, values);
|
||||
logger.info("租金估值图片上传至服务器:{}", url);
|
||||
return url;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 通过EvaluationModel组装Evaluation实体
|
||||
*
|
||||
* @param model
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
private Evaluation buildEvaluation(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request) {
|
||||
Evaluation evaluation = new Evaluation();
|
||||
evaluation.setCity(model.getCity());
|
||||
evaluation.setRegion(model.getRegion());
|
||||
evaluation.setDistrictName(model.getName());
|
||||
evaluation.setDirection(DIRECTION.get(model.getDirection()));
|
||||
evaluation.setFitment(EVALUATION_MODEL_FITMENT.get(FITMENT.get(model.getDecoration())));
|
||||
evaluation.setAtticArea(model.getAtticSize());
|
||||
evaluation.setTerraceArea(model.getOuterSize());
|
||||
evaluation.setBasementArea(model.getBasementSize());
|
||||
evaluation.setGardenArea(model.getGardenSize());
|
||||
String worryFacility = model.getNegativeFactor();
|
||||
evaluation.setCreationDate(new Date());
|
||||
evaluation.setModificationDate(new Date());
|
||||
evaluation.setClientNo(model.getClientNo());
|
||||
if (0 != model.getAge()) {
|
||||
evaluation.setAge(model.getAge());
|
||||
}
|
||||
evaluation.setArea(model.getSize());
|
||||
if (StringUtils.isNotBlank(model.getBuildType())) {
|
||||
evaluation.setBuildType(model.getBuildType());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getCarport())) {
|
||||
evaluation.setCarport(model.getCarport());
|
||||
}
|
||||
if (model.getDecorationAge() >= 1 && model.getDecorationAge() <= 5) {
|
||||
evaluation.setFitmentAge(model.getDecorationAge());
|
||||
}
|
||||
evaluation.setCarportPrice(model.getCarportPrice());
|
||||
evaluation.setCity(model.getCity());
|
||||
evaluation.setDistrictName(model.getName());
|
||||
if (model.getDecorationCost() != null && model.getDecorationCost() > 0) {
|
||||
evaluation.setFitmentCharge(model.getDecorationCost());
|
||||
}
|
||||
if (0 != model.getHall()) {
|
||||
evaluation.setHall(model.getHall());
|
||||
}
|
||||
if (null != model.getFloor()) {
|
||||
evaluation.setFloor(model.getFloor());
|
||||
}
|
||||
if (model.getMaxFloor() != null && 0 != model.getMaxFloor()) {
|
||||
evaluation.setHeight(model.getMaxFloor());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getBuildingNumber())) {
|
||||
evaluation.setHouseNum(model.getBuildingNumber());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getHouseProperty())) {
|
||||
evaluation.setHouseProperty(model.getHouseProperty());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getInsidepos())) {
|
||||
evaluation.setInsidepos(model.getInsidepos());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLightingVentilation())) {
|
||||
evaluation.setLightingVentilation(model.getLightingVentilation());
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(model.getOtherFactor())) {
|
||||
evaluation.setOtherFactor(model.getOtherFactor());
|
||||
}
|
||||
evaluation.setRegion(model.getDistrict());
|
||||
if (0 != model.getRoom()) {
|
||||
evaluation.setRoom(model.getRoom());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getRoomNumber())) {
|
||||
evaluation.setRoomNum(model.getRoomNumber());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getSideset())) {
|
||||
evaluation.setSideset(model.getSideset());
|
||||
}
|
||||
if (0 != model.getToilet()) {
|
||||
evaluation.setToilet(model.getToilet());
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(model.getFacade())) {
|
||||
evaluation.setFacade(model.getFacade());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getTraffice())) {
|
||||
evaluation.setTraffice(model.getTraffice());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShape())) {
|
||||
evaluation.setShape(model.getShape());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getObstacle())) {
|
||||
evaluation.setObstacle(model.getObstacle());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getIndustry())) {
|
||||
evaluation.setIndustry(model.getIndustry());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getSupportDevice())) {
|
||||
evaluation.setSupportDevice(model.getSupportDevice());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLeftPosition())) {
|
||||
evaluation.setLeftPosition(model.getLeftPosition());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShopType())) {
|
||||
evaluation.setShopType(model.getShopType());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getShopUnique())) {
|
||||
evaluation.setShopUnique(model.getShopUnique());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLocation())) {
|
||||
evaluation.setLocation(model.getLocation());
|
||||
}
|
||||
if (model.isNewDistrict()) {
|
||||
evaluation.setNewDistrict(model.isNewDistrict());
|
||||
}
|
||||
String ip = AnalyticsUtil.getIpAddress(ActionContext.getContext().getRequest());
|
||||
evaluation.setIp(ip);
|
||||
AuthenticationBean authenticationBean = (AuthenticationBean) request
|
||||
.getAttribute(AuthenticationBean.ATTR_AUTHENTICATION_BEAN);
|
||||
if (authenticationBean != null && authenticationBean.apiUser != null) {
|
||||
evaluation.setFromUserId(String.valueOf(authenticationBean.apiUser.getId()));
|
||||
}
|
||||
evaluation.setSource("api_property_evaluation");
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
public Object saveHighChartAsImageAction(HttpServletRequest request, HttpServletResponse response) {
|
||||
String svg = request.getParameter("svg");
|
||||
ByteArrayOutputStream out = null;
|
||||
String url = null;
|
||||
try {
|
||||
svg = svg.replaceAll(":rect", "rect");
|
||||
Transcoder t = new PNGTranscoder();
|
||||
out = new ByteArrayOutputStream();
|
||||
TranscoderInput input = new TranscoderInput(new StringReader(svg));
|
||||
TranscoderOutput output = new TranscoderOutput(out);
|
||||
t.transcode(input, output);
|
||||
byte[] data = out.toByteArray();
|
||||
String id = new ObjectId().toString();
|
||||
fileService.save(id,"o", data, "image/png", "o");
|
||||
url = "http://img0.fangjia.com/stc/fs/" + DataUtil.encodePicUrl(id, "o", "house");
|
||||
} catch (TranscoderException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
out.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
JsonModel model = new JsonModel();
|
||||
AutoCompleteBo bo = new AutoCompleteBo();
|
||||
bo.setAddress(url);
|
||||
model.setJsonModel(bo);
|
||||
return model;
|
||||
}
|
||||
|
||||
public void downloadAction(HttpServletResponse response, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String district,
|
||||
@Param(name = "name") String name, @Param(name = "address") String address,
|
||||
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
|
||||
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) throws IOException {
|
||||
response.setHeader("Content-disposition", "attachment; filename=report.pdf");
|
||||
response.setContentType("application/pdf;charset=UTF-8");
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
OutputStream os = null;
|
||||
try {
|
||||
os = response.getOutputStream();
|
||||
ITextRenderer renderer = new ITextRenderer(35, 24);
|
||||
ITextFontResolver fontResolver = renderer.getFontResolver();
|
||||
String fontUrl = ActionContext.getContext().getServletContext().getRealPath("/") + "fonts" + File.separator + "simsun.ttc";
|
||||
fontResolver.addFont(fontUrl, BaseFont.IDENTITY_H, BaseFont.NOT_EMBEDDED);
|
||||
final String path = "http://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath();
|
||||
String url = path + "/ccbRent/downloadPage?city=" + DataUtil.urlEncode(city) + "&district="+DataUtil.urlEncode(district)+"&name="+DataUtil.urlEncode(name);
|
||||
url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment+"&chartImgUrl="+chartImgUrl;
|
||||
renderer.setDocument(url);
|
||||
renderer.layout();
|
||||
renderer.createPDF(os, true);
|
||||
renderer.finishPDF();
|
||||
os.flush();
|
||||
os.close();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
if (os != null) {
|
||||
os.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 方法同DistrictController#suggestAction
|
||||
*/
|
||||
@@ -112,8 +575,67 @@ public class CcbRentController {
|
||||
return ret;
|
||||
}
|
||||
|
||||
@Autowired
|
||||
private static final Map<String, String> DIRECTION = new HashMap<String, String>() {
|
||||
{
|
||||
put(null, "南");
|
||||
put("1", "西南");
|
||||
put("2", "东南");
|
||||
put("3", "南");
|
||||
put("4", "西北");
|
||||
put("5", "东北");
|
||||
put("6", "东");
|
||||
put("7", "西");
|
||||
put("8", "北");
|
||||
put("9", "东西");
|
||||
put("10", "南北");
|
||||
}
|
||||
};
|
||||
|
||||
private static final Map<String, String> FITMENT = new HashMap<String, String>() {
|
||||
{
|
||||
put(null, "简装");
|
||||
put("1", "毛坯");
|
||||
put("2", "简装");
|
||||
put("3", "精装");
|
||||
put("4", "豪装");
|
||||
}
|
||||
};
|
||||
|
||||
private static final Map<String, String> EVALUATION_MODEL_FITMENT = new HashMap<String, String>() {
|
||||
{
|
||||
put(null, "简单装修");
|
||||
put("简装", "简单装修");
|
||||
put("毛坯", "毛坯");
|
||||
put("精装", "精装修");
|
||||
put("豪装", "豪华装修");
|
||||
}
|
||||
};
|
||||
|
||||
public void setDistrictService(DistrictService districtService) {
|
||||
this.districtService = districtService;
|
||||
}
|
||||
|
||||
public void setSeawaterService(SeawaterService seawaterService) {
|
||||
this.seawaterService = seawaterService;
|
||||
}
|
||||
|
||||
public void setDistrictRoomSqmetreRentService(DistrictRoomSqmetreRentService districtRoomSqmetreRentService) {
|
||||
this.districtRoomSqmetreRentService = districtRoomSqmetreRentService;
|
||||
}
|
||||
|
||||
public void setEvaluationService(EvaluationService evaluationService) {
|
||||
this.evaluationService = evaluationService;
|
||||
}
|
||||
|
||||
public void setHouseService(HouseService houseService) {
|
||||
this.houseService = houseService;
|
||||
}
|
||||
|
||||
public void setRentDealPriceService(RentDealPriceService rentDealPriceService) {
|
||||
this.rentDealPriceService = rentDealPriceService;
|
||||
}
|
||||
|
||||
public void setFileService(FileService fileService) {
|
||||
this.fileService = fileService;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,24 @@ public class AddesssMatchModel extends FangjiaApiModel {
|
||||
private String province;
|
||||
private String debugName;
|
||||
private boolean useMap;
|
||||
private String address_1;
|
||||
private String address_2;
|
||||
|
||||
public String getAddress_1() {
|
||||
return address_1;
|
||||
}
|
||||
|
||||
public void setAddress_1(String address_1) {
|
||||
this.address_1 = address_1;
|
||||
}
|
||||
|
||||
public String getAddress_2() {
|
||||
return address_2;
|
||||
}
|
||||
|
||||
public void setAddress_2(String address_2) {
|
||||
this.address_2 = address_2;
|
||||
}
|
||||
public boolean isContainsLocation() {
|
||||
return containsLocation;
|
||||
}
|
||||
|
||||
+12
@@ -2,6 +2,8 @@ package com.fangjia.open.model.weixin;
|
||||
|
||||
import com.fangjia.open.model.BaseModel;
|
||||
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* 建行租房估值
|
||||
*
|
||||
@@ -69,6 +71,8 @@ public class PropertyEvaluationModel extends BaseModel {
|
||||
*/
|
||||
private String fitment;
|
||||
|
||||
private Map<String, Object> result;
|
||||
|
||||
@Override
|
||||
public String getCityName() {
|
||||
return cityName;
|
||||
@@ -166,4 +170,12 @@ public class PropertyEvaluationModel extends BaseModel {
|
||||
public void setFitment(String fitment) {
|
||||
this.fitment = fitment;
|
||||
}
|
||||
|
||||
public Map<String, Object> getResult() {
|
||||
return result;
|
||||
}
|
||||
|
||||
public void setResult(Map<String, Object> result) {
|
||||
this.result = result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
package com.fangjia.open.util;
|
||||
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.service.mongo.fangjia.FileService;
|
||||
import org.apache.batik.transcoder.Transcoder;
|
||||
import org.apache.batik.transcoder.TranscoderException;
|
||||
import org.apache.batik.transcoder.TranscoderInput;
|
||||
import org.apache.batik.transcoder.TranscoderOutput;
|
||||
import org.apache.batik.transcoder.image.JPEGTranscoder;
|
||||
import org.apache.batik.transcoder.image.PNGTranscoder;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.jfree.chart.ChartFactory;
|
||||
import org.jfree.chart.ChartUtilities;
|
||||
import org.jfree.chart.JFreeChart;
|
||||
import org.jfree.chart.axis.NumberAxis;
|
||||
import org.jfree.chart.plot.CategoryPlot;
|
||||
import org.jfree.chart.plot.PlotOrientation;
|
||||
import org.jfree.chart.renderer.category.LineAndShapeRenderer;
|
||||
import org.jfree.data.category.DefaultCategoryDataset;
|
||||
|
||||
import java.awt.*;
|
||||
import java.io.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* 折线图绘图工具类
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/4/9
|
||||
*/
|
||||
public class LineChartsUtils {
|
||||
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
File file = new File("d://test.png");
|
||||
List<String> columns = new ArrayList<String>();
|
||||
columns.add("2020-02");
|
||||
columns.add("2020-03");
|
||||
columns.add("2020-04");
|
||||
columns.add("2020-05");
|
||||
columns.add("2020-06");
|
||||
columns.add("2020-07");
|
||||
columns.add("2020-08");
|
||||
columns.add("2020-09");
|
||||
columns.add("2020-10");
|
||||
columns.add("2020-11");
|
||||
columns.add("2020-12");
|
||||
columns.add("2021-01");
|
||||
|
||||
List<Integer> values = new ArrayList<Integer>();
|
||||
values.add(1000);
|
||||
values.add(2000);
|
||||
values.add(2000);
|
||||
values.add(2000);
|
||||
values.add(2000);
|
||||
values.add(2000);
|
||||
values.add(2000);
|
||||
values.add(3000);
|
||||
values.add(3000);
|
||||
values.add(3000);
|
||||
values.add(3000);
|
||||
values.add(3000);
|
||||
|
||||
createChartAndSave2Png("d://test.png", "1居室", columns, values);
|
||||
}
|
||||
|
||||
public static String createChartAndSave2Png(String path, String rowKey, List<String> columns, List<Integer> values) {
|
||||
File file = new File(path);
|
||||
try {
|
||||
ChartUtilities.saveChartAsPNG(file, createChart(rowKey, columns, values),800,300);
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
public static String createChartAndUpload(FileService fileService, String rowKey, List<String> columns, List<Integer> values) {
|
||||
ByteArrayOutputStream os = new ByteArrayOutputStream();
|
||||
try {
|
||||
ChartUtilities.writeChartAsPNG(os, createChart(rowKey, columns, values),800,300);
|
||||
byte[] data = os.toByteArray();
|
||||
String id = new ObjectId().toString();
|
||||
fileService.save(id,"o", data, "image/png", "o");
|
||||
String url = "http://img0.fangjia.com/stc/fs/" + DataUtil.encodePicUrl(id, "o", "house");
|
||||
|
||||
return url;
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
if (os != null) {
|
||||
try {
|
||||
os.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static void svg2ImgTest() {
|
||||
String svg = "<svg height=\"200\" xmlns=\"http://www.w3.org/2000/svg\"><polygon points=\"10,10 75,150 150,60\" style=\"fill:#63BCF7;stroke:black;stroke-width:3\"/></svg>";
|
||||
String type="image/png";
|
||||
String filename="svg";
|
||||
if (null != type && null != svg) {
|
||||
svg = svg.replaceAll(":rect", "rect");
|
||||
String ext = "";
|
||||
Transcoder t = null;
|
||||
if (type.equals("image/png")) {
|
||||
ext = "png";
|
||||
t = new PNGTranscoder();
|
||||
}
|
||||
else if (type.equals("image/jpeg")) {
|
||||
ext = "jpg";
|
||||
t = new JPEGTranscoder();
|
||||
}
|
||||
if (null != t) {
|
||||
OutputStream out = null;
|
||||
try {
|
||||
out = new FileOutputStream("D:\\"+filename+"."+ext);
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
TranscoderInput input = new TranscoderInput(new StringReader(svg));
|
||||
TranscoderOutput output = new TranscoderOutput(out);
|
||||
try{
|
||||
t.transcode(input, output);
|
||||
System.out.println("out");
|
||||
}
|
||||
catch (TranscoderException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 定义图表对象
|
||||
* @param rowKey 曲线名,如一居室
|
||||
* @param columns 横坐标名
|
||||
* @param values 纵坐标值
|
||||
*/
|
||||
public static JFreeChart createChart(String rowKey, List<String> columns, List<Integer> values) {
|
||||
createDataset(rowKey, columns, values);
|
||||
// 折线图名称, 横坐标名称, 纵坐标名称 数据水平竖直 PlotOrientation.HORIZONTAL, 水平显示图像 true, 水平显示图像 true, include legend true, tooltips false, urls
|
||||
JFreeChart chart = ChartFactory.createLineChart("", "", "", createDataset(rowKey, columns, values), PlotOrientation.VERTICAL, true, true, false);
|
||||
CategoryPlot plot = chart.getCategoryPlot();
|
||||
plot.setRangeGridlinesVisible(false); // 是否显示格子线
|
||||
plot.setBackgroundAlpha(0.3f); // 设置背景透明度
|
||||
LineAndShapeRenderer renderer = (LineAndShapeRenderer) plot.getRenderer();
|
||||
renderer.setSeriesPaint(0, Color.BLUE);
|
||||
NumberAxis rangeAxis = (NumberAxis) plot.getRangeAxis();
|
||||
rangeAxis.setStandardTickUnits(NumberAxis.createIntegerTickUnits());
|
||||
rangeAxis.setAutoRangeIncludesZero(true);
|
||||
rangeAxis.setUpperMargin(0.20);
|
||||
rangeAxis.setLabelAngle(Math.PI / 2.0);
|
||||
return chart;
|
||||
}
|
||||
|
||||
private static DefaultCategoryDataset createDataset(String rowKey, List<String> columns, List<Integer> values) {
|
||||
DefaultCategoryDataset dataset = new DefaultCategoryDataset();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
String column = columns.get(i);
|
||||
if (values.get(i) != null) {
|
||||
dataset.addValue(Double.valueOf(values.get(i)), rowKey, column);
|
||||
} else {
|
||||
dataset.addValue(null, rowKey, column);
|
||||
}
|
||||
}
|
||||
return dataset;
|
||||
}
|
||||
}
|
||||
@@ -1,37 +1,45 @@
|
||||
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
|
||||
<#compress>
|
||||
<!DOCTYPE html>
|
||||
<html xmlns="http://www.w3.org/1999/html">
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||||
<meta name="referrer" content="no-referrer"/>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
|
||||
<title>租金报告中心</title>
|
||||
<meta name="viewport" content="initial-scale=1, maximum-scale=1">
|
||||
<link rel="shortcut icon" href="/favicon.ico">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
<meta name="viewport" content="initial-scale=1, maximum-scale=1"/>
|
||||
<link rel="shortcut icon" href="/favicon.ico"/>
|
||||
<meta name="apple-mobile-web-app-capable" content="yes"/>
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
|
||||
|
||||
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css">
|
||||
<link rel="stylesheet" href="../static/style/wx.css">
|
||||
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
|
||||
<link rel="stylesheet" href="../static/style/wx.css"/>
|
||||
<style type="text/css">
|
||||
body {
|
||||
overflow: hidden;
|
||||
overflow-y: scroll;
|
||||
background: white;
|
||||
}
|
||||
.main {
|
||||
margin: 4px 8px 4px 8px;
|
||||
background: #f8f8f8;
|
||||
background: white;
|
||||
}
|
||||
.title {
|
||||
position: relative;
|
||||
color: #1E90FF;
|
||||
font-size: 20px;
|
||||
font-weight: bold;
|
||||
}
|
||||
.title1 {
|
||||
font-size: 14px;
|
||||
font-size: 18px;
|
||||
font-weight: bold;
|
||||
color: #1E90FF;
|
||||
margin-top: 2px;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
.p {
|
||||
font-size: 12px;
|
||||
font-size: 18px;
|
||||
line-height: 30px;
|
||||
}
|
||||
.table {
|
||||
width: 100%;
|
||||
@@ -63,11 +71,9 @@
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<div class="main">
|
||||
<div class="title">
|
||||
估值报告
|
||||
<span class="download" id="download">下载</span>
|
||||
估值报告<span class="download" id="download">下载</span>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">1 评估申明</div>
|
||||
@@ -76,12 +82,12 @@
|
||||
<div>
|
||||
<div class="title1">2 评估时间</div>
|
||||
<div class="p">
|
||||
<span name="cal_date"></span><br>
|
||||
评估结果<br>
|
||||
楼盘名称:<span name="name"></span><br>
|
||||
楼盘地址:<span name="address"></span><br>
|
||||
租金估值(RMB):<span name="totalPrice"></span>元/月<br>
|
||||
大写(人民币):<span name="totalPriceUpper"></span><br>
|
||||
<span name="cal_date"></span><br/>
|
||||
<b>评估结果</b><br/>
|
||||
<b>楼盘名称</b>:<span name="name"></span><br/>
|
||||
<b>楼盘地址</b>:<span name="address"></span><br/>
|
||||
<b>租金估值(RMB)</b>:<span name="totalPrice"></span>元/月<br/>
|
||||
<b>大写(人民币)</b>:<span name="totalPriceUpper"></span><br/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
@@ -91,20 +97,21 @@
|
||||
<div>
|
||||
<div class="title1">4 房产状况</div>
|
||||
<div class="p">
|
||||
详细地址:<span name="address"></span><br>
|
||||
楼盘名称:<span name="name"></span><br>
|
||||
室号:<span name="room"></span><br>
|
||||
朝向:<span name="direction"></span><br>
|
||||
建筑面积:<span name="size"></span>平米<br>
|
||||
所在楼层:<span name="floor"></span> 层 <span name="height"></span>总层数<br>
|
||||
电梯配套:<span name="hasLift"></span><br>
|
||||
装修情况:<span name="fitment"></span><br>
|
||||
建筑年代:<span name="buildYear"></span><br>
|
||||
<b>详细地址</b>:<span name="address"></span><br/>
|
||||
<b>楼盘名称</b>:<span name="name"></span><br/>
|
||||
<b>室号</b>:<span name="room"></span><br/>
|
||||
<b>朝向</b>:<span name="direction"></span><br/>
|
||||
<b>建筑面积</b>:<span name="size"></span>平米<br/>
|
||||
<b>所在楼层</b>:<span name="floor"></span> 层 <span name="height"></span>总层数<br/>
|
||||
<b>电梯配套</b>:<span name="hasLift"></span><br/>
|
||||
<b>装修情况</b>:<span name="fitment"></span><br/>
|
||||
<b>建筑年代</b>:<span name="buildYear"></span><br/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">5 历史租金</div>
|
||||
<div class="charts" id="districtRentTrendsChart"></div>
|
||||
<input type="hidden" id="chartImgUrl"/>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">6 已租房源</div>
|
||||
@@ -123,29 +130,8 @@
|
||||
<th>成交日期</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr class="">
|
||||
<td class="td01">上海</td>
|
||||
<td class="td02">浦东区</td>
|
||||
<td>永泰花苑</td>
|
||||
<td>***</td>
|
||||
<td>82</td>
|
||||
<td>2</td>
|
||||
<td>0</td>
|
||||
<td>普通装修</td>
|
||||
<td>2020-07-29</td>
|
||||
</tr>
|
||||
<tr class="bg">
|
||||
<td class="td01">上海</td>
|
||||
<td class="td02">浦东区</td>
|
||||
<td>永泰花苑</td>
|
||||
<td>***</td>
|
||||
<td>82</td>
|
||||
<td>2</td>
|
||||
<td>0</td>
|
||||
<td>普通装修</td>
|
||||
<td>2020-07-29</td>
|
||||
</tr>
|
||||
<tbody id="trades">
|
||||
<td colspan="6">暂无数据</td>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
@@ -168,7 +154,7 @@
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="referenceHouses">
|
||||
<td colspan="6" style="text-align:center">暂无数据</td>
|
||||
<td colspan="6">暂无数据</td>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
@@ -189,7 +175,7 @@
|
||||
</thead>
|
||||
<tbody id="sameDistricts">
|
||||
<tr class="">
|
||||
<td colspan="6" style="text-align:center">暂无数据</td>
|
||||
<td colspan="6">暂无数据</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
@@ -199,44 +185,87 @@
|
||||
|
||||
<script type='text/javascript' src='http://g.alicdn.com/sj/lib/zepto/zepto.js' charset='utf-8'></script>
|
||||
<script type='text/javascript' src='http://g.alicdn.com/msui/sm/0.6.2/js/sm.js' charset='utf-8'></script>
|
||||
<script type="text/javascript" src="http://cdn.hcharts.cn/highcharts/highcharts.js"></script>
|
||||
<!-- 暂时注释 -->
|
||||
<script type="text/javascript" src="http://cdn.highcharts.com.cn/highcharts/9.0.1/highcharts.js"></script>
|
||||
<script type="text/javascript" src="https://code.highcharts.com.cn/highcharts/modules/exporting.js"></script>
|
||||
<script type='text/javascript' src='../static/js/wx.js' charset='utf-8'></script>
|
||||
<script type="text/javascript" src="../static/js/trend-charts.js"></script>
|
||||
<script src="https://html2canvas.hertzen.com/dist/html2canvas.js"></script>
|
||||
<#-- <script src="../static/js/purify.min.js"></script>-->
|
||||
<#-- <script src="https://cdnjs.cloudflare.com/ajax/libs/jspdf/2.3.1/jspdf.umd.min.js"></script>-->
|
||||
<#-- <script src="../static/js/jspdf.umd.js"></script>-->
|
||||
<#-- <script src="../static/js/SourceHanSans-normal.js"></script>-->
|
||||
<#-- <script src="../static/js/simsun-normal.js"></script>-->
|
||||
<#-- <script type="text/javascript">-->
|
||||
<#-- $("#download").bind('click', function(){-->
|
||||
<#-- window.location.href = "download"-->
|
||||
<#-- // 打印所有可用的字体-->
|
||||
<#-- // jspdf.html(document.body, {-->
|
||||
<#-- // callback: function (doc) {-->
|
||||
<#-- // doc.save("租金报告.pdf");-->
|
||||
<#-- // },-->
|
||||
<#-- // x: 10,-->
|
||||
<#-- // y: 10-->
|
||||
<#-- // });-->
|
||||
<#-- // jspdf.html("test测试", {-->
|
||||
<#-- // callback: function (doc) {-->
|
||||
<#-- // doc.save("租金报告.pdf");-->
|
||||
<#-- // },-->
|
||||
<#-- // x: 10,-->
|
||||
<#-- // y: 10-->
|
||||
<#-- // });-->
|
||||
<#-- });-->
|
||||
<#-- $(function () {-->
|
||||
<#-- $('span[name="name"]').text('场中小区');-->
|
||||
<#-- });-->
|
||||
<#-- </script>-->
|
||||
<script type="text/javascript">
|
||||
$("#download").bind("click", function(){
|
||||
|
||||
|
||||
// window.jsPDF = window.jspdf.jsPDF
|
||||
// var jspdf = new jsPDF();
|
||||
// jspdf.addFileToVFS('SourceHanSans-Normal.ttf',normalChinese);
|
||||
// jspdf.addFont('SourceHanSans-Normal.ttf', 'SourceHanSans-Normal', 'normal');
|
||||
// jspdf.setFont('SourceHanSans-Light');
|
||||
//console.log(jspdf.getFontList());
|
||||
var chart;
|
||||
$("#download").bind('click', function(){
|
||||
var chartImgUrl = $('#chartImgUrl').val();
|
||||
window.location.href = "download?city="+params.city+"&district="+params.district+"&name="+params.name+"&address="+params.address+"&size="+params.size+"&floor="+params.floor
|
||||
+"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment+"&chartImgUrl=" + chartImgUrl;
|
||||
// jspdf.html(document.body, {
|
||||
// callback: function (doc) {
|
||||
// doc.save("租金报告.pdf");
|
||||
// },
|
||||
// x: 0,
|
||||
// y: 0,
|
||||
// fontFaces: [{
|
||||
// family: 'SimSun',
|
||||
// weight: 'bold',
|
||||
// //style: style,
|
||||
// src: [{
|
||||
// format: "truetype",
|
||||
// url: "../static/fonts/SimSun.ttf"
|
||||
// }]
|
||||
// }]
|
||||
// });
|
||||
});
|
||||
|
||||
// 后台传参过来用
|
||||
let params = {
|
||||
'city': sessionStorage.city,
|
||||
'district': sessionStorage.district,
|
||||
'name': sessionStorage.name,
|
||||
'address': sessionStorage.address,
|
||||
'size': sessionStorage.size,
|
||||
'floor': sessionStorage.floor,
|
||||
'height': sessionStorage.height,
|
||||
'room': sessionStorage.room,
|
||||
'hall': sessionStorage.hall,
|
||||
'toilet': sessionStorage.toilet,
|
||||
'direction': sessionStorage.direction,
|
||||
'fitment': sessionStorage.fitment
|
||||
};
|
||||
|
||||
function init() {
|
||||
// 后台传参过来用
|
||||
<#--let params = {-->
|
||||
<#-- 'city': '${cityName}',-->
|
||||
<#-- 'district': '${district}',-->
|
||||
<#-- 'name': '${name}',-->
|
||||
<#-- 'address': '${address}',-->
|
||||
<#-- 'size': '${size}',-->
|
||||
<#-- 'floor': '${floor}',-->
|
||||
<#-- 'height': '${height}',-->
|
||||
<#-- 'room': '${room}',-->
|
||||
<#-- 'hall': '${hall}',-->
|
||||
<#-- 'toilet': '${toilet}',-->
|
||||
<#-- 'direction': <#if direction??>${direction}<#else>''</#if>,-->
|
||||
<#-- 'fitment': <#if fitment??>${fitment}<#else>''</#if>-->
|
||||
<#--};-->
|
||||
let params = {
|
||||
'city': sessionStorage.city,
|
||||
'district': sessionStorage.district,
|
||||
'name': sessionStorage.name,
|
||||
'address': sessionStorage.address,
|
||||
'size': sessionStorage.size,
|
||||
'floor': sessionStorage.floor,
|
||||
'height': sessionStorage.height,
|
||||
'room': sessionStorage.room,
|
||||
'hall': sessionStorage.hall,
|
||||
'toilet': sessionStorage.toilet,
|
||||
'direction': sessionStorage.direction,
|
||||
'fitment': sessionStorage.fitment
|
||||
};
|
||||
$.ajax({
|
||||
// 注意需要加..
|
||||
url: '../property/rentReport',
|
||||
@@ -260,7 +289,11 @@
|
||||
// data: [3800, 3900, 4000, 4100, 4200, 4200, 4200, 4200]
|
||||
// }];
|
||||
if (result.trends.distMap[params.room] == undefined) {
|
||||
console.error("返回历史租金房间数有误,不存在" + params.room + "居室数据")
|
||||
console.log("返回历史租金,不存在" + params.room + "居室数据")
|
||||
$('#districtRentTrendsChart').empty();
|
||||
$('#districtRentTrendsChart').removeClass("charts");
|
||||
$('#districtRentTrendsChart').addClass("p table");
|
||||
$('#districtRentTrendsChart').append('<table style="width: 300px"><tr colspan="6">暂无数据<td></tr></table>');
|
||||
} else {
|
||||
var arr = [];
|
||||
result.trends.times.forEach(function (d) {
|
||||
@@ -275,7 +308,6 @@
|
||||
charts.xData = result.trends.times; // 横坐标值
|
||||
createChartNow('districtRentTrendsChart', charts, true);
|
||||
}
|
||||
|
||||
// 房产状况
|
||||
$('span[name="room"]').text(result.room)
|
||||
$('span[name="direction"]').text(result.direction)
|
||||
@@ -286,22 +318,44 @@
|
||||
$('span[name="fitment"]').text(result.fitment)
|
||||
$('span[name="buildYear"]').text(result.buildYear)
|
||||
|
||||
// 已租房源
|
||||
var trades = result.trades;
|
||||
if (trades && trades.length > 0) {
|
||||
$('#trades').empty();
|
||||
trades.forEach(function(d){
|
||||
var fitment = d.fitment != undefined ? d.fitment : '';
|
||||
var hall = d.hall != undefined ? d.hall : '';
|
||||
$('#trades').append('<tr class="bg"> \
|
||||
<td class="td01">' + d.city + '</td> \
|
||||
<td class="td02">' + d.region + '</td> \
|
||||
<td>' + d.districtName + '</td> \
|
||||
<td>' + (d.totalPrice != undefined ? d.totalPrice : '') + '</td> \
|
||||
<td>' + (d.area != undefined ? d.area : '') + '</td> \
|
||||
<td>' + (d.room != undefined ? d.room : '') + '</td> \
|
||||
<td>' + hall + '</td> \
|
||||
<td>' + fitment + '</td> \
|
||||
<td>' + formatDate(d.tradeDate) + '</td> \
|
||||
</tr>');
|
||||
})
|
||||
}
|
||||
|
||||
// 样本参考
|
||||
var referenceHouses = result.houses;
|
||||
if (referenceHouses && referenceHouses.length > 0) {
|
||||
$('#referenceHouses').empty();
|
||||
referenceHouses.forEach(function(d){
|
||||
var fitment = d.fitment != undefined ? d.fitment : '-';
|
||||
var fitment = d.fitment != undefined ? d.fitment : '';
|
||||
var date = (d.pDate + '').substring(0, 4) + "-" + (d.pDate + '').substring(4, 6) + "-" + (d.pDate + '').substring(6, 8)
|
||||
$('#referenceHouses').append('<tr class="bg"> \
|
||||
<td class="td01">' + d.city + '</td> \
|
||||
<td class="td02">' + d.region + '</td> \
|
||||
<td>' + d.districtName + '</td> \
|
||||
<td>' + d.totalPrice + '</td> \
|
||||
<td>' + d.area + '</td> \
|
||||
<td>' + d.room + '</td> \
|
||||
<td>' + d.hall + '</td> \
|
||||
<td>' + (d.totalPrice != undefined ? d.totalPrice : '') + '</td> \
|
||||
<td>' + (d.area != undefined ? d.area : '') + '</td> \
|
||||
<td>' + (d.room != undefined ? d.room : '') + '</td> \
|
||||
<td>' + (d.hall != undefined ? d.hall : '')+ '</td> \
|
||||
<td>' + fitment + '</td> \
|
||||
<td>' + d.pDate + '</td> \
|
||||
<td>' + date + '</td> \
|
||||
</tr>');
|
||||
})
|
||||
|
||||
@@ -315,9 +369,9 @@
|
||||
$('#sameDistricts').append('<tr class=""> \
|
||||
<td class="td01">' + d.city + '</td> \
|
||||
<td class="td02">' + d.region + '</td> \
|
||||
<td>' + d.districtName + '</td> \
|
||||
<td>' + d.roomPrice + '</td> \
|
||||
<td>' + d.room + '</td> \
|
||||
<td>' + (d.districtName != undefined ? d.districtName:'' ) + '</td> \
|
||||
<td>' + (d.roomPrice != undefined ? d.roomPrice:'') + '</td> \
|
||||
<td>' + (d.room != undefined ? d.room:'') + '</td> \
|
||||
<td>1</td> \
|
||||
</tr>');
|
||||
})
|
||||
@@ -331,6 +385,15 @@
|
||||
}
|
||||
})
|
||||
};
|
||||
var formatDate = function (dateStr) {
|
||||
var date = new Date(dateStr);
|
||||
var y = date.getFullYear();
|
||||
var m = date.getMonth() + 1;
|
||||
m = m < 10 ? ('0' + m) : m;
|
||||
var d = date.getDate();
|
||||
d = d < 10 ? ('0' + d) : d;
|
||||
return y + '-' + m + '-' + d;
|
||||
};
|
||||
function createChartNow(renderId, data, showTradeCount) {
|
||||
if(data == null || data == 'undefined' || typeof(data) == undefined) {
|
||||
return;
|
||||
@@ -373,8 +436,9 @@
|
||||
var config = {yaxis : yaxis};
|
||||
createHighChartNow(renderId, data, config);
|
||||
};
|
||||
|
||||
function createHighChartNow(renderId, data, config) {
|
||||
var chart = new Highcharts.Chart({
|
||||
chart = new Highcharts.Chart({
|
||||
credits:{
|
||||
enabled:true,
|
||||
text: '',
|
||||
@@ -393,6 +457,9 @@
|
||||
renderTo: renderId,
|
||||
type: 'spline'
|
||||
},
|
||||
exporting: {
|
||||
enabled: false
|
||||
},
|
||||
title: {
|
||||
text: data.title,
|
||||
floating: false,
|
||||
@@ -421,12 +488,31 @@
|
||||
if(this.series.name.indexOf('成交量') > -1) {
|
||||
return Highcharts.dateFormat('%Y年%m月', this.x) +" 成交: <span style='color:#F50;font-weight: bold;'>"+ this.y +'</span>套';
|
||||
}else {
|
||||
return Highcharts.dateFormat('%Y-%m-%d', this.x) +'<br>'+ this.series.name+ ": <span style='color:#F50;font-weight: bold;'>"+ this.y +'</span>元/平米';
|
||||
return Highcharts.dateFormat('%Y-%m-%d', this.x) +'<br/>'+ this.series.name+ ": <span style='color:#F50;font-weight: bold;'>"+ this.y +'</span>元/平米';
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
saveHighChartImg();
|
||||
};
|
||||
|
||||
function saveHighChartImg() {
|
||||
var svg = chart.getSVG();
|
||||
$.ajax({
|
||||
// 注意需要加..
|
||||
url: "saveHighChartAsImage",
|
||||
data: {
|
||||
svg:svg
|
||||
},
|
||||
dataType: "json",
|
||||
type: "post",
|
||||
success: function (ret) {
|
||||
if (ret.address != null) {
|
||||
$('#chartImgUrl').val(ret.address)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
$(function () {
|
||||
init();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
|
||||
<#compress>
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||||
<meta name="referrer" content="no-referrer"/>
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
|
||||
<title>租金报告中心</title>
|
||||
<meta name="viewport" content="initial-scale=1, maximum-scale=1"/>
|
||||
<link rel="shortcut icon" href="/favicon.ico"/>
|
||||
<meta name="apple-mobile-web-app-capable" content="yes"/>
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
|
||||
|
||||
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
|
||||
<link rel="stylesheet" href="../static/style/wx.css"/>
|
||||
<link rel="stylesheet" href="../static/style/report.css"/>
|
||||
<style type="text/css">
|
||||
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="content">
|
||||
<div class="t1">
|
||||
<em>1</em>估价结果
|
||||
</div>
|
||||
<div class="st1"><em>1.1</em>评估申明:</div>
|
||||
<div class="text1">建设银行遵循中立、客观、公正的原则,通过周密细致的测算,评估求得分析对象的房<br/>地产市场参考。</div>
|
||||
|
||||
<div class="st1"><em>1.2</em>评估时间:</div>
|
||||
<div class="text1"><#if result.cal_date??>${result.cal_date}</#if></div>
|
||||
<div class="st1"><em>1.3</em>评估结果:</div>
|
||||
<div class="text2">
|
||||
<b>楼盘名称</b>:<#if result.name??>${result.name}</#if><br/><br/>
|
||||
<b>楼盘地址</b>:<#if result.address??>${result.address}</#if><br/><br/>
|
||||
<b>租金估值(RMB)</b>:<#if result.totalPrice??>${result.totalPrice}</#if>元/月<br/><br/>
|
||||
<b>大写(人民币)</b>:<#if result.totalPriceUpper??>${result.totalPriceUpper}</#if><br/><br/>
|
||||
</div>
|
||||
<div class="st1"><em>1.4</em>报告有效期:</div>
|
||||
<div class="text1" style="margin-bottom: 30px">本报告在市场无较大波动的情况下,该评估价格在报告出具起三个月内有效。</div>
|
||||
<div class="t1"><em>2</em>房产状况</div>
|
||||
<div class="c1">
|
||||
<table class="distTab" width="700" border="0" cellspacing="0" cellpadding="0">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="textTd" style="width: 15%;">楼盘名称</td>
|
||||
<td class="textRight1" style="width: 35%;"><#if result.name??>${result.name}</#if></td>
|
||||
<td class="textTd" style="width: 15%;">地址</td>
|
||||
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.address??>${result.address}</#if></td>
|
||||
</tr>
|
||||
<tr class="tdbg">
|
||||
<td class="textTd">室号</td>
|
||||
<td class="textRight1"><#if result.room??>${result.room}</#if></td>
|
||||
<td class="textTd">朝向</td>
|
||||
<td class="textRight2" colspan="3"><#if result.direction??>${result.direction}</#if></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="textTd" style="width: 15%;">建筑面积</td>
|
||||
<td class="textRight1" style="width: 35%;"><#if result.size??>${result.size}</#if>平米</td>
|
||||
<td class="textTd" style="width: 15%;">所在楼层</td>
|
||||
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.floor??>${result.floor}</#if> 层 <#if result.height??>${result.height}</#if>总层数</td>
|
||||
</tr>
|
||||
<tr class="tdbg">
|
||||
<td class="textTd">电梯配套</td>
|
||||
<td class="textRight1"><#if result.hasLift??>${result.hasLift}</#if></td>
|
||||
<td class="textTd">装修情况</td>
|
||||
<td class="textRight2" colspan="3"><#if result.fitment??>${result.fitment}</#if></td>
|
||||
</tr>
|
||||
<tr class="tdbg">
|
||||
<td class="textTd">建筑年代</td>
|
||||
<td class="textRight1"><#if result.buildYear??>${result.buildYear}</#if></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="t1">
|
||||
<em>3</em>历史租金
|
||||
</div>
|
||||
<div>
|
||||
<#if result.tradesImgUrl??>
|
||||
<#-- <img width="800px" height="300px" src="../static/images/test.png"></img>-->
|
||||
<img width="600px" height="300px" src="${result.tradesImgUrl}"></img>
|
||||
<#else>
|
||||
<div class="p table">
|
||||
<table class="inline">
|
||||
<tbody><tr><td>暂无数据</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</#if>
|
||||
</div>
|
||||
<div class="t1">
|
||||
<em>4</em>已租房源
|
||||
</div>
|
||||
<div style="margin-bottom: 30px;">
|
||||
<table class="distTab" width="780" border="0" cellspacing="0" cellpadding="0">
|
||||
<thead>
|
||||
<tr class="tdbg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>成交价</th>
|
||||
<th>面积</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
<th>装修</th>
|
||||
<th>成交日期</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<#if (result.trades??&&result.trades?size>0)>
|
||||
<#list result.trades as bo>
|
||||
<tr class="tdbg">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.area??>${bo.area}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}</#if></td>
|
||||
<td><#if bo.fitment??>${bo.fitment}</#if></td>
|
||||
<td><#if bo.source??>${bo.source}</#if></td>
|
||||
</tr>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="t1">
|
||||
<em>5</em>样本参考
|
||||
</div>
|
||||
<div style="margin-bottom: 30px;">
|
||||
<table class="distTab" width="780" border="0" cellspacing="0" cellpadding="0">
|
||||
<thead>
|
||||
<tr class="tdbg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>租 金</th>
|
||||
<th>面积</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
<th>装修</th>
|
||||
<th>成交日期</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="referenceHouses">
|
||||
<#if (result.houses??&&result.houses?size>0)>
|
||||
<#list result.houses as bo>
|
||||
<tr class="tdbg">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.area??>${bo.area}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}</#if></td>
|
||||
<td><#if bo.fitment??>${bo.fitment}</#if></td>
|
||||
<td><#if bo.root??>${bo.root}</#if></td>
|
||||
</tr>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="t1">
|
||||
<em>6</em>周边同类小区租金
|
||||
</div>
|
||||
<div style="margin-bottom: 30px;">
|
||||
<table class="distTab" width="500" border="0" cellspacing="0" cellpadding="0">
|
||||
<thead>
|
||||
<tr class="tdbg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>平米租金</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="sameDistricts">
|
||||
<#if (result.districts??&&result.districts?size>0)>
|
||||
<#list result.districts as bo>
|
||||
<tr class="tdbg">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}<#else>1</#if></td>
|
||||
</tr>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
</#compress>
|
||||
@@ -30,8 +30,6 @@
|
||||
<div class="item-content">
|
||||
<div class="item-inner">
|
||||
<input id="cityName" value="上海" name="city" type="hidden">
|
||||
<!-- 区域 -->
|
||||
<input value="" name="district" type="hidden">
|
||||
<input value="2000" name="radius" type="hidden">
|
||||
<div class="item-title label">城市:<font>上海</font></div>
|
||||
</div>
|
||||
@@ -45,6 +43,8 @@
|
||||
<input type="text" placeholder="请输入小区名或地址" id="districtSearch" name="name" class="search_input" required>
|
||||
<input type="hidden" name="address" />
|
||||
</div>
|
||||
<!-- 区域 -->
|
||||
<input value="" name="district" type="hidden" required>
|
||||
<div class="item-media"><span>(必填项)</span></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -68,9 +68,9 @@
|
||||
<div class="item-inner">
|
||||
<div class="content-block">
|
||||
<div class="item-title label di">户型:</div>
|
||||
<input name="room" type="hidden" id="room">
|
||||
<input name="hall" type="hidden" id="hall">
|
||||
<input name="toilet" type="hidden" id="toilet">
|
||||
<input name="room" type="hidden" id="room" placeholder="请选择室" value="1">
|
||||
<input name="hall" type="hidden" id="hall" placeholder="请选择厅">
|
||||
<input name="toilet" type="hidden" id="toilet" placeholder="请选择卫">
|
||||
<p class="di">
|
||||
<a href="#" class="create-actions01-now">1</a>室 <a
|
||||
href="#" class="create-actions02-now">0</a>厅 <a
|
||||
@@ -147,6 +147,7 @@
|
||||
<div class="content-block">
|
||||
<p><a id="evaluateBtn" class="button button-big button-fill">查看租金结果</a></p>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
@@ -212,15 +213,54 @@
|
||||
}
|
||||
|
||||
$("#evaluateBtn").on('click', function(){
|
||||
var inputs = $("input")
|
||||
for (var i = 0; i < inputs.length; i++) {
|
||||
var input = $(inputs[i])
|
||||
var fieldName = input.attr("placeholder").substring(3)
|
||||
var fieldVal = $.trim(input.val())
|
||||
if (input.attr("required") && fieldVal == "") {
|
||||
if (input.attr("name").indexOf("district") != -1) {
|
||||
$.toast("请选择输入地址时提示的小区")
|
||||
} else {
|
||||
$.toast(fieldName + "不能为空");
|
||||
}
|
||||
return
|
||||
}
|
||||
if (input.attr("type") == 'number' && fieldVal != "") {
|
||||
if (input.attr("name").indexOf("size") != -1) {
|
||||
if (!/^[0-9]+(.[0-9]+)?$/.test(fieldVal)) {
|
||||
$.toast(fieldName + "非法");
|
||||
return
|
||||
}
|
||||
|
||||
} else {
|
||||
if (!/^[0-9]+$/.test(fieldVal)) {
|
||||
$.toast(fieldName + "必须是正整数");
|
||||
return
|
||||
}
|
||||
}
|
||||
var min = input.attr("min")
|
||||
var max = input.attr("max")
|
||||
if (max && min && (parseFloat(min) > parseFloat(fieldVal) || parseFloat(max) < parseFloat(fieldVal))) {
|
||||
$.toast(fieldName + "的取值范围是:" + min + "到" + max);
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if(parseFloat($("#houseHight").val()) < parseFloat($("#houseFloor").val())){
|
||||
$.toast("楼层不能大于楼高");
|
||||
return
|
||||
}
|
||||
storeSession()
|
||||
|
||||
if($("#houseHight").val() && parseFloat($("#houseHight").val())>=60){
|
||||
$.toast("请输入合理的楼层数");
|
||||
return
|
||||
}
|
||||
$("#evaluateBtn").prop("disabled", true)
|
||||
$.ajax({
|
||||
// 注意需要加..
|
||||
url: '../property/evaluateRent',
|
||||
url: '../ccbRent/evaluateRent',
|
||||
data:$("#evaluateForm").serialize(),
|
||||
dataType:"json",
|
||||
type:"post",
|
||||
@@ -228,11 +268,13 @@
|
||||
if(data.code == 200){
|
||||
window.location.href = "report?address=" + data.result.address + "&totalPrice=" + data.result.totalPrice
|
||||
}else{
|
||||
$.toast(data.msg);
|
||||
$.toast("估价失败,请重新评估");
|
||||
$("#evaluateBtn").prop("disabled", false)
|
||||
}
|
||||
},
|
||||
error: function () {
|
||||
|
||||
//$.toast("服务器异常,请重新评估");
|
||||
$("#evaluateBtn").prop("disabled", false)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,21 +3,18 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta charset="utf-8" />
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge"/>
|
||||
<title>租金报告中心</title>
|
||||
<meta name="viewport" content="initial-scale=1, maximum-scale=1">
|
||||
<link rel="shortcut icon" href="/favicon.ico">
|
||||
<meta name="apple-mobile-web-app-capable" content="yes">
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black">
|
||||
|
||||
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css">
|
||||
<link rel="stylesheet" href="../static/style/wx.css">
|
||||
<meta name="viewport" content="initial-scale=1, maximum-scale=1"/>
|
||||
<link rel="shortcut icon" href="/favicon.ico"/>
|
||||
<meta name="apple-mobile-web-app-capable" content="yes" />
|
||||
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
|
||||
|
||||
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
|
||||
<link rel="stylesheet" href="../static/style/wx.css"/>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<div class="page-group">
|
||||
<div class="page page-current">
|
||||
<div class="content">
|
||||
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 3.1 KiB |
@@ -0,0 +1,141 @@
|
||||
body {
|
||||
overflow: hidden;
|
||||
overflow-y: scroll;
|
||||
background: white;
|
||||
/** 特别注意,否则中文不生效 **/
|
||||
font-family: SimSun;
|
||||
}
|
||||
.content {
|
||||
width: 840px;
|
||||
height: 100%;
|
||||
margin: 0 auto;
|
||||
background: #fff;
|
||||
padding-left: 20px;
|
||||
}
|
||||
.content .t1 {
|
||||
background: url(../images/fjt/bg.png) 0 0 no-repeat;
|
||||
width: 820px;
|
||||
height: 42px;
|
||||
color: #704f1c;
|
||||
font-size: 20px;
|
||||
font-weight: bold;
|
||||
text-indent: 14px;
|
||||
line-height: 46px;
|
||||
margin-bottom: 25px;
|
||||
}
|
||||
.content .t1 em {
|
||||
font-family: "Arial";
|
||||
color: #FFF;
|
||||
margin-right: 20px;
|
||||
}
|
||||
em {
|
||||
font-style: normal;
|
||||
}
|
||||
.content .st1 {
|
||||
background: url(../images/fjt/bg.png) 0 -60px no-repeat;
|
||||
width: 172px;
|
||||
height: 40px;
|
||||
color: #704f1c;
|
||||
font-size: 18px;
|
||||
font-weight: bold;
|
||||
text-indent: 7px;
|
||||
line-height: 46px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.content .st1 em {
|
||||
font-family: "Arial";
|
||||
color: #FFF;
|
||||
margin-right: 14px;
|
||||
}
|
||||
.content .text1 {
|
||||
font-size: 18px;
|
||||
line-height: 26px;
|
||||
margin: 0 20px 10px 50px;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
.text1 td {
|
||||
text-align: center
|
||||
}
|
||||
.content .text2 {
|
||||
font-size: 18px;
|
||||
line-height: 40px;
|
||||
padding: 10px 20px 10px 50px;
|
||||
background: #eee;
|
||||
width: 750px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
.content .text3 {
|
||||
font-size: 16px;
|
||||
line-height: 40px;
|
||||
padding: 10px 20px 10px 50px;
|
||||
background: #eee;
|
||||
width: 750px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
.content .c1 {
|
||||
height: 200px;
|
||||
}
|
||||
.content .distTab {
|
||||
font-size: 18px
|
||||
}
|
||||
.content .distTab .textTd {
|
||||
font-weight: bold;
|
||||
}
|
||||
.content .distTab tr td {
|
||||
height: 30px;
|
||||
}
|
||||
.content .distTab tr th {
|
||||
text-align: left;
|
||||
}
|
||||
.content .distTab .textRight1 {
|
||||
text-align: right;
|
||||
padding-right: 80px;
|
||||
}
|
||||
.content .distTab .textRight2 {
|
||||
text-align: right;
|
||||
padding-right: 5px;
|
||||
}
|
||||
.title {
|
||||
position: relative;
|
||||
color: #1E90FF;
|
||||
}
|
||||
.title1 {
|
||||
font-size: 19px;
|
||||
color: #1E90FF;
|
||||
margin-top: 3px;
|
||||
margin-bottom: 3px;
|
||||
}
|
||||
.p {
|
||||
font-size: 16px;
|
||||
}
|
||||
table {
|
||||
border-collapse: separate;
|
||||
text-indent: initial;
|
||||
border-spacing: 2px;
|
||||
}
|
||||
tbody {
|
||||
display: table-row-group;
|
||||
vertical-align: middle;
|
||||
border-color: inherit;
|
||||
}
|
||||
.inline {
|
||||
width: 550px;
|
||||
border-spacing: 0;
|
||||
text-align: left;
|
||||
}
|
||||
.table .inline th {
|
||||
color: #999;
|
||||
font-weight: normal;
|
||||
line-height: 30px;
|
||||
text-align: left;
|
||||
}
|
||||
.download {
|
||||
float: right;
|
||||
color: #3d4145;
|
||||
border: solid 1px #333;
|
||||
order-radius: 3px;
|
||||
font-size: 12px;
|
||||
line-height: 1rem;
|
||||
width: 43px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
@@ -17,8 +17,8 @@
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<encoding>GBK</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
@@ -179,7 +179,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.4.1</version>
|
||||
<version>4.5.10</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.ant</groupId>
|
||||
@@ -315,16 +315,6 @@
|
||||
<artifactId>metadata-extractor</artifactId>
|
||||
<version>2.4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.http</groupId>
|
||||
<artifactId>httpasyncclient</artifactId>
|
||||
<version>4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.http</groupId>
|
||||
<artifactId>httpcore-nio</artifactId>
|
||||
<version>4.1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.sanselan</groupId>
|
||||
<artifactId>sanselan</artifactId>
|
||||
@@ -448,23 +438,18 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>x-pack-transport</artifactId>
|
||||
<version>6.4.2</version>
|
||||
<artifactId>elasticsearch-rest-high-level-client</artifactId>
|
||||
<version>7.11.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.plugin</groupId>
|
||||
<artifactId>x-pack-sql-jdbc</artifactId>
|
||||
<version>6.4.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>transport</artifactId>
|
||||
<version>6.4.2</version>
|
||||
<version>7.11.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-core</artifactId>
|
||||
<version>7.4.0</version>
|
||||
<version>8.7.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
@@ -491,6 +476,13 @@
|
||||
<artifactId>aliyun-java-sdk-address-purification</artifactId>
|
||||
<version>1.0.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.eclipsesource.j2v8</groupId>
|
||||
<artifactId>j2v8_win32_x86_64</artifactId>
|
||||
<version>4.6.0</version>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
<repositories>
|
||||
<repository>
|
||||
|
||||
+2
-2
@@ -13,7 +13,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -147,7 +147,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
|
||||
|
||||
def findHouseMaxArea(String city, String region, String name, List alias) {
|
||||
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24)
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-3
@@ -13,7 +13,7 @@ import com.mongodb.client.MongoCollection
|
||||
import com.mongodb.client.model.UpdateOptions
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -94,7 +94,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it->
|
||||
names << it.name
|
||||
}
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -129,7 +129,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
|
||||
def findHouseCount(String city, String region, String name, List alias, int months) {
|
||||
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months)
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+5
-9
@@ -7,18 +7,14 @@ import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement;
|
||||
import java.sql.Statement
|
||||
|
||||
/**
|
||||
*
|
||||
* ÀúÊ·¾ù¼Û¼ÆËã
|
||||
@@ -27,7 +23,7 @@ import java.sql.Statement;
|
||||
*/
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class HistoricalAverageRevaluationTask extends BaseBatch{
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
|
||||
public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat')
|
||||
|
||||
+2
-13
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.avgprice
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
@@ -9,18 +9,7 @@ import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.Bytes
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement;
|
||||
import com.mongodb.*
|
||||
|
||||
/**
|
||||
* 无样本小区历史遗漏数据补充
|
||||
|
||||
+1
-1
@@ -218,7 +218,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "工商大楼");
|
||||
// q.put("name","东寓新村")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package com.fangjia.script.companyCrawl
|
||||
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AiqichaCrawl {
|
||||
|
||||
def static final proxyPort = 9020;
|
||||
def static final proxyServer = "http-dyn.abuyun.com";
|
||||
def static final proxyUser = "HZU1XU28H3G4Z44D";
|
||||
def static final proxyPass = "466C86ED77FE43D4";
|
||||
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
|
||||
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
|
||||
public final static int AbuyunMaxCount = 10;
|
||||
|
||||
static main(args) {
|
||||
def task = new AiqichaCrawl();
|
||||
task.process()
|
||||
}
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
def url = "https://aiqicha.baidu.com/s/l?q=%E6%96%B0%E5%8F%8B&t=&p=3&s=10&o=0&f=%7B%7D "
|
||||
for(def key : companyKeys){
|
||||
def queryBody = null
|
||||
def s = crawl(url, queryBody);
|
||||
if(s){
|
||||
print s
|
||||
|
||||
|
||||
}
|
||||
println s
|
||||
}
|
||||
}
|
||||
|
||||
def crawl(url, queryBody){
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","aiqicha.baidu.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
headers.put("sec-ch-ua",'"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"')
|
||||
headers.put("sec-ch-ua-mobile","?0")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
headers.put("Sec-Fetch-Site","same-origin")
|
||||
headers.put("Sec-Fetch-Mode","navigate")
|
||||
headers.put("Sec-Fetch-User","?1")
|
||||
headers.put("Sec-Fetch-Dest","document")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
|
||||
String s = doPost(url, queryBody, headers)
|
||||
if(s && s.contains("window.pageData = ")){
|
||||
def prefix = "window.pageData ="
|
||||
def s1 = s.substring(s.indexOf(prefix)+prefix.size())
|
||||
s1 = s1.substring(0, s1.indexOf("}};")+2)
|
||||
|
||||
JSONObject jsonObject = JSONObject.fromObject(s1);
|
||||
print jsonObject
|
||||
}
|
||||
|
||||
return s
|
||||
|
||||
}
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
checkAbuyun(true);
|
||||
|
||||
def responseMap = [:]
|
||||
// 创建代理服务器地址对象
|
||||
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
|
||||
// 创建HTTP类型代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
|
||||
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
if(content){
|
||||
outputStream.write(content.getBytes());
|
||||
}
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static checkAbuyun(boolean check) {
|
||||
if (check) {
|
||||
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
AbuyunRequestCount.incrementAndGet();
|
||||
} else {
|
||||
AbuyunRequestCount.decrementAndGet();
|
||||
}
|
||||
}
|
||||
}
|
||||
+715
@@ -0,0 +1,715 @@
|
||||
package com.fangjia.script.companyCrawl
|
||||
|
||||
import com.eclipsesource.v8.V8
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import jdk.nashorn.api.scripting.ScriptObjectMirror
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.Header
|
||||
import org.apache.http.HttpResponse
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
|
||||
import javax.script.ScriptEngine
|
||||
import javax.script.ScriptEngineManager;
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class GsxtCrawl {
|
||||
|
||||
def static final proxyPort = 9020;
|
||||
def static final proxyServer = "http-dyn.abuyun.com";
|
||||
def static final proxyUser = "HZU1XU28H3G4Z44D";
|
||||
def static final proxyPass = "466C86ED77FE43D4";
|
||||
|
||||
public final static int AbuyunMaxCount = 10;
|
||||
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
|
||||
|
||||
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
|
||||
static main(args) {
|
||||
def task = new GsxtCrawl();
|
||||
//task.process()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
getObject()
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
|
||||
get.setHeader("Upgrade-Insecure-Requests", "1");
|
||||
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
|
||||
|
||||
HttpResponse response = client.execute(get);
|
||||
def __jsluid_h = getJsluid(response)
|
||||
def s = getResponseBodyAsString(response)
|
||||
println (s)
|
||||
def __jsl_clearance = getJsContent(s);
|
||||
|
||||
//Cookie: __jsluid_h=26cb8385493c2e424767eca767720c87; __jsl_clearance=1618899529.627|-1|ao39lP6IVSloPMvq5%2FOSBqSiPZ0%3D
|
||||
def newCookie = __jsluid_h+";"+__jsl_clearance
|
||||
get.setHeader("Cookie", newCookie)
|
||||
HttpResponse response2 = client.execute(get);
|
||||
def s2 = getResponseBodyAsString(response2)
|
||||
def __jsl_clearance2 = getJsContent2(s2);
|
||||
println(__jsl_clearance2)
|
||||
|
||||
// fristCrawl()
|
||||
}
|
||||
|
||||
|
||||
//该函数测试Java与js对象
|
||||
static void getObject() throws Exception{
|
||||
String str = "var _0xbe8c = ['w4LCg8Onw7o=', 'ZAjDvCw=', 'YcKOw6Fm', 'wpHCqsO3CQ==', 'UMKCXnw=', 'w7Aww6XCnQ==', 'BcKNwrvCng==', 'J2zCnsKi', 'wq02WMOx', 'RQXDrUw=', 'eCnCnDw=', 'Q8KowpzDjA==', 'YznDucOn', 'YcOrXxk=', 'w4t9PsKM', 'DMO4QmA=', 'w5HCklPDmQ==', 'w5rCv8Omw6Q=', 'NcOwc8K5', 'GUxjw7Q=', 'w7zDicK8w4I=', 'VCDDpsKv', 'cwLChT4=', 'wpvCl1vDqw==', 'wrvCvMOKNQ==', 'RTnDrAc=', 'GMKFw4hk', 'SxnDvC0=', 'H2NHNQ==', 'w6Aqw7LCqg==', 'wq0mW8Ok', 'L1zCv8Kp', 'B8OqwoI2', 'Jl7DuX8=', 'JcOBwpIR', 'U8OnfSo=', '6K2K5rOG6aqL6KyM', 'f8Kiwo/Dhg==', 'aMK6w6vDog==', 'JcKgw4FC', 'w61vJEg=', 'LMOrwoY0', 'GcK4w6lt', 'w4XDrsO5wpY=', 'J8OhwoM1', 'w5Mzw7J/', 'wocbcBA=', 'SibDt8Oj', 'w7xgwoXDkQ==', 'AcKyDEQ=', 'Rg7Dkw==', 'w6UFK8Kw', 'TTzDjcKK', 'w6QhDcKy', 'ZBLDols=', 'ZjPDrk4=', 'w6XDqX3Ctw==', 'w6EvJcKT', 'JHtUDQ==', 'wq8mI8KE', 'JMOhwoJ9', 'MsK1AkE=', 'w7fCmhcm', 'VsK2w5fDtQ==', 'w7EUw4PCnw==', 'dz/DgsKu', 'UzfDtcK0', 'wpd9fWE=', 'worDvcKFw4M=', 'w4PDr0zDhg==', 'w7ocSMKg', 'wp1uwoZd', 'eAjDv8Oz', 'aATCgTw=', 'w7UGw7k=', 'wqZCwotq', 'HFIBw5s=', 'WsOdw7JB', 'w5PDpXHCpA==', 'GMO8wqdD', 'w4TCsQs=', 'w6/DmcODwpg=', 'w7MuwrjDoQ==', 'dDHDhsOj', 'G308w70=', 'CsO6wog9', 'FcONwrVw', 'M3vCmMK4', 'w6/Dm8OvwpI=', 'QsOLw5ZM', 'TgLDicKy', 'QSHCuQA=', 'UsOlw5RI', 'w7EqwqDDrw==', 'w6jDusK5w4k=', 'OsKaw5rDlg==', 'wosLQsOk', 'DcKxw7Vo', 'w5VuIEo=', 'w7sXw6TCqw==', 'wrLDr8KRw7s=', 'wr9+wpJR', 'w4LDscO0wrQ=', 'FFIPw6U=', 'J8Ksw5rDlA==', 'd8KZwqjDgw==', 'PcKvDF4=', 'MHh2GA==', 'dgDDs0Y=', 'wpxSS1Y=', 'w7l5JMKh', 'w7N8JMKC', 'XsKhw7hR', 'QxDDqm4=', 'PcKGw5ZJ', 'w7HCuAIQ', 'E8Kmw53Dmg==', 'CMOrwok3', 'U8K9wpvDnA==', 'IW06w5U=', 'w7DDjcOGwpQ=', 'X8KlwpHDpA==', 'w6TDn8O1wpo=', 'ZjnDv08=', 'w7/Cqjsp', 'wqLDk8KYw58=', 'w6fDvMKXw4g=', 'FsK8IUY=', 'w4knw4Rh', 'w5zCu8Ogw5M=', 'wqJ0ckc=', 'T8Kgw69G', 'GF0hw74=', 'FMOkwqkH', 'wokmbMOM', 'djHDtcKW', 'FXkow64=', 'B8OHwowH', 'VSjCpQU=', 'w5vDs8K0w4s=', 'w7vCkUV1', 'w7dMIcK8', 'WMOjZyI=', 'LcKBw7po', 'BMK7HkY=', 'B8K0w7F4', 'YsK+w67Drw==', 'HlQKw7I=', 'wpLCrMKrNg==', 'w7IjcsKy', 'w6zDusKhw4o=', 'WhPDm8Om', 'dMKlw7RO', 'dcOHw5R3', 'Nkpww7M=', 'wrnCsMKLdg==', 'dDTDqMKb', 'S8KXZXE=', 'wq/CssOpJA==', 'w7J/KFw=', 'QMKSWEo=', 'woNxbEc=', 'wqVIwow=', 'TD/Dqg4=', 'w7TDpmzCiw==', 'w7HDjWDCgA==', 'w7IEw6vCoA==', 'Q8K+w4zDiQ==', 'PElvw7k=', 'cDXDsRo=', 'FMOLX0c=', 'Z8KHwojDoA==', 'TTTCqzo=', 'woXCrsK6Uw==', 'w4lhHnY=', 'w6Q/dsKj', 'KsKIwpLCtQ==', 'QQTDn8Oy', 'D8Omw7zDoQ==', 'E8KzGk0=']; (function(_0x44c192, _0xbe8c61) {\n" +
|
||||
"\t\tvar _0x1533fb = function(_0x5b499f) {\n" +
|
||||
"\t\t\twhile (--_0x5b499f) {\n" +
|
||||
"\t\t\t\t_0x44c192['push'](_0x44c192['shift']());\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x1533fb(++_0xbe8c61);\n" +
|
||||
"\t} (_0xbe8c, 0x185));\n" +
|
||||
"\tvar _0x1533 = function(_0x44c192, _0xbe8c61) {\n" +
|
||||
"\t\t_0x44c192 = _0x44c192 - 0x0;\n" +
|
||||
"\t\tvar _0x1533fb = _0xbe8c[_0x44c192];\n" +
|
||||
"\t\tif (_0x1533['GlBvhA'] === undefined) { (function() {\n" +
|
||||
"\t\t\t\tvar _0xa131c3 = function() {\n" +
|
||||
"\t\t\t\t\tvar _0x9829e5;\n" +
|
||||
"\t\t\t\t\ttry {\n" +
|
||||
"\t\t\t\t\t\t_0x9829e5 = Function('return\\x20(function()\\x20' + '{}.constructor(\\x22return\\x20this\\x22)(\\x20)' + ');')();\n" +
|
||||
"\t\t\t\t\t} catch(_0x1ba3fa) {\n" +
|
||||
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t\treturn _0x9829e5;\n" +
|
||||
"\t\t\t\t};\n" +
|
||||
"\t\t\t\tvar _0x5e86d6 = _0xa131c3();\n" +
|
||||
"\t\t\t\tvar _0x67fb0c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';\n" +
|
||||
"\t\t\t\t_0x5e86d6['atob'] || (_0x5e86d6['atob'] = function(_0x1cb044) {\n" +
|
||||
"\t\t\t\t\tvar _0xa69f93 = String(_0x1cb044)['replace'](/=+\$/, '');\n" +
|
||||
"\t\t\t\t\tvar _0x26bfb9 = '';\n" +
|
||||
"\t\t\t\t\tfor (var _0x2f1d79 = 0x0,\n" +
|
||||
"\t\t\t\t\t_0x3713bb, _0x4e514b, _0x29733b = 0x0; _0x4e514b = _0xa69f93['charAt'](_0x29733b++);~_0x4e514b && (_0x3713bb = _0x2f1d79 % 0x4 ? _0x3713bb * 0x40 + _0x4e514b: _0x4e514b, _0x2f1d79++%0x4) ? _0x26bfb9 += String['fromCharCode'](0xff & _0x3713bb >> ( - 0x2 * _0x2f1d79 & 0x6)) : 0x0) {\n" +
|
||||
"\t\t\t\t\t\t_0x4e514b = _0x67fb0c['indexOf'](_0x4e514b);\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t\treturn _0x26bfb9;\n" +
|
||||
"\t\t\t\t});\n" +
|
||||
"\t\t\t} ());\n" +
|
||||
"\t\t\tvar _0x38a7ff = function(_0x5dd34b, _0x3a0f2e) {\n" +
|
||||
"\t\t\t\tvar _0x3e7a99 = [],\n" +
|
||||
"\t\t\t\t_0x3e2c47 = 0x0,\n" +
|
||||
"\t\t\t\t_0x383663,\n" +
|
||||
"\t\t\t\t_0x14720e = '',\n" +
|
||||
"\t\t\t\t_0x5864f4 = '';\n" +
|
||||
"\t\t\t\t_0x5dd34b = atob(_0x5dd34b);\n" +
|
||||
"\t\t\t\tfor (var _0x28a6d2 = 0x0,\n" +
|
||||
"\t\t\t\t_0x2ea622 = _0x5dd34b['length']; _0x28a6d2 < _0x2ea622; _0x28a6d2++) {\n" +
|
||||
"\t\t\t\t\t_0x5864f4 += '%' + ('00' + _0x5dd34b['charCodeAt'](_0x28a6d2)['toString'](0x10))['slice']( - 0x2);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\t_0x5dd34b = decodeURIComponent(_0x5864f4);\n" +
|
||||
"\t\t\t\tvar _0x3091fb;\n" +
|
||||
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3091fb;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
|
||||
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb] + _0x3a0f2e['charCodeAt'](_0x3091fb % _0x3a0f2e['length'])) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\t_0x3091fb = 0x0;\n" +
|
||||
"\t\t\t\t_0x3e2c47 = 0x0;\n" +
|
||||
"\t\t\t\tfor (var _0x8c68bc = 0x0; _0x8c68bc < _0x5dd34b['length']; _0x8c68bc++) {\n" +
|
||||
"\t\t\t\t\t_0x3091fb = (_0x3091fb + 0x1) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb]) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
|
||||
"\t\t\t\t\t_0x14720e += String['fromCharCode'](_0x5dd34b['charCodeAt'](_0x8c68bc) ^ _0x3e7a99[(_0x3e7a99[_0x3091fb] + _0x3e7a99[_0x3e2c47]) % 0x100]);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\treturn _0x14720e;\n" +
|
||||
"\t\t\t};\n" +
|
||||
"\t\t\t_0x1533['sMWqUz'] = _0x38a7ff;\n" +
|
||||
"\t\t\t_0x1533['AbOzCI'] = {};\n" +
|
||||
"\t\t\t_0x1533['GlBvhA'] = !![];\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x5b499f = _0x1533['AbOzCI'][_0x44c192];\n" +
|
||||
"\t\tif (_0x5b499f === undefined) {\n" +
|
||||
"\t\t\tif (_0x1533['uRkTAI'] === undefined) {\n" +
|
||||
"\t\t\t\t_0x1533['uRkTAI'] = !![];\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x1533fb = _0x1533['sMWqUz'](_0x1533fb, _0xbe8c61);\n" +
|
||||
"\t\t\t_0x1533['AbOzCI'][_0x44c192] = _0x1533fb;\n" +
|
||||
"\t\t} else {\n" +
|
||||
"\t\t\t_0x1533fb = _0x5b499f;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\treturn _0x1533fb;\n" +
|
||||
"\t};\n" +
|
||||
"\tfunction hash(_0x1a5a87) {\n" +
|
||||
"\t\tvar _0x23a3e0 = {};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x93', 'Kp2R') + 'J'] = function(_0x5ebeb2, _0x4681fc) {\n" +
|
||||
"\t\t\treturn _0x5ebeb2 ^ _0x4681fc;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x17', '(bZ%') + 'H'] = function(_0x5108c3, _0x49fe65) {\n" +
|
||||
"\t\t\treturn _0x5108c3 & _0x49fe65;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x88', 'N#(C') + 'T'] = function(_0x3d3f10, _0x3f2a86) {\n" +
|
||||
"\t\t\treturn _0x3d3f10 !== _0x3f2a86;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x60', '(cNt') + 'Z'] = _0x1533('0x70', 'K)PG') + 'J';\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x49', 'Y8u@') + 'h'] = function(_0xc3ff68, _0x129ab0) {\n" +
|
||||
"\t\t\treturn _0xc3ff68 >> _0x129ab0;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x9c', '\$qIi') + 'S'] = function(_0x591dc1, _0x158d9d) {\n" +
|
||||
"\t\t\treturn _0x591dc1 + _0x158d9d;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x7f', '(bZ%') + 'f'] = function(_0xd6deb, _0x557760) {\n" +
|
||||
"\t\t\treturn _0xd6deb >> _0x557760;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x4e', 'S9Bn') + 'd'] = function(_0xd05226, _0x29b6e8) {\n" +
|
||||
"\t\t\treturn _0xd05226 * _0x29b6e8;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x7e', 'y^BN') + 'n'] = function(_0x899f8d, _0x3dbd26) {\n" +
|
||||
"\t\t\treturn _0x899f8d < _0x3dbd26;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x9a', '(U[3') + 'e'] = function(_0x39678a, _0x37d488) {\n" +
|
||||
"\t\t\treturn _0x39678a * _0x37d488;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xa3', 'sahq') + 'U'] = function(_0x4cf877, _0x14e1b0) {\n" +
|
||||
"\t\t\treturn _0x4cf877 << _0x14e1b0;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xa7', 'K#Hh') + 'u'] = function(_0x556336, _0x56b41c) {\n" +
|
||||
"\t\t\treturn _0x556336 - _0x56b41c;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x3d', '(bZ%') + 'p'] = function(_0x372c20, _0x534fe1) {\n" +
|
||||
"\t\t\treturn _0x372c20 >> _0x534fe1;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x1d', 'S9Bn') + 'g'] = function(_0x26a0f8, _0x4f1660) {\n" +
|
||||
"\t\t\treturn _0x26a0f8 >>> _0x4f1660;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xae', ']*f@') + 'Y'] = function(_0x3423f4, _0x53e1a9) {\n" +
|
||||
"\t\t\treturn _0x3423f4 | _0x53e1a9;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x26', 's4Dp') + 'P'] = function(_0x19382a, _0x220110) {\n" +
|
||||
"\t\t\treturn _0x19382a | _0x220110;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x65', 's4Dp') + 'p'] = function(_0x7b95bc, _0x22e933) {\n" +
|
||||
"\t\t\treturn _0x7b95bc & _0x22e933;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x0', '\$qIi') + 'V'] = function(_0x13269b, _0x227e3b) {\n" +
|
||||
"\t\t\treturn _0x13269b ^ _0x227e3b;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x29', 'J21I') + 'R'] = function(_0x1ba947, _0x4b40e4) {\n" +
|
||||
"\t\t\treturn _0x1ba947(_0x4b40e4);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x50', 's4Dp') + 'n'] = _0x1533('0xa', 'qRqa') + 'e';\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x42', 'ksVw') + 'b'] = function(_0x466fa6, _0x3ce8da) {\n" +
|
||||
"\t\t\treturn _0x466fa6 ^ _0x3ce8da;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x2c', 'rqYb') + 'f'] = function(_0x1e608a, _0x241888) {\n" +
|
||||
"\t\t\treturn _0x1e608a - _0x241888;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x78', 'J21I') + 'u'] = function(_0x564b78, _0x18d236, _0x8fd805) {\n" +
|
||||
"\t\t\treturn _0x564b78(_0x18d236, _0x8fd805);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x69', 'zx(Q') + 'D'] = function(_0x3b7666, _0x47f238, _0x57b05e) {\n" +
|
||||
"\t\t\treturn _0x3b7666(_0x47f238, _0x57b05e);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x5f', 'ZaCV') + 'm'] = function(_0x452fa7, _0x493d76, _0x1446e7, _0x4af814, _0x5e0f59) {\n" +
|
||||
"\t\t\treturn _0x452fa7(_0x493d76, _0x1446e7, _0x4af814, _0x5e0f59);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x22', 'Rh@#') + 'W'] = function(_0x51a858, _0x440b1c) {\n" +
|
||||
"\t\t\treturn _0x51a858(_0x440b1c);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x8', 'xPGU') + 'A'] = function(_0x5e96f2, _0x167b41) {\n" +
|
||||
"\t\t\treturn _0x5e96f2 + _0x167b41;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x6d', 'rqYb') + 'X'] = function(_0x46cd37, _0x5d8684) {\n" +
|
||||
"\t\t\treturn _0x46cd37 + _0x5d8684;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x3e', 'rqYb') + 'g'] = function(_0x26bc4b, _0x16bc46) {\n" +
|
||||
"\t\t\treturn _0x26bc4b(_0x16bc46);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tvar _0x179134 = _0x23a3e0;\n" +
|
||||
"\t\tfunction _0x5be8c0(_0x296dd6, _0x778566) {\n" +
|
||||
"\t\t\treturn _0x179134[_0x1533('0x56', 'ZaCV') + 'J'](_0x179134[_0x1533('0x7c', 'o&Q4') + 'H'](_0x296dd6, 0x7fffffff) + (_0x778566 & 0x7fffffff), _0x179134[_0x1533('0x52', 'K)PG') + 'H'](_0x296dd6, 0x80000000)) ^ _0x778566 & 0x80000000;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x225a28(_0x2f57af) {\n" +
|
||||
"\t\t\tvar _0x3c49d8 = {};\n" +
|
||||
"\t\t\t_0x3c49d8[_0x1533('0x55', 'qRqa') + 'N'] = function(_0x15bfb8, _0x51e1eb) {\n" +
|
||||
"\t\t\t\treturn _0x15bfb8 < _0x51e1eb;\n" +
|
||||
"\t\t\t};\n" +
|
||||
"\t\t\tvar _0x567e0f = _0x3c49d8;\n" +
|
||||
"\t\t\tvar _0x5b2e48 = _0x1533('0xa4', 'rQf(') + _0x1533('0x4', 'rqYb') + _0x1533('0x95', 'Rh@#') + _0x1533('0x3', 'OhQr');\n" +
|
||||
"\t\t\tvar _0x321f21 = '';\n" +
|
||||
"\t\t\tfor (var _0x2a5528 = 0x7; _0x2a5528 >= 0x0; _0x2a5528--) {\n" +
|
||||
"\t\t\t\tif (_0x179134[_0x1533('0xb2', 'Gfc!') + 'T'](_0x1533('0x2', 'w*Nn') + 'h', _0x179134[_0x1533('0x6c', 'OhQr') + 'Z'])) {\n" +
|
||||
"\t\t\t\t\t_0x321f21 += _0x5b2e48[_0x1533('0x66', ')T5g') + 'At'](_0x179134[_0x1533('0x9f', 'zx(Q') + 'H'](_0x179134[_0x1533('0x5', 'OhQr') + 'h'](_0x2f57af, _0x2a5528 * 0x4), 0xf));\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\treturn _0x567e0f[_0x1533('0x5a', 'y*wC') + 'N'](t, 0x14) ? 0x5a827999: t < 0x28 ? 0x6ed9eba1: _0x567e0f[_0x1533('0x43', 'zx(Q') + 'N'](t, 0x3c) ? -0x70e44324: -0x359d3e2a;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\treturn _0x321f21;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0xde630f(_0x10fb9a) {\n" +
|
||||
"\t\t\tvar _0x5b5b6d = _0x179134[_0x1533('0x16', 'GvA9') + 'S'](_0x179134[_0x1533('0x87', 'N#(C') + 'f'](_0x179134[_0x1533('0x83', '[H[u') + 'S'](_0x10fb9a[_0x1533('0x58', 'OhQr') + 'th'], 0x8), 0x6), 0x1),\n" +
|
||||
"\t\t\t_0x797793 = new Array(_0x179134[_0x1533('0x59', 'xPGU') + 'd'](_0x5b5b6d, 0x10));\n" +
|
||||
"\t\t\tfor (var _0x109346 = 0x0; _0x179134[_0x1533('0x61', 'dgE&') + 'n'](_0x109346, _0x179134[_0x1533('0x90', 'y^BN') + 'e'](_0x5b5b6d, 0x10)); _0x109346++) {\n" +
|
||||
"\t\t\t\tif (_0x179134[_0x1533('0x2a', 'NNf8') + 'T'](_0x1533('0x12', 'sahq') + 'y', _0x1533('0x67', 'y*wC') + 'G')) {\n" +
|
||||
"\t\t\t\t\t_0x797793[_0x109346] = 0x0;\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\tfor (_0x109346 = 0x0; _0x109346 < _0x10fb9a[_0x1533('0x40', 'ad*k') + 'th']; _0x109346++) {\n" +
|
||||
"\t\t\t\t_0x797793[_0x109346 >> 0x2] |= _0x179134[_0x1533('0xa9', '@gLq') + 'U'](_0x10fb9a[_0x1533('0x1c', 'GvA9') + _0x1533('0xf', 'OhQr') + 'At'](_0x109346), _0x179134[_0x1533('0xb3', 'S9Bn') + 'u'](0x18, (_0x109346 & 0x3) * 0x8));\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x797793[_0x179134[_0x1533('0xb1', 'qRqa') + 'p'](_0x109346, 0x2)] |= 0x80 << 0x18 - (_0x109346 & 0x3) * 0x8;\n" +
|
||||
"\t\t\t_0x797793[_0x5b5b6d * 0x10 - 0x1] = _0x10fb9a[_0x1533('0xb', '@mqx') + 'th'] * 0x8;\n" +
|
||||
"\t\t\treturn _0x797793;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x50e067(_0x45a1cc, _0x3bcee1) {\n" +
|
||||
"\t\t\treturn _0x45a1cc << _0x3bcee1 | _0x179134[_0x1533('0x5b', 'Y8u@') + 'g'](_0x45a1cc, _0x179134[_0x1533('0x91', '@mqx') + 'u'](0x20, _0x3bcee1));\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x526d3b(_0xb72077, _0xed7f25, _0x5bc269, _0x572b62) {\n" +
|
||||
"\t\t\tif (_0xb72077 < 0x14) return _0x179134[_0x1533('0x98', 'Gfc!') + 'Y'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x68', 'OhQr') + 'H'](~_0xed7f25, _0x572b62));\n" +
|
||||
"\t\t\tif (_0x179134[_0x1533('0x9e', 'w*Nn') + 'n'](_0xb72077, 0x28)) return _0x179134[_0x1533('0x99', ')T5g') + 'J'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
|
||||
"\t\t\tif (_0xb72077 < 0x3c) return _0x179134[_0x1533('0x13', 'CNzQ') + 'P'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x45', '@mqx') + 'p'](_0xed7f25, _0x572b62)) | _0x179134[_0x1533('0xa0', 'avK7') + 'p'](_0x5bc269, _0x572b62);\n" +
|
||||
"\t\t\treturn _0x179134[_0x1533('0x5e', 'avK7') + 'V'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x496785(_0x583c97) {\n" +
|
||||
"\t\t\treturn _0x583c97 < 0x14 ? 0x5a827999: _0x583c97 < 0x28 ? 0x6ed9eba1: _0x583c97 < 0x3c ? -0x70e44324: -0x359d3e2a;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x45b840 = _0x179134[_0x1533('0x39', 'R#BF') + 'R'](_0xde630f, _0x1a5a87);\n" +
|
||||
"\t\tvar _0x53258d = new Array(0x50);\n" +
|
||||
"\t\tvar _0xc86ce6 = 0x67452301;\n" +
|
||||
"\t\tvar _0x3ee50e = -0x10325477;\n" +
|
||||
"\t\tvar _0xdb1008 = -0x67452302;\n" +
|
||||
"\t\tvar _0xe99bcb = 0x10325476;\n" +
|
||||
"\t\tvar _0x56c860 = -0x3c2d1e10;\n" +
|
||||
"\t\tfor (var _0x4c314c = 0x0; _0x4c314c < _0x45b840[_0x1533('0x46', '\$qIi') + 'th']; _0x4c314c += 0x10) {\n" +
|
||||
"\t\t\tvar _0x41e8e8 = _0xc86ce6;\n" +
|
||||
"\t\t\tvar _0x1b7277 = _0x3ee50e;\n" +
|
||||
"\t\t\tvar _0x30f7c3 = _0xdb1008;\n" +
|
||||
"\t\t\tvar _0x279c95 = _0xe99bcb;\n" +
|
||||
"\t\t\tvar _0xf6e888 = _0x56c860;\n" +
|
||||
"\t\t\tfor (var _0x77e31b = 0x0; _0x179134[_0x1533('0x34', 'Y8u@') + 'n'](_0x77e31b, 0x50); _0x77e31b++) {\n" +
|
||||
"\t\t\t\tif (_0x77e31b < 0x10) {\n" +
|
||||
"\t\t\t\t\tif (_0x179134[_0x1533('0xa6', 'Mifp') + 'T'](_0x1533('0x3f', 'o&Q4') + 'G', _0x179134[_0x1533('0xa1', 'rqYb') + 'n'])) {\n" +
|
||||
"\t\t\t\t\t\t_0x53258d[_0x77e31b] = _0x45b840[_0x4c314c + _0x77e31b];\n" +
|
||||
"\t\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\t\tsleepT = _0x179134[_0x1533('0x48', 'NNf8') + 'R'](parseInt, data['wt']) > ret[0x1] ? _0x179134[_0x1533('0x97', 'd[i)') + 'R'](parseInt, data['wt']) - ret[0x1] : 0x1f4;\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\t_0x53258d[_0x77e31b] = _0x50e067(_0x179134[_0x1533('0x96', 'rQf(') + 'b'](_0x179134[_0x1533('0xb0', 'Gfc!') + 'b'](_0x53258d[_0x179134[_0x1533('0xa5', 'K)PG') + 'u'](_0x77e31b, 0x3)], _0x53258d[_0x179134[_0x1533('0x51', 'K)PG') + 'f'](_0x77e31b, 0x8)]) ^ _0x53258d[_0x77e31b - 0xe], _0x53258d[_0x179134[_0x1533('0xaf', '(U[3') + 'f'](_0x77e31b, 0x10)]), 0x1);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\tt = _0x179134[_0x1533('0x18', 'GvA9') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x23', '\$qIi') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x8d', 'Mifp') + 'D'](_0x50e067, _0xc86ce6, 0x5), _0x179134[_0x1533('0x4c', 'xPGU') + 'm'](_0x526d3b, _0x77e31b, _0x3ee50e, _0xdb1008, _0xe99bcb)), _0x5be8c0(_0x5be8c0(_0x56c860, _0x53258d[_0x77e31b]), _0x179134[_0x1533('0xad', 'rqYb') + 'W'](_0x496785, _0x77e31b)));\n" +
|
||||
"\t\t\t\t_0x56c860 = _0xe99bcb;\n" +
|
||||
"\t\t\t\t_0xe99bcb = _0xdb1008;\n" +
|
||||
"\t\t\t\t_0xdb1008 = _0x50e067(_0x3ee50e, 0x1e);\n" +
|
||||
"\t\t\t\t_0x3ee50e = _0xc86ce6;\n" +
|
||||
"\t\t\t\t_0xc86ce6 = t;\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0xc86ce6 = _0x5be8c0(_0xc86ce6, _0x41e8e8);\n" +
|
||||
"\t\t\t_0x3ee50e = _0x5be8c0(_0x3ee50e, _0x1b7277);\n" +
|
||||
"\t\t\t_0xdb1008 = _0x5be8c0(_0xdb1008, _0x30f7c3);\n" +
|
||||
"\t\t\t_0xe99bcb = _0x5be8c0(_0xe99bcb, _0x279c95);\n" +
|
||||
"\t\t\t_0x56c860 = _0x5be8c0(_0x56c860, _0xf6e888);\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\treturn _0x179134[_0x1533('0x86', 'Gfc!') + 'A'](_0x179134[_0x1533('0x44', 'DDSy') + 'X'](_0x179134[_0x1533('0x8e', 'xPGU') + 'g'](_0x225a28, _0xc86ce6), _0x179134[_0x1533('0x77', 'n*Rs') + 'g'](_0x225a28, _0x3ee50e)) + _0x225a28(_0xdb1008) + _0x225a28(_0xe99bcb), _0x179134[_0x1533('0xd', 'DDSy') + 'g'](_0x225a28, _0x56c860));\n" +
|
||||
"\t}\n" +
|
||||
"\tfunction go(_0x394549) {\n" +
|
||||
"\t\tvar _0x33d0d6 = {};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x2b', 'sahq') + 'H'] = _0x1533('0x14', 'rQf(') + _0x1533('0x15', 'sahq');\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0xab', 'dgE&') + 'H'] = function(_0x23f209, _0x4ed415) {\n" +
|
||||
"\t\t\treturn _0x23f209 < _0x4ed415;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x5c', 'xPGU') + 'g'] = function(_0x39bd0a, _0x4d5764) {\n" +
|
||||
"\t\t\treturn _0x39bd0a != _0x4d5764;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x37', 'y*wC') + 'N'] = function(_0x3c9a26, _0x109b89) {\n" +
|
||||
"\t\t\treturn _0x3c9a26 + _0x109b89;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x63', 'gkl*') + 'z'] = function(_0x584935, _0x58347c) {\n" +
|
||||
"\t\t\treturn _0x584935 + _0x58347c;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x21', 'ZaCV') + 'z'] = function(_0x3fc795, _0x6962f6) {\n" +
|
||||
"\t\t\treturn _0x3fc795 + _0x6962f6;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x27', '(cNt') + 'I'] = function(_0x303446, _0x3388fb) {\n" +
|
||||
"\t\t\treturn _0x303446 == _0x3388fb;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x6b', 'y*wC') + 't'] = function(_0x5cb515, _0x31e7f3) {\n" +
|
||||
"\t\t\treturn _0x5cb515(_0x31e7f3);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x75', 'Rh@#') + 'E'] = function(_0x529d8e, _0xc1423a) {\n" +
|
||||
"\t\t\treturn _0x529d8e + _0xc1423a;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x62', 'rQf(') + 'a'] = function(_0x453505, _0x50cd56) {\n" +
|
||||
"\t\t\treturn _0x453505 + _0x50cd56;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x2e', 'NNf8') + 'B'] = _0x1533('0x28', ']*f@') + _0x1533('0x57', 'Kp2R') + '\\x20/';\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0xe', 'Y8u@') + 'y'] = function(_0x25bcfd, _0x4b634e, _0x1bef22) {\n" +
|
||||
"\t\t\treturn _0x25bcfd(_0x4b634e, _0x1bef22);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x92', 'J21I') + 'i'] = function(_0xccfc4d, _0x9a8620) {\n" +
|
||||
"\t\t\treturn _0xccfc4d > _0x9a8620;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x11', '@H8e') + 'n'] = function(_0x1b9fc7, _0x7d3ce) {\n" +
|
||||
"\t\t\treturn _0x1b9fc7(_0x7d3ce);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x1b', 'N#(C') + 'O'] = _0x1533('0x7', 'N#(C') + '失败';\n" +
|
||||
"\t\tvar _0x2eda7d = _0x33d0d6;\n" +
|
||||
"\t\tfunction _0x29e502() {\n" +
|
||||
"\t\t\tvar _0x24a3c1 = window[_0x1533('0x35', 'ad*k') + _0x1533('0x5d', 'Y8u@') + 'r'][_0x1533('0x19', 'avK7') + _0x1533('0x7d', 'cJ3U') + 't'],\n" +
|
||||
"\t\t\t_0x5f1192 = [_0x2eda7d[_0x1533('0x24', '(bZ%') + 'H']];\n" +
|
||||
"\t\t\tfor (var _0x2d8686 = 0x0; _0x2eda7d[_0x1533('0x1a', 'avK7') + 'H'](_0x2d8686, _0x5f1192[_0x1533('0x3b', 'Y8u@') + 'th']); _0x2d8686++) {\n" +
|
||||
"\t\t\t\tif (_0x2eda7d[_0x1533('0x2f', 'y*wC') + 'g'](_0x24a3c1[_0x1533('0x76', 'y*wC') + _0x1533('0x33', 'ZaCV')](_0x5f1192[_0x2d8686]), -0x1)) {\n" +
|
||||
"\t\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\tif (window[_0x1533('0x41', 'dgE&') + _0x1533('0x6e', 'dgE&') + _0x1533('0x85', 'NNf8')] || window[_0x1533('0x30', 'o&Q4') + _0x1533('0x38', 'OhQr')] || window[_0x1533('0xc', 'OhQr') + _0x1533('0x74', 'qRqa')] || window[_0x1533('0x25', '(bZ%') + _0x1533('0x4f', 'avK7') + 'r'][_0x1533('0xa2', 'xPGU') + _0x1533('0x53', ')T5g') + 'r'] || window[_0x1533('0xaa', 'cJ3U') + _0x1533('0x6', 'K#@S') + 'r'][_0x1533('0x8f', 'rqYb') + _0x1533('0xac', '(bZ%') + _0x1533('0x9', 'Rh@#') + _0x1533('0x4d', 'rQf(') + 'e'] || window[_0x1533('0x31', 'N#(C') + _0x1533('0x79', 'dgE&') + 'r'][_0x1533('0x4b', 'ksVw') + _0x1533('0x10', 'gkl*') + _0x1533('0xa8', 'd[i)') + _0x1533('0x89', '\$qIi') + _0x1533('0x84', 's4Dp')]) {\n" +
|
||||
"\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tif (_0x29e502()) {\n" +
|
||||
"\t\t\treturn;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x1642ef = new Date();\n" +
|
||||
"\t\tfunction _0x1a1fd9(_0x55baea, _0x579edb) {\n" +
|
||||
"\t\t\tvar _0x551ac6 = _0x394549[_0x1533('0x4a', 'y*wC') + 's'][_0x1533('0x71', 'K#@S') + 'th'];\n" +
|
||||
"\t\t\tfor (var _0x54ccec = 0x0; _0x2eda7d[_0x1533('0x6f', 'dFiI') + 'H'](_0x54ccec, _0x551ac6); _0x54ccec++) {\n" +
|
||||
"\t\t\t\tfor (var _0x562c2d = 0x0; _0x2eda7d[_0x1533('0x9b', '[H[u') + 'H'](_0x562c2d, _0x551ac6); _0x562c2d++) {\n" +
|
||||
"\t\t\t\t\tvar _0x1b8d8c = _0x2eda7d[_0x1533('0x80', '[H[u') + 'N'](_0x2eda7d[_0x1533('0x3a', 'w*Nn') + 'z'](_0x2eda7d[_0x1533('0x8b', 'cJ3U') + 'z'](_0x579edb[0x0], _0x394549[_0x1533('0x32', 'R#BF') + 's'][_0x1533('0x82', '@mqx') + 'tr'](_0x54ccec, 0x1)), _0x394549[_0x1533('0x1f', '*hmq') + 's'][_0x1533('0x47', '(cNt') + 'tr'](_0x562c2d, 0x1)), _0x579edb[0x1]);\n" +
|
||||
"\t\t\t\t\tif (_0x2eda7d[_0x1533('0x6a', '(bZ%') + 'I'](_0x2eda7d[_0x1533('0x9d', 'Kp2R') + 't'](hash, _0x1b8d8c), _0x55baea)) {\n" +
|
||||
"\t\t\t\t\t\treturn [_0x1b8d8c, new Date() - _0x1642ef];\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tvar _0x1e73b9 = _0x2eda7d[_0x1533('0x36', 'sahq') + 'y'](_0x1a1fd9, _0x394549['ct'], _0x394549[_0x1533('0x2d', '\$qIi')]);\n" +
|
||||
"\t\tif (_0x1e73b9) {\n" +
|
||||
"\t\t\tvar _0x3b1427;\n" +
|
||||
"\t\t\tif (_0x394549['wt']) {\n" +
|
||||
"\t\t\t\t_0x3b1427 = _0x2eda7d[_0x1533('0x1', 'zx(Q') + 'i'](parseInt(_0x394549['wt']), _0x1e73b9[0x1]) ? _0x2eda7d[_0x1533('0x54', 'avK7') + 'n'](parseInt, _0x394549['wt']) - _0x1e73b9[0x1] : 0x1f4;\n" +
|
||||
"\t\t\t} else {\n" +
|
||||
"\t\t\t\t_0x3b1427 = 0x5dc;\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x2eda7d[_0x1533('0x72', 'qRqa') + 'y'](setTimeout,\n" +
|
||||
"\t\t\tfunction() {\n" +
|
||||
"\t\t\t\tdocument[_0x1533('0x20', 'rQf(') + 'ie'] = _0x2eda7d[_0x1533('0x73', 'rQf(') + 'E'](_0x2eda7d[_0x1533('0x3c', 'o&Q4') + 'a'](_0x394549['tn'] + '=' + _0x1e73b9[0x0] + (_0x1533('0x8a', 'ksVw') + _0x1533('0x1e', 'GvA9') + '='), _0x394549['vt']), _0x2eda7d[_0x1533('0x8c', 'Gfc!') + 'B']);\n" +
|
||||
"\t\t\t\tlocation[_0x1533('0x7a', 'sahq')] = location[_0x1533('0x64', 'd[i)') + _0x1533('0x81', '(U[3')] + location[_0x1533('0x94', 'sahq') + 'ch'];\n" +
|
||||
"\t\t\t},\n" +
|
||||
"\t\t\t_0x3b1427);\n" +
|
||||
"\t\t} else {\n" +
|
||||
"\t\t\talert(_0x2eda7d[_0x1533('0x7b', ')T5g') + 'O']);\n" +
|
||||
"\t\t}\n" +
|
||||
"\t};\n" +
|
||||
"\tgo({\n" +
|
||||
"\t\t\"bts\": [\"1618915727.945|0|HM5\", \"VoVNIPqcrvFO13woIZK2RY%3D\"],\n" +
|
||||
"\t\t\"chars\": \"ZDUZR0cDjTEXznMkfgavFO\",\n" +
|
||||
"\t\t\"ct\": \"3049e28877124ad21ca0db55e3af95ff316ceef6\",\n" +
|
||||
"\t\t\"ha\": \"sha1\",\n" +
|
||||
"\t\t\"tn\": \"__jsl_clearance\",\n" +
|
||||
"\t\t\"vt\": \"3600\",\n" +
|
||||
"\t\t\"wt\": \"1500\"\n" +
|
||||
"\t})"
|
||||
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
engine.eval(str);
|
||||
//获取对象
|
||||
System.out.println();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getJsContent(def page){
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
//得到脚本引擎
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
//处理加密js
|
||||
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
|
||||
try {
|
||||
//得到解密后的js
|
||||
js = js.replace("document.cookie","var cookie")
|
||||
js = js.replace("location.href=location.pathname+location.search", "")
|
||||
engine.eval(js);
|
||||
def v = engine.get("cookie")
|
||||
return v.toString().substring(0, v.toString().indexOf(";"))
|
||||
} catch (ScriptException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getJsContent2(def page){
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
//得到脚本引擎
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
//处理加密js
|
||||
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
|
||||
try {
|
||||
//得到解密后的js
|
||||
//得到解密后的js
|
||||
String result = (String) engine.eval(js);
|
||||
result = result.substring(result.indexOf("var cd"),result.indexOf("dc+=cd;")+7);
|
||||
result = result.replaceAll("document*.*toLowerCase\\(\\)", "'x'");
|
||||
//得到cookie
|
||||
String jsl = (String) engine.eval(result);
|
||||
return jsl;
|
||||
} catch (ScriptException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 通过破解动态JavaScript脚本,
|
||||
* 获取cookie名为 __jsl_clearance的值
|
||||
*
|
||||
* @param body 相应内容(一般为第一次请求获取到的动态js字符串)
|
||||
* @return cookie名为 __jsl_clearance的值
|
||||
*/
|
||||
private static String getJslClearance(String body) {
|
||||
//V8:谷歌开源的运行JavaScript脚本的库. 参数:globalAlias=window, 表示window为全局别名,
|
||||
// 告诉V8在运行JavaScript代码时, 不要从代码里找window的定义.
|
||||
V8 runtime = V8.createV8Runtime("window");
|
||||
//将第一次请求pdf资源时获取到的字符串提取成V8可执行的JavaScript代码
|
||||
body = body.trim()
|
||||
.replace("<script>", "")
|
||||
.replace("</script>", "")
|
||||
.replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))",
|
||||
"y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})");
|
||||
//用V8执行该段代码获取新的动态JavaScript脚本
|
||||
String result = runtime.executeStringScript(body);
|
||||
|
||||
//获取 jsl_clearance 的第一段, 格式形如: 1543915851.312|0|
|
||||
String startStr = "document.cookie='";
|
||||
int i1 = result.indexOf(startStr) + startStr.length();
|
||||
int i2 = result.indexOf("|0|");
|
||||
String cookie1 = result.substring(i1, i2 + 3);
|
||||
/*
|
||||
获取 jsl_clearance 的第二段,格式形如: DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D。
|
||||
主要原理是: 新的动态JavaScript脚本是为浏览器设置cookie, 且cookie名为__jsl_clearance
|
||||
其中第一段值(格式形如:1543915851.312|0|)已经明文写好, 用字符串处理方法即可获取.
|
||||
第二段则是一段JavaScript函数, 需要有V8运行返回,
|
||||
该函数代码需要通过一些字符串定位, 提取出来, 交给V8运行.
|
||||
*/
|
||||
startStr = "|0|'+(function(){";
|
||||
int i3 = result.indexOf(startStr) + startStr.length();
|
||||
int i4 = result.indexOf("})()+';Expires");
|
||||
String code = result.substring(i3, i4).replace(";return", ";");
|
||||
String cookie2 = runtime.executeStringScript(code);
|
||||
|
||||
/*
|
||||
拼接两段字符串, 返回jsl_clearance的完整的值.
|
||||
格式形如: 1543915851.312|0|DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D
|
||||
*/
|
||||
return cookie1 + cookie2;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 将HTTP响应体转换为字符串返回
|
||||
*
|
||||
* @param response HTTP响应
|
||||
* @return 响应体的字符串形式
|
||||
* @throws IOException IO异常
|
||||
*/
|
||||
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
|
||||
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
|
||||
}
|
||||
|
||||
|
||||
|
||||
static String getJsluid(HttpResponse response) {
|
||||
Header header = response.getFirstHeader("set-cookie");
|
||||
String[] split = header.getValue().split(";");
|
||||
for (String s : split) {
|
||||
if (s.contains("__jsluid")) {
|
||||
return s.trim();
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
|
||||
def fristCrawl(){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","www.gsxt.gov.cn")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
// headers.put("Content-Type","application/x-www-form-urlencoded")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
// headers.put("Accept-Encoding","gzip, deflate")
|
||||
|
||||
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
|
||||
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
|
||||
for(def key : companyKeys){
|
||||
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
|
||||
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
|
||||
def s = crawl(url, queryBody);
|
||||
if(s){
|
||||
print s
|
||||
parser(s);
|
||||
|
||||
}
|
||||
println s
|
||||
}
|
||||
}
|
||||
|
||||
def parser(content){
|
||||
def document = Jsoup.parse(content);
|
||||
if(document){
|
||||
def detailUrls = []
|
||||
Elements elements = JQueryUtils.select([".search_list_item"], document)
|
||||
if(elements && elements.size()>0){
|
||||
for(def el : elements){
|
||||
detailUrls << el.attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def static crawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","www.gsxt.gov.cn")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","226")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("Origin","http://www.gsxt.gov.cn")
|
||||
headers.put("Content-Type","application/x-www-form-urlencoded")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
|
||||
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
|
||||
|
||||
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
checkAbuyun(true);
|
||||
|
||||
def responseMap = [:]
|
||||
// 创建代理服务器地址对象
|
||||
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
|
||||
// 创建HTTP类型代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
|
||||
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
if(content){
|
||||
outputStream.write(content.getBytes());
|
||||
}
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static checkAbuyun(boolean check) {
|
||||
if (check) {
|
||||
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
AbuyunRequestCount.incrementAndGet();
|
||||
} else {
|
||||
AbuyunRequestCount.decrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+17
-9
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
}
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
|
||||
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
|
||||
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
|
||||
|
||||
while(flag){
|
||||
println "开始加载解析数据......"
|
||||
@@ -116,7 +116,7 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
|
||||
count++;
|
||||
}catch (Exception e){
|
||||
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.getMessage()
|
||||
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.printStackTrace()
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
@@ -442,16 +442,21 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
if(regionMap){
|
||||
regionMap.each{k,v->
|
||||
def arrays = k.split("###")
|
||||
if(arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
try{
|
||||
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}else{//无区域
|
||||
cityMap.each{r,dMap->
|
||||
dMap.each{k,v->
|
||||
def arrays = k.split("###")
|
||||
if(arrays[1] && arrays[1]==keyword){
|
||||
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
||||
}
|
||||
}
|
||||
@@ -596,10 +601,13 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def getOpenApiMatch(def city, def region, def name){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
|
||||
if(name){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
+9
-9
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
|
||||
houseLoudng.fitment_cost = ret.fitment_charge
|
||||
houseLoudng.car_port = ret.carport
|
||||
houseLoudng.house_height = ret.house_hight
|
||||
houseLoudng.worry_facility" : null,
|
||||
houseLoudng.fitment_age = ret.fitment_age
|
||||
houseLoudng.lighting_ventilation" : null,
|
||||
houseLoudng.build_type = ret.build_type
|
||||
houseLoudng.house_property" : null,
|
||||
houseLoudng.noise" : null,
|
||||
houseLoudng.sight" : null,
|
||||
houseLoudng.fitment_rank" : null,
|
||||
houseLoudng.sideset" : null,
|
||||
// houseLoudng.worry_facility" : null,
|
||||
// houseLoudng.fitment_age = ret.fitment_age
|
||||
// houseLoudng.lighting_ventilation" : null,
|
||||
// houseLoudng.build_type = ret.build_type
|
||||
// houseLoudng.house_property" : null,
|
||||
// houseLoudng.noise" : null,
|
||||
// houseLoudng.sight" : null,
|
||||
// houseLoudng.fitment_rank" : null,
|
||||
// houseLoudng.sideset" : null,
|
||||
|
||||
|
||||
houseLoudng.district_no = district._id
|
||||
|
||||
+3
-3
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -37,7 +37,7 @@ class HouseExportBatch extends BaseBatch{
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -352,7 +352,7 @@ class HouseExportBatch extends BaseBatch{
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-2020*", queryBuilder);
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "latest_seawater", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
|
||||
+2
-2
@@ -4,13 +4,13 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class AddEsHouseField {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+2
-3
@@ -9,10 +9,9 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
@@ -63,7 +62,7 @@ class AddThirdPriceDistrict {
|
||||
def province = null
|
||||
def resAddrMap = [:]
|
||||
def filterEstateTypeMap = [:]
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.esms.common.util.CommonUtil
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.huaban.analysis.jieba.JiebaSegmenter
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -485,7 +485,7 @@ class CrawlDataMatch {
|
||||
}
|
||||
|
||||
def findAllAddress(def cityName){
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,12 +1,12 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class EsSearchUtils {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -8,10 +8,9 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
@@ -20,7 +19,7 @@ class FZDataExport {
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -42,10 +42,10 @@ class GaoDeMapCrawl {
|
||||
def updateErrorLists = []
|
||||
def flagLists = []
|
||||
def index = 0
|
||||
def prefix = "road_"
|
||||
def prefix = "district_"
|
||||
// def categorys = ["141202", "141203"]//中学,小学
|
||||
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
def categorys = ["190301"]//道路相关
|
||||
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
// def categorys = ["190301"]//道路相关
|
||||
|
||||
static main(args) {
|
||||
def task = new GaoDeMapCrawl();
|
||||
@@ -109,8 +109,8 @@ class GaoDeMapCrawl {
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
|
||||
// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"]
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
def cityLations = findAllLation(city);
|
||||
@@ -198,9 +198,11 @@ class GaoDeMapCrawl {
|
||||
if(m.rets){
|
||||
processData(m.rets);
|
||||
location.count = m.rets.size()
|
||||
location.content = m.toString()
|
||||
flagLists.add(location)
|
||||
}else{
|
||||
location.count = 0
|
||||
location.content = m.toString()
|
||||
flagLists.add(location)
|
||||
}
|
||||
|
||||
@@ -336,7 +338,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : updateLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"status","完成").append("m_date", new Date())
|
||||
def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
@@ -358,7 +360,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : flagLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date())
|
||||
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
@@ -382,7 +384,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : updateErrorLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"status","失败").append("m_date", new Date())
|
||||
def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class LogStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new LogStat()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区名单.xlsx",{ d ->
|
||||
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
|
||||
if(d['A']){
|
||||
def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
if(district){
|
||||
def startDate = DateUtil.addMonth(new Date(), -6)
|
||||
def start = 20200901//startDate.replace("-","").toInteger()
|
||||
def end = 20201201//DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
def count = findHouses(d['A'], d['B'], d['C'], alias, start, end)
|
||||
d['D'] = count
|
||||
println(d)
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
def count = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
count = data.c
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
// def findTestDatas(def city, def value) {
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("user_id", "1007751")
|
||||
// q.put("parameter_map.city", city)
|
||||
// q.put("request_uri", "/property/evaluateForTest")
|
||||
//
|
||||
// def keys = new BasicDBObject()
|
||||
// keys.put("res_map",1)
|
||||
// def cursor = LOG.find(q)
|
||||
// def count = 0
|
||||
// println "总量:"+cursor.count()
|
||||
// def datas = []
|
||||
// for(def d : cursor){
|
||||
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
|
||||
// def resMap = d.res_map.result
|
||||
// if(resMap.calcType==value){
|
||||
// def data = []
|
||||
// def rowMap = [:]
|
||||
// rowMap.city = resMap.city
|
||||
// rowMap.region = resMap.district
|
||||
// rowMap.name = resMap.name
|
||||
// rowMap.calDate = d.parameter_map.calDate
|
||||
// data << rowMap
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return datas;
|
||||
// }
|
||||
|
||||
// for(def i=0;i<datas.size();i++){
|
||||
// def d = datas[i];
|
||||
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
//
|
||||
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// if(district){
|
||||
// def startDate = DateUtil.addMonth(calDate, -6)
|
||||
// def start = startDate.replace("-","").toInteger()
|
||||
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
// def alias = []
|
||||
// if(district.alias){
|
||||
// if(!district.alias.contains(district.name)){
|
||||
// district.alias << district.name
|
||||
// }
|
||||
// alias = district.alias
|
||||
// }else{
|
||||
// alias << district.name
|
||||
// }
|
||||
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
|
||||
// d['G'] = count
|
||||
// }
|
||||
|
||||
|
||||
|
||||
}
|
||||
+2
-3
@@ -7,11 +7,10 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -27,7 +26,7 @@ class NoHouseDistrictStatBatch extends BaseBatch{
|
||||
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -3,14 +3,14 @@ package com.fangjia.script.gj
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class RoomStat {
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -1,30 +1,25 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.idsQuery
|
||||
|
||||
class TradeDataMatch {
|
||||
|
||||
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-4
File diff suppressed because one or more lines are too long
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-02-01"
|
||||
static def statDate = "2021-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package com.fangjia.script.gj.address
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AddressParser {
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddressParser();
|
||||
task.run()
|
||||
// task.gaoDeAddress()
|
||||
}
|
||||
|
||||
def gaoDeAddress(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match2(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def match(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
// if(region && !name.contains(region)){
|
||||
// locationMap = matchLocation(city, region + name, "name")
|
||||
// }else{
|
||||
locationMap = matchLocation(city + region, name, "name")
|
||||
// }
|
||||
// if(!locationMap){
|
||||
// locationMap = matchLocation(city, address, "address")
|
||||
// }
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("school_lat1", locationMap.glat)
|
||||
data.put("school_lng1", locationMap.glng)
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def match2(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
|
||||
locationMap = matchLocation2(city + region, name, "name")
|
||||
|
||||
if(!locationMap){
|
||||
locationMap = matchLocation2(city, address, "address")
|
||||
}
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("gaode_name", locationMap.name)
|
||||
data.put("gaode_address", locationMap.address)
|
||||
if(locationMap.location){
|
||||
def arrs = locationMap.location.split(",")
|
||||
data.put("school_lat", arrs[1].toDouble())
|
||||
data.put("school_lng", arrs[0].toDouble())
|
||||
}
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
def matchLocation(city, address, cat){
|
||||
|
||||
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def datas = geocodes.get(0)
|
||||
if(datas.formatted_address && cat=="name"){
|
||||
if(datas.formatted_address.contains(address)){
|
||||
def glocation = datas.location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}else{
|
||||
return null
|
||||
}
|
||||
}else{
|
||||
def glocation = geocodes.get(0).location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def matchLocation2(city, address, cat){
|
||||
|
||||
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("pois")) {
|
||||
def geocodes = gloJson.get("pois")
|
||||
def datas = geocodes.get(0)
|
||||
return datas
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("school_name","苏州市娄葑实验小学")
|
||||
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DistrictSchoolDataExport {
|
||||
|
||||
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def regionCodeMap = [:]
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSchoolDataExport()
|
||||
// task.run()
|
||||
task.megre()
|
||||
}
|
||||
|
||||
def megre(){
|
||||
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
|
||||
def ids = []
|
||||
for(def d : datas1){
|
||||
ids << d['U']
|
||||
}
|
||||
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
|
||||
for(def d : datas2){
|
||||
def id = d['U']
|
||||
if(!ids.contains(id)){
|
||||
datas1 << d
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "苏州"
|
||||
getRegionCodeByCity(city)
|
||||
getRegionCodeByCity("合肥")
|
||||
getRegionCodeByCity("西安")
|
||||
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts = []
|
||||
districts.addAll(districts0)
|
||||
districts.addAll(districts1)
|
||||
districts.addAll(districts2)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
def rets = []
|
||||
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
|
||||
def i = 0
|
||||
for(def k=0;k<districts.size();k++){
|
||||
final def dist = districts.get(k)
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
|
||||
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
|
||||
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
|
||||
if(primarySchools || midSchools){
|
||||
def info1 = null;
|
||||
def info2 = null;
|
||||
def distance1 = 0
|
||||
def distance2 = 0
|
||||
if(primarySchools && dist.lng){
|
||||
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
|
||||
if(info1 && info1.school_lat2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
|
||||
if(distance1>3000){
|
||||
info1 = null
|
||||
}
|
||||
}else{
|
||||
info1 = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(midSchools) {
|
||||
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
|
||||
if(info2 && info2.school_lng2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
|
||||
if(distance2>3000){
|
||||
info2 = null
|
||||
}
|
||||
}else{
|
||||
info2 = null
|
||||
}
|
||||
}
|
||||
if(info1|| info2){
|
||||
def row = []
|
||||
row << cityCodeMap.get(dist.city)
|
||||
row << regionCodeMap.get(dist.city +"-"+dist.region)
|
||||
row << dist.name
|
||||
row << dist.lng/10000000000.0
|
||||
row << dist.lat/10000000000.0
|
||||
row << dist.address
|
||||
if(info1){
|
||||
row << info1.school_name
|
||||
row << info1.school_address!=null?info1.school_address:''
|
||||
row << info1.attribute!=null?info1.attribute:''
|
||||
row << info1.school_lng2!=null?info1.school_lng2:''
|
||||
row << info1.school_lat2!=null?info1.school_lat2:''
|
||||
// row << distance1
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
|
||||
if(info2){
|
||||
row << info2.school_name
|
||||
row << info2.school_address!=null?info2.school_address:''
|
||||
row << info2.attribute!=null?info2.attribute:''
|
||||
row << info2.school_lng2!=null?info2.school_lng2:''
|
||||
row << info2.school_lat2!=null?info2.school_lat2:''
|
||||
// row << distance2
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
row << ""
|
||||
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << "fjw"
|
||||
row << dist._id.toString()
|
||||
row << "对口"
|
||||
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def selectSchool(lng, lat, def schools){
|
||||
def minSchool = null
|
||||
def level0 = []//国家
|
||||
def level1 = []//省重点
|
||||
def level2 = []//市重点
|
||||
def level3 = []//区重点
|
||||
def level4 = []//普通
|
||||
def level5 = []//其它
|
||||
for(def s : schools){
|
||||
if(s.attribute){
|
||||
if(s.attribute.contains("国家重点")){
|
||||
level0 << s
|
||||
}else if(s.attribute.contains("省重点")){
|
||||
level1 << s
|
||||
}else if(s.attribute.contains("市重点")){
|
||||
level2 << s
|
||||
}else if(s.attribute.contains("区重点")){
|
||||
level3 << s
|
||||
}else if(s.attribute.contains("普通")){
|
||||
level4 << s
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(level0){
|
||||
if(level0.size()==1){
|
||||
minSchool = level0.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level0)
|
||||
}
|
||||
}else if(level1){
|
||||
if(level1.size()==1){
|
||||
minSchool = level1.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level1)
|
||||
}
|
||||
}else if(level2){
|
||||
if(level2.size()==1){
|
||||
minSchool = level2.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level2)
|
||||
}
|
||||
}else if(level3){
|
||||
if(level3.size()==1){
|
||||
minSchool = level3.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level3)
|
||||
}
|
||||
}else if(level4){
|
||||
if(level4.size()==1){
|
||||
minSchool = level4.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level4)
|
||||
}
|
||||
}else if(level5){
|
||||
if(level5.size()==1){
|
||||
minSchool = level5.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level5)
|
||||
}
|
||||
}
|
||||
|
||||
def rets = []
|
||||
|
||||
if(minSchool){
|
||||
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
for(def d : rets){
|
||||
if(!minSchool.school_address){
|
||||
minSchool.school_address = d.address
|
||||
}
|
||||
if(!minSchool.attribute){
|
||||
minSchool.attribute = d.attribute
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(minSchool){
|
||||
if(minSchool.school_map_lng){
|
||||
minSchool.school_lng2 = minSchool.school_map_lng
|
||||
minSchool.school_lat2 = minSchool.school_map_lat
|
||||
}else if(minSchool.school_lng1){
|
||||
minSchool.school_lng2 = minSchool.school_lng1
|
||||
minSchool.school_lat2 = minSchool.school_lat1
|
||||
}else if(minSchool.school_lng){
|
||||
minSchool.school_lng2 = minSchool.school_lng
|
||||
minSchool.school_lat2 = minSchool.school_lat
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return minSchool;
|
||||
}
|
||||
|
||||
|
||||
def findSchoolInfo(def city, def region, def schoolName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", schoolName)
|
||||
def dbcursor = school_col.find(q)
|
||||
def rets = []
|
||||
for(def obj : dbcursor){
|
||||
rets << obj
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def selectMinDistanceSchool(def lat, def lng, def schools){
|
||||
def minSchool = null;
|
||||
def minDistance = Integer.MAX_VALUE
|
||||
for(def s1 : schools){
|
||||
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
if(s1.school_map_lng){
|
||||
lng2 = s1.school_map_lng
|
||||
lat2 = s1.school_map_lat
|
||||
}else if(s1.school_lng1){
|
||||
lng2 = s1.school_lng1
|
||||
lat2 = s1.school_lat1
|
||||
}else if(s1.school_lng){
|
||||
lng2 = s1.school_lng
|
||||
lat2 = s1.school_lat
|
||||
}
|
||||
if(lat2){
|
||||
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
|
||||
println s1.district_name +" "+ s1.school_name+" "+distance
|
||||
if(distance<minDistance){
|
||||
minDistance = distance
|
||||
minSchool = s1
|
||||
minSchool.school_lng2 = lng2
|
||||
minSchool.school_lat2 = lat2
|
||||
}
|
||||
}
|
||||
}
|
||||
return minSchool
|
||||
}
|
||||
|
||||
def findPrimarySchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
def findMiddleSchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getRegionCodeByCity(def city){
|
||||
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
|
||||
for(def d : datas){
|
||||
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-12
@@ -1,24 +1,17 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.SimpleDateFormat
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -26,12 +19,11 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
|
||||
class HouseExport {
|
||||
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class RegionPriceExport {
|
||||
|
||||
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new RegionPriceExport();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def months = [202010,202011,202012,202101,202102,202103]
|
||||
def regions = CommonUtils.getAllRegions("福州","region")
|
||||
def rets = []
|
||||
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
|
||||
regions.each{region,values->
|
||||
def data = []
|
||||
data << "福州"
|
||||
data << region
|
||||
for(def month :months){
|
||||
def dbo = findPrice("福州", region, month);
|
||||
if(dbo){
|
||||
data << dbo.avg_price
|
||||
}else{
|
||||
data << ""
|
||||
}
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
def findPrice(city, region, month){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("date", month)
|
||||
def dbo = REGION_SELL_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
}
|
||||
+2
-4
@@ -5,16 +5,14 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
class ExportData {
|
||||
|
||||
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+3
-8
@@ -5,16 +5,11 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -27,7 +22,7 @@ class DistrictWash {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
class FZloationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new FZloationUpdate();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
|
||||
def index = 0
|
||||
def locationMap = getMoreSameLocationDist(districts)
|
||||
def total = 0
|
||||
locationMap.each {k,v->
|
||||
total = updateLocation(v, total);
|
||||
println "---------------------------------------------------------------------------------------------------------------"
|
||||
}
|
||||
}
|
||||
|
||||
def updateLocation(def lists, def total){
|
||||
|
||||
|
||||
for(def dist : lists){
|
||||
|
||||
def addrDistMap = getLocation(dist, dist.address)
|
||||
def nameDistMap = getLocation(dist, dist.region + dist.name)
|
||||
|
||||
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
|
||||
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
|
||||
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
|
||||
|
||||
if(addrDistMap){
|
||||
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
|
||||
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
|
||||
|
||||
def baidu = null
|
||||
def lat = null
|
||||
def lng = null
|
||||
def upldateFlag = ""
|
||||
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
|
||||
lat = addrDistMap.lat
|
||||
lng = addrDistMap.lng
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Address"
|
||||
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
|
||||
lng = nameDistMap.lng
|
||||
lat = nameDistMap.lat
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Name"
|
||||
}
|
||||
if(baidu){
|
||||
total++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
Long baiduLat = baidu.y*10000000000L
|
||||
Long baiduLng = baidu.x*10000000000L
|
||||
udata.put("lat2", baiduLat)
|
||||
udata.put("lng2", baiduLng)
|
||||
Long lat0 = lat*10000000000L
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
def getMoreSameLocationDist(def districts){
|
||||
def locationMap = [:]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def dist = districts[i]
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def locationKey = lng + "-" + lat
|
||||
def list = locationMap.get(locationKey)
|
||||
if(!list){
|
||||
list = []
|
||||
locationMap.put(locationKey, list);
|
||||
}
|
||||
list << dist
|
||||
}
|
||||
def resMap = [:]
|
||||
locationMap.each{k,v->
|
||||
if(v.size()>=2){
|
||||
|
||||
v.sort{ a,b->
|
||||
return a.name.size().compareTo(b.name.size())
|
||||
}
|
||||
resMap.put(k,v)
|
||||
}
|
||||
}
|
||||
|
||||
def lastRetMaps = [:]
|
||||
resMap.each{k,v->
|
||||
def swname = null
|
||||
def count = 0
|
||||
for(def i=0;i<v.size();i++){
|
||||
def d = v[i]
|
||||
if(i==0){
|
||||
if(d.name.contains("期")||d.name.contains("区")){
|
||||
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
|
||||
}else{
|
||||
swname = d.name
|
||||
}
|
||||
}else{
|
||||
if(d.name.contains(swname)||swname.contains(d.name)){
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if(count+1!=v.size()){
|
||||
lastRetMaps.put(k,v)
|
||||
}
|
||||
}
|
||||
return lastRetMaps;
|
||||
}
|
||||
|
||||
|
||||
def deleteRegion(def s, def region){
|
||||
def key = ""
|
||||
if(s.indexOf(region)>-1){
|
||||
if(s.indexOf(region+"区")>-1){
|
||||
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"县")>-1){
|
||||
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"市")>-1){
|
||||
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
|
||||
}
|
||||
}else{
|
||||
key = s;
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getLocation(dist, addr){
|
||||
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
if(location){
|
||||
def locMap = location.split(",")
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
|
||||
rMap.distance = distance
|
||||
rMap.level = level
|
||||
rMap.lat = locMap[1].toDouble()
|
||||
rMap.lng = locMap[0].toDouble()
|
||||
rMap.keyword = map.formatted_address
|
||||
rMap.location = location
|
||||
rMap.region = map.district
|
||||
return rMap;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
class FzLocationBatch {
|
||||
}
|
||||
+20
-1
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
|
||||
// task.matchHouse();
|
||||
|
||||
// task.villa()
|
||||
task.addressParse()
|
||||
// task.addressParse()
|
||||
|
||||
task.deleteProcess()
|
||||
}
|
||||
|
||||
def repeat(){
|
||||
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
|
||||
return floor;
|
||||
}
|
||||
|
||||
def deleteProcess(){
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
|
||||
def addrList = []
|
||||
for(def d : datas){
|
||||
def addr = d['G']
|
||||
addrList << addr
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
|
||||
def addr = row['C']
|
||||
if(addrList.contains(addr)){
|
||||
row['R'] = "已调查"
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
println "wwww"
|
||||
+1
-2
@@ -18,8 +18,7 @@ import org.apache.commons.lang.StringUtils
|
||||
class AutocompleteDataUpdaterBatch extends BaseBatch {
|
||||
|
||||
|
||||
|
||||
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "192.168.0.51", 29009)
|
||||
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "mongo.db.fangjia.com")
|
||||
private static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
private static final DSCollection AUTOCOMPLETE_COLL = new DataStore("192.168.0.38", 27008).getCollection("fangjia.autocomplete")
|
||||
private static final long C_DATE = DateUtil.formatToYYYYMMDDhhmmss(new Date()).toLong()
|
||||
|
||||
+17
-27
@@ -2,38 +2,28 @@ package com.fangjia.script.jiangnanblank
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.DecimalFormat
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||
import com.fangjia.datastore.driver.DSCollection;
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.commData.CommonDatautils;
|
||||
import com.fangjia.script.util.DateUtils;
|
||||
import com.fangjia.script.util.ExcelParser
|
||||
import com.fangjia.script.common.DataHandleUtils;
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.script.util.MongoUtils;
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.Map.Entry;
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.DecimalFormat
|
||||
import java.util.Map.Entry
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CZBlankDataExport {
|
||||
|
||||
@@ -62,7 +52,7 @@ class CZBlankDataExport {
|
||||
def directionMap = [1:"ÄÏ",2:"±±",3:"Äϱ±",4:"¶«",5:"Î÷",6:"¶«Î÷",7:"¶«ÄÏ",8:"Î÷ÄÏ",9:"¶«±±",10:"Î÷±±"]
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
-1
@@ -12,7 +12,6 @@ import com.jcraft.jsch.ChannelSftp
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
/**
|
||||
* 江南银行线上数据交付
|
||||
|
||||
+2
-17
@@ -6,42 +6,27 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.CrawlCommandConsumerBatch
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.BatchUpdateException
|
||||
import java.sql.ResultSet
|
||||
import java.text.NumberFormat
|
||||
import java.util.Map.Entry
|
||||
import java.util.concurrent.BlockingQueue
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.LinkedBlockingDeque
|
||||
import java.util.concurrent.LinkedBlockingQueue
|
||||
import java.util.concurrent.ThreadPoolExecutor
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@BatchOwner(Watcher.WeiLei)
|
||||
class HousingOverviewExportBatch extends BaseBatch {
|
||||
|
||||
static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque<Runnable>())
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+7
-10
@@ -1,11 +1,10 @@
|
||||
package com.fangjia.script.jusfoun
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.GroovyRowResult
|
||||
@@ -18,9 +17,7 @@ import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import redis.clients.jedis.JedisPool
|
||||
import redis.clients.jedis.JedisPoolConfig
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.*
|
||||
import java.text.DateFormat
|
||||
@@ -56,7 +53,7 @@ class JusfounCommon {
|
||||
public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
|
||||
public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
|
||||
|
||||
private static JdbcDataSource dataSource = null
|
||||
private static EsDataSource dataSource = null
|
||||
|
||||
private static Map<String, BasicDBObject> _map_seaweed = new HashMap<>();
|
||||
private static Map<String, Map<String, DBObject>> mainCityMap = new HashMap<>();
|
||||
@@ -390,7 +387,7 @@ class JusfounCommon {
|
||||
return DataUtil.hanyuQuanpin(hanzi, '')
|
||||
}
|
||||
|
||||
static JdbcDataSource getESDataSource() {
|
||||
static EsDataSource getESDataSource() {
|
||||
if (dataSource != null) {
|
||||
return dataSource;
|
||||
} else {
|
||||
@@ -399,7 +396,7 @@ class JusfounCommon {
|
||||
return dataSource;
|
||||
}
|
||||
try {
|
||||
dataSource = new JdbcDataSource();
|
||||
dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties();
|
||||
@@ -625,13 +622,13 @@ class JusfounCommon {
|
||||
String scrollID = null) {
|
||||
try {
|
||||
if (transportClient == null) {
|
||||
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
}
|
||||
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
try {
|
||||
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
|
||||
} catch (Exception e1) {
|
||||
e1.printStackTrace()
|
||||
|
||||
-8
@@ -8,21 +8,13 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.sql.BatchUpdateException
|
||||
import java.util.concurrent.LinkedBlockingDeque
|
||||
import java.util.concurrent.ThreadPoolExecutor
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class RentHouseExportBatch extends BaseBatch {
|
||||
|
||||
+2
-2
@@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
def List find_rent_house_prices(String city, String region, String name, Date start, Date end){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+5
-8
@@ -1,20 +1,17 @@
|
||||
package com.fangjia.script.log
|
||||
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class MongoDBLogDisMonitorBatch extends BaseBatch {
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
|
||||
+6
-8
@@ -1,20 +1,18 @@
|
||||
package com.fangjia.script.log
|
||||
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class MongoDBLogMonitorBatch extends BaseBatch {
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
|
||||
+3
-2
@@ -10,7 +10,8 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
@@ -25,7 +26,7 @@ class DistrictHouseTypeAnalyzeBatch extends BaseBatch {
|
||||
|
||||
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final JdbcDataSource dataSource = new JdbcDataSource()
|
||||
private static final EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-4
@@ -5,7 +5,7 @@ import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -55,7 +55,6 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
Map userInfo = getUserInfo()
|
||||
def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
|
||||
def endDate = DateUtil.truncateDay(new Date())
|
||||
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
|
||||
|
||||
@@ -64,7 +63,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
long count = 0
|
||||
long total = cursor.count()
|
||||
print("total="+total)
|
||||
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("192.168.0.206", "fangjia")
|
||||
BulkProcessor bulkProcessor = ElasticUtils.createBulkProcessor("seawater.es.db.fangjia.com")
|
||||
for(BasicDBObject log : cursor){
|
||||
Map<String,Object> parameter_map = (Map) log.get("parameter_map")
|
||||
if(parameter_map == null) {
|
||||
@@ -128,7 +127,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
cmd.put("user_name", user.user_name)
|
||||
cmd.put("company_name", user.company)
|
||||
}
|
||||
bulkProcessor.add(new IndexRequest("open_uri_log_normal", "_doc", log.get("_id").toString()).source(cmd))
|
||||
bulkProcessor.add(new IndexRequest("open_uri_log_normal").id(log.get("_id").toString()).source(cmd))
|
||||
println(cmd)
|
||||
count++
|
||||
if(count % 100 == 0){
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202101);
|
||||
def count2 = findTotalCount(source, 202102);
|
||||
def count1 = findTotalCount(source, 202102);
|
||||
def count2 = findTotalCount(source, 202103);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class AddFangSchoolRegion {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddFangSchoolRegion();
|
||||
//task.process()
|
||||
//task.match()
|
||||
task.addMapLocation()
|
||||
}
|
||||
|
||||
def addMapLocation(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
def data = findSchool(d.city, d.region, d.school_name)
|
||||
if(data && data.location){
|
||||
def location = data.location
|
||||
if(location){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("school_map_lng", location[0])
|
||||
udata.put("school_map_lat", location[1])
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
index ++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def match(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// if(!data.match_name){
|
||||
println i +" "+data.district_name
|
||||
matching(data);
|
||||
// }
|
||||
|
||||
} catch (Exception e) {
|
||||
println "Òì³£"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matching(def it){
|
||||
def rMap = matchDistrict(it.city, it.region, it.district_name)
|
||||
if(!rMap){
|
||||
rMap = matchDistrict(it.city, it.region, it.address)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("match_city", rMap.city)
|
||||
data.put("match_region", rMap.region)
|
||||
data.put("match_name", rMap.name)
|
||||
data.put("match_address", rMap.address)
|
||||
data.put("match_lat", rMap.lat)
|
||||
data.put("match_lng", rMap.lng)
|
||||
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict(def city, def region, def address){
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dto = doc.result
|
||||
if(dto && dto.searchAddress){
|
||||
def ret = dto.matchResult
|
||||
if(ret.match){
|
||||
rMap.city = dto.city
|
||||
rMap.region = dto.district
|
||||
rMap.name = dto.name
|
||||
rMap.lat = dto.lat
|
||||
rMap.lng = dto.lng
|
||||
rMap.id = dto.id
|
||||
rMap.address = dto.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
def process(){
|
||||
def datas = findNoRegionData();
|
||||
findSimialData(datas);
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findNoRegionData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findSimialData(datas){
|
||||
for(def d : datas){
|
||||
def obj = findThirdPriceData(d.city, d.url+"/")
|
||||
if(!obj){
|
||||
obj = findThirdPriceByName(d.city, d.district_name)
|
||||
}
|
||||
if(obj){
|
||||
d.region = obj.region
|
||||
if(!d.district_addrss){
|
||||
d.district_address = obj.address
|
||||
}
|
||||
updateDistrictSchool(d);
|
||||
|
||||
}else{
|
||||
println d.school_name+"" + d.district_name+"û²éµ½"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findThirdPriceData(def city, def url){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("district_id", url)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findThirdPriceByName(def city, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("name", name)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def updateDistrictSchool(d){
|
||||
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("region", d.region)
|
||||
data.put("district_address", d.district_address)
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
|
||||
def findSchool(city, region, school_name){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("city", city)
|
||||
data.put("region", region)
|
||||
data.put("name", school_name)
|
||||
return school_col.findOne(data)
|
||||
}
|
||||
}
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.alibaba.fastjson.annotation.JSONField
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.NameValuePair
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.http.HttpEntity
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpPost
|
||||
import org.apache.http.impl.client.CloseableHttpClient
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
|
||||
import java.nio.charset.Charset
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GushuwangCrawl {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
static main(args) {
|
||||
def task = new GushuwangCrawl();
|
||||
task.process()
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
// chaxuequCrawl()
|
||||
suzhouCrawl();
|
||||
|
||||
}
|
||||
|
||||
def suzhouCrawl(){
|
||||
def page = 1
|
||||
def stop = false
|
||||
def list = []
|
||||
while(!stop){
|
||||
def s = schoolList(page);
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.ok){
|
||||
def datas = jsonObject.extra.list
|
||||
if(datas){
|
||||
list.addAll(datas)
|
||||
page++
|
||||
println page
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(list){
|
||||
def schools = []
|
||||
for(def d : list){
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = "苏州"
|
||||
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(d.schoolname)
|
||||
schoolData.name = d.schoolname
|
||||
schoolData.address = d.adr
|
||||
/**来源网站*/
|
||||
schoolData.source = "苏州学区查询"
|
||||
/**房源详情url*/
|
||||
schools << schoolData
|
||||
}
|
||||
|
||||
saveChaxuequSchool(schools)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def chaxuequCrawl(){
|
||||
|
||||
def page = 1
|
||||
def stop = false
|
||||
|
||||
def schools = []
|
||||
|
||||
while(!stop){
|
||||
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
|
||||
println "内容:"+s
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject.code == 200){
|
||||
def data = jsonObject.data
|
||||
def totalPage = data.pages
|
||||
if(data.list){
|
||||
page++
|
||||
if(page+1>totalPage){
|
||||
stop = true;
|
||||
}
|
||||
schools.addAll(data.list)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
print(schools.size());
|
||||
|
||||
def districtSchools = []
|
||||
def schoolInfos = []
|
||||
if(schools.size()>0){
|
||||
|
||||
def url = "https://51yongka.com/fapi/school/details"
|
||||
for(def d : schools){
|
||||
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
|
||||
def s = chaxuequInfoCrawl(url, params)
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.code==200){
|
||||
|
||||
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
|
||||
def districts = jsonObject.data.nearbyPropertyList
|
||||
for(def dist : districts){
|
||||
def dataMap = new BasicDBObject()
|
||||
dataMap.city = jsonObject.data.city
|
||||
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
dataMap.category = converCategory(jsonObject.data.schoolName)
|
||||
dataMap.district_name = dist.name.trim()
|
||||
dataMap.district_address = dist.address
|
||||
dataMap.school_name = jsonObject.data.schoolName.trim()
|
||||
dataMap.school_address = jsonObject.data.address
|
||||
dataMap.source = "查学区房"
|
||||
dataMap.url = url + "?"+params
|
||||
dataMap.telephone = jsonObject.data.officialTelephone;
|
||||
districtSchools << dataMap
|
||||
}
|
||||
}else{
|
||||
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = jsonObject.data.city
|
||||
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(jsonObject.data.schoolName)
|
||||
schoolData.name = jsonObject.data.schoolName.trim()
|
||||
schoolData.address = jsonObject.data.address
|
||||
/**来源网站*/
|
||||
schoolData.source = "查学区房"
|
||||
/**房源详情url*/
|
||||
schoolData.telephone = jsonObject.data.officialTelephone;
|
||||
schoolInfos << schoolData
|
||||
println jsonObject.data.schoolName +" 无周边楼盘"
|
||||
}
|
||||
|
||||
}
|
||||
println s;
|
||||
}
|
||||
}
|
||||
|
||||
println "小区:"+districtSchools.size()
|
||||
// saveChaxuequDistrictSchool(districtSchools)
|
||||
saveChaxuequSchool(schoolInfos)
|
||||
|
||||
}
|
||||
|
||||
def saveChaxuequSchool(schoolInfos){
|
||||
for(def ds : schoolInfos){
|
||||
def obj = findSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def saveChaxuequDistrictSchool(def districtSchools){
|
||||
for(def ds : districtSchools){
|
||||
def obj = findDistrictSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
district_school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findSchoolData(city, school_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
return school_col.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
def findDistrictSchoolData(city, school_name, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
q.put("district_name", district_name)
|
||||
return district_school_col.findOne(q)
|
||||
}
|
||||
|
||||
def converCategory(def schoolName){
|
||||
|
||||
if(schoolName){
|
||||
if(schoolName.contains("初中部")||schoolName.contains("中学")){
|
||||
return "初中"
|
||||
}else if(schoolName.contains("小学")){
|
||||
return "小学"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def static chaxuequInfoCrawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","51yongka.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","75")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
|
||||
headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def static schoolList(page) {
|
||||
def responseMap = [:]
|
||||
String s = null
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","m.gusuwang.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","40")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
|
||||
//params信息
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
|
||||
// //post请求
|
||||
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
|
||||
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
outputStream.write(content.getBytes());
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
// byte[] data = new byte[1024];
|
||||
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
|
||||
// while (inputStream.read(data) != -1) {
|
||||
// String s = new String(data, Charset.forName("utf-8"));
|
||||
// sb.append(s);
|
||||
// }
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+11
-37
@@ -1,50 +1,24 @@
|
||||
package com.fangjia.script.trade
|
||||
///µ¼Èë¼Û¸ñÖ¸Êý
|
||||
import com.fangjia.script.BaseBatch;
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
|
||||
///导入价格指数
|
||||
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
|
||||
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.poi.xssf.usermodel.XSSFCell
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.script.util.DateUtils;
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
|
||||
import groovy.json.JsonSlurper
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.NumberFormat;
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.fangjia.core.util.DBIterator
|
||||
|
||||
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
import com.fangjia.datastore.driver.DSCollection;
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.commData.CommonDatautils;
|
||||
import com.ibm.icu.text.SimpleDateFormat;
|
||||
import com.fangjia.storehouse.DBIterator
|
||||
import com.fangjia.storehouse.DBIterator;
|
||||
import com.fangjia.storehouse.StorehouseDB;
|
||||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ImportIndexStats extends BaseBatch{
|
||||
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
|
||||
@@ -61,7 +35,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
public static final def dingjiYear = 2016;
|
||||
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final JdbcDataSource dataSource = new JdbcDataSource()
|
||||
private static final EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-8
@@ -1,17 +1,12 @@
|
||||
package com.fangjia.script.trade
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
@@ -22,7 +17,7 @@ class TradeDealPriceStat {
|
||||
|
||||
|
||||
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ package com.fangjia.script.usahouse
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
|
||||
|
||||
class AddressMatch {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+1
-4
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.usahouse
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.BaseBatch
|
||||
@@ -8,14 +8,11 @@ import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSON
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
@@ -425,6 +425,7 @@ class ABuYunUtil {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace()
|
||||
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
|
||||
} finally {
|
||||
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())
|
||||
|
||||
+173
-156
@@ -1,122 +1,169 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import org.apache.commons.lang3.SystemUtils
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchType
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.settings.Settings
|
||||
import org.elasticsearch.common.transport.TransportAddress
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit
|
||||
import org.elasticsearch.common.unit.ByteSizeValue
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.sort.FieldSortBuilder
|
||||
import org.elasticsearch.search.sort.SortOrder
|
||||
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticUtils {
|
||||
|
||||
private static TransportClient client = null;
|
||||
|
||||
public static TransportClient getClient(def host){
|
||||
if(client != null){
|
||||
return client;
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
|
||||
def certPath = ""
|
||||
if(SystemUtils.IS_OS_WINDOWS){
|
||||
certPath= "D:\\goojiaconfig\\seawater\\conf\\elastic-certificates.p12"
|
||||
}else{
|
||||
certPath = "/opt/elasticmongo/elastic-certificates.p12"
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
Settings.Builder builder = Settings.builder()
|
||||
.put("cluster.name", "fangjia")
|
||||
.put("client.transport.sniff", true)
|
||||
|
||||
builder.put("xpack.security.user", "developer" + ":" + "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
.put("xpack.security.transport.ssl.enabled", "true")
|
||||
.put("xpack.security.transport.ssl.truststore.path", certPath)
|
||||
.put("xpack.security.transport.ssl.keystore.path", certPath)
|
||||
.put("xpack.security.transport.ssl.verification_mode", "certificate")
|
||||
|
||||
Settings settings = builder.build();
|
||||
client = new PreBuiltXPackTransportClient(settings)
|
||||
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
|
||||
return client
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure, String type= "_doc"){
|
||||
|
||||
client = getClient(host);
|
||||
|
||||
BulkProcessor bulkProcessor = BulkProcessor.builder(
|
||||
client,
|
||||
new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId,
|
||||
BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId,
|
||||
BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId,
|
||||
BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
})
|
||||
.setBulkActions(10000)
|
||||
.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(5))
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10))
|
||||
.build()
|
||||
|
||||
SearchResponse scrollResp = null;
|
||||
int errorCount = 0;
|
||||
def hits ;
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
try{
|
||||
scrollResp = client.prepareSearch(indexName)
|
||||
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
|
||||
.setScroll(new TimeValue(600000))
|
||||
.setQuery(qb)
|
||||
.setSize(1000).get()
|
||||
hits = scrollResp.getHits().getHits()
|
||||
stop = true;
|
||||
}catch(Exception e){
|
||||
errorCount++;
|
||||
if(errorCount>11){
|
||||
stop = true;
|
||||
}
|
||||
println "错误:${errorCount} "+e.getMessage()
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 60);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
if(hits){
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
}
|
||||
|
||||
for (int k=0;k<hits.size();k++) {
|
||||
final SearchHit hit = hits[k];
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
@@ -124,96 +171,66 @@ class ElasticUtils {
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
// println(hits)
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
|
||||
while(scrollResp.getHits().getHits().length != 0){
|
||||
hits = scrollResp.getHits().getHits();
|
||||
for (int k=0;k<hits.size();k++) {
|
||||
final SearchHit hit = hits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
// println(hits)
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
|
||||
}
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
client = getClient(host);
|
||||
SearchResponse scrollResp = null;
|
||||
int errorCount = 0;
|
||||
def hits ;
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
try{
|
||||
scrollResp = client.prepareSearch(indexName)
|
||||
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
|
||||
.setScroll(new TimeValue(600000))
|
||||
.setQuery(qb)
|
||||
.setSize(10000)
|
||||
.setScroll(TimeValue.timeValueMinutes(1))//设置 search context 维护1分钟的有效期
|
||||
.get()
|
||||
|
||||
stop = true;
|
||||
}catch(Exception e){
|
||||
errorCount++;
|
||||
if(errorCount>11){
|
||||
stop = true;
|
||||
}
|
||||
println "错误:${errorCount} "+e.getMessage()
|
||||
}
|
||||
}
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while(scrollResp.getHits() && scrollResp.getHits().getHits().length != 0){
|
||||
for (SearchHit searchHit : scrollResp.getHits().getHits()) {
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet();
|
||||
//当searchHits的数组为空的时候结束循环,至此数据全部读取完毕
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
//删除scroll
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollResp.getScrollId());
|
||||
client.clearScroll(clearScrollRequest).actionGet();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-84
@@ -1,84 +0,0 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import org.apache.commons.lang3.SystemUtils
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.settings.Settings
|
||||
import org.elasticsearch.common.transport.TransportAddress
|
||||
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticsearchHelper {
|
||||
|
||||
|
||||
static getClient(String host, String clusterName = "fangjia"){
|
||||
Settings.Builder builder = Settings.builder()
|
||||
.put("cluster.name", clusterName)
|
||||
.put("client.transport.sniff", false)
|
||||
.put("client.transport.ping_timeout", "60s")
|
||||
.put("xpack.security.user","developer:5e8edd851d2fdfbd7415232c67367cc3")
|
||||
.put("xpack.security.transport.ssl.enabled", "true")
|
||||
if(SystemUtils.IS_OS_WINDOWS){
|
||||
builder
|
||||
.put("xpack.security.transport.ssl.truststore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
|
||||
.put("xpack.security.transport.ssl.keystore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
|
||||
}else{
|
||||
builder
|
||||
.put("xpack.security.transport.ssl.truststore.path", "/opt/elasticmongo/elastic-certificates.p12")
|
||||
.put("xpack.security.transport.ssl.keystore.path", "/opt/elasticmongo/elastic-certificates.p12")
|
||||
}
|
||||
builder.put("xpack.security.transport.ssl.verification_mode", "certificate").build()
|
||||
Settings settings = builder.build()
|
||||
return new PreBuiltXPackTransportClient(settings)
|
||||
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param clusterName
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor( String host, String clusterName){
|
||||
TransportClient client = getClient(host, clusterName)
|
||||
BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request, Throwable failure) {
|
||||
failure.printStackTrace()
|
||||
}
|
||||
}).setBulkActions(10000).setConcurrentRequests(20).build()
|
||||
|
||||
return bulkProcessor
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor){
|
||||
|
||||
bulkProcessor.awaitClose(10, TimeUnit.MINUTES)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -1,19 +1,17 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import com.mongodb.DBCursor
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
|
||||
class HouseUtils{
|
||||
static main(args){
|
||||
|
||||
+2
-2
@@ -121,9 +121,9 @@ public class OpenApiClient {
|
||||
params.put("suggestName", suggestName)
|
||||
}
|
||||
|
||||
String content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8")
|
||||
while (content == null) {
|
||||
content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
|
||||
content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8")
|
||||
println("${city} ${input} ${cat}程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
|
||||
+2
-3
@@ -1,19 +1,18 @@
|
||||
package com.fangjia.script.vipsite
|
||||
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchType
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.SearchHits
|
||||
|
||||
try {
|
||||
TransportClient transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
TransportClient transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
|
||||
QueryBuilder dateRange_avg_price = QueryBuilders.rangeQuery("avg_price").lte(0);
|
||||
QueryBuilder dateRange_total_price = QueryBuilders.rangeQuery("total_price").lte(0);
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.weilei
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
@@ -8,7 +8,7 @@ import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -16,7 +16,7 @@ import java.sql.SQLException
|
||||
import java.sql.Statement
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties();
|
||||
|
||||
+12
-2557
File diff suppressed because it is too large
Load Diff
@@ -1,31 +1,16 @@
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.CommandResult
|
||||
import com.mongodb.DB
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.*;
|
||||
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import static com.mongodb.client.model.Filters.and
|
||||
import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
|
||||
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
|
||||
@@ -43,7 +28,7 @@ cityList.each { def city ->
|
||||
}
|
||||
})
|
||||
roots.each { def root ->
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-14
@@ -1,23 +1,12 @@
|
||||
package com.fangjia.script.wenyan.address
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.util.QQKeyPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -44,7 +33,7 @@ import java.sql.Statement
|
||||
// data << (k.split("_") as List)
|
||||
//}
|
||||
//ExcelUtils.write("D:\\ÎÞ¾ù¼ÛÊý¾Ý.xlsx", data, "½á¹û")
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-3
@@ -1,20 +1,20 @@
|
||||
package com.fangjia.script.wenyan.avgPrice
|
||||
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
def findEstateType(String city, String region, String name, List alias) {
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-7
@@ -10,17 +10,12 @@ import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.math3.ml.clustering.CentroidCluster
|
||||
import org.apache.commons.math3.ml.clustering.DoublePoint
|
||||
import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017)
|
||||
|
||||
def findHouse(BasicDBObject district){
|
||||
@@ -54,7 +49,7 @@ def findHouse(BasicDBObject district){
|
||||
sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ")
|
||||
sqlBuffer.append(" ORDER BY p_date desc")
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
package com.fangjia.script.wenyan.seawater
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
package com.fangjia.script.wenyan.seawater
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+28
-287
@@ -1,6 +1,8 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
@@ -9,294 +11,33 @@ import net.sf.json.JSONObject
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
//
|
||||
//def map = [:]
|
||||
//ExcelUtils.read("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批0829.xlsx", 1).each {List item->
|
||||
// if(item["AF"] && item["AH"]){
|
||||
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
// map.get(key, []) << key
|
||||
// }
|
||||
//}
|
||||
|
||||
//ExcelUtils.modify("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批(房价网交付0905).xlsx", { List item ->
|
||||
//// if(item["AF"] && item["AH"]){
|
||||
//// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
//// List nums = map.get(key)
|
||||
//// item["AT"] = nums.size()
|
||||
//// }
|
||||
//
|
||||
// def city = "上海"
|
||||
// def addr = item["AC"]
|
||||
// if(addr){
|
||||
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
|
||||
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
// println searchAddress
|
||||
//// item["AD"] = searchAddress.province
|
||||
//// item["AE"] = searchAddress.city
|
||||
//// item["AF"] = searchAddress.district
|
||||
// if(searchAddress.standardAddress){
|
||||
//// item["AG"] = searchAddress.street
|
||||
//// if(searchAddress.streetNumber){
|
||||
//// item["AG"] = item["AG"] + searchAddress.streetNumber
|
||||
//// }
|
||||
//
|
||||
// if(searchAddress.township){
|
||||
// item["AP"] = searchAddress.township
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(item["AG"]){
|
||||
// def addrMap = getRoad(item["AG"])
|
||||
// if(addrMap){
|
||||
// if(addrMap.road){
|
||||
// item["AQ"] = addrMap.road
|
||||
// }
|
||||
// if(addrMap.roadNumber){
|
||||
// item["AR"] = addrMap.roadNumber
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
//// if (item["AH"] && item["AF"]) {
|
||||
//// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["AE"], item["AF"], item["AH"]))
|
||||
//// if (seaweed) {
|
||||
//// if (item["AR"] && item["AS"]) {
|
||||
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2, item["AS"].toInteger(), item["AR"].toInteger())
|
||||
//// if (eva) {
|
||||
//// item["AL"] = eva.avgPrice
|
||||
//// item["AM"] = eva.totalPrice
|
||||
//// }
|
||||
//// } else {
|
||||
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2)
|
||||
//// if (eva) {
|
||||
//// item["AL"] = eva.avgPrice
|
||||
//// item["AM"] = eva.totalPrice
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
////
|
||||
//// }
|
||||
//
|
||||
//},1)
|
||||
|
||||
|
||||
// if(item["AK"] =~ /^\d{4}$/){
|
||||
// item["AR"] = item["AK"].toString().substring(0, 2)
|
||||
// }else if(item["AK"] =~ /^\d{3}$/){
|
||||
// item["AR"] = item["AK"].toString().substring(0, 1)
|
||||
// }
|
||||
// if(item["AR"] && item["AH"]){
|
||||
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AS"] = nums[nums.size() - 1]
|
||||
// item["AT"] = nums.unique().size()
|
||||
// }
|
||||
// println(item)
|
||||
// item["AG"] = null
|
||||
// def city = "上海"
|
||||
// def addr = item["AC"]
|
||||
// def area = item["o"]
|
||||
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
|
||||
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
// item["AD"] = searchAddress.province
|
||||
// item["AE"] = searchAddress.city
|
||||
// item["AF"] = searchAddress.district
|
||||
// if(searchAddress.standardAddress){
|
||||
// item["AG"] = searchAddress.street
|
||||
// if(searchAddress.streetNumber){
|
||||
// item["AG"] = item["AG"] + searchAddress.streetNumber
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(resp.getDouble("credit") > 0){
|
||||
// item["AE"] = resp.city
|
||||
// item["AF"] = resp.district
|
||||
// item["AH"] = resp.name
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
|
||||
// item["AL"] = eva.avgPrice
|
||||
// item["AM"] = eva.totalPrice
|
||||
// item["AN"] = new Date().format("yyyy-MM-dd")
|
||||
// if(!item["AG"]){
|
||||
// item["AG"] = resp.address
|
||||
// }
|
||||
// }
|
||||
// item["AI"] = searchAddress.buildingNumber
|
||||
// item["AJ"] = searchAddress.unitNumber
|
||||
// item["AK"] = searchAddress.roomNumber
|
||||
// println(item)
|
||||
//},1)
|
||||
//
|
||||
def sheetIndexs = ["湘潭": 0, "嘉兴": 1, "苏州": 2]
|
||||
sheetIndexs.each { String city, int sheetIndex ->
|
||||
// def map = [:]
|
||||
// ExcelUtils.read("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批0828(测试)1.xlsx", sheetIndex).each {List item->
|
||||
// if(item["Q"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// map.get(key, []) << key
|
||||
// }
|
||||
// }
|
||||
ExcelUtils.multiThreadingModify("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批(房价网交付0905).xlsx", { List item ->
|
||||
// if(item["Q"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AE"] = nums.size()
|
||||
// }
|
||||
|
||||
// item["W"] = null
|
||||
// item["X"] = null
|
||||
|
||||
// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["P"], item["Q"], item["S"]))
|
||||
// if(seaweed){
|
||||
// if(item["AC"] && item["AD"]){
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2, item["AD"].toInteger(), item["AC"].toInteger())
|
||||
// if(eva){
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// }
|
||||
// }else{
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2)
|
||||
// if(eva){
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(item["AC"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AD"] = nums[nums.size() - 1]
|
||||
// item["AE"] = nums.unique().size()
|
||||
// }
|
||||
|
||||
// if(item["V"] =~ /^\d{4}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 2)
|
||||
// }else if(item["V"] =~ /^\d{3}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 1)
|
||||
// }else if(item["V"] =~ /^\d{6,7}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 2).replaceAll("^0", "")
|
||||
// }
|
||||
|
||||
|
||||
def addr = item["B"].toString().replaceAll("(", "").replace('区/县', "区").replace("街道/镇", "镇")
|
||||
.replaceAll(")", "").replaceAll("【.*】", "").replaceAll("室/单元", "号")
|
||||
// def area = item["L"]
|
||||
|
||||
|
||||
def region = null
|
||||
if (city == "嘉兴") {
|
||||
region = "嘉善"
|
||||
} else if (city == "苏州") {
|
||||
region = "昆山"
|
||||
}
|
||||
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property", false, region)
|
||||
JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
|
||||
def township = ""
|
||||
if(searchAddress && searchAddress.township){
|
||||
item["AP"] = searchAddress.township
|
||||
township = searchAddress.township
|
||||
}
|
||||
if(item["S"]){
|
||||
def addrMap = getRoad(item["S"])
|
||||
if(addrMap){
|
||||
if(addrMap.road){
|
||||
def s = addrMap.road;
|
||||
if(township){
|
||||
if(township.contains(s)){
|
||||
item["AQ"] = s.replace(township, "")
|
||||
}else{
|
||||
if(township.contains("镇")){
|
||||
township = township.replace("镇","")
|
||||
}
|
||||
item["AQ"] = s.replace(township, "")
|
||||
}
|
||||
}else{
|
||||
item["AQ"] = addrMap.road
|
||||
}
|
||||
}
|
||||
if(addrMap.roadNumber){
|
||||
item["AR"] = addrMap.roadNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
println searchAddress
|
||||
// item["O"] = searchAddress.province
|
||||
// item["P"] = searchAddress.city
|
||||
// item["Q"] = searchAddress.district
|
||||
// if (searchAddress.standardAddress) {
|
||||
// item["R"] = searchAddress.street
|
||||
// if (searchAddress.streetNumber) {
|
||||
// item["R"] = item["R"] + searchAddress.streetNumber
|
||||
// }
|
||||
// }
|
||||
// if (resp.getDouble("credit") > 0) {
|
||||
// item["O"] = resp.province
|
||||
// item["P"] = resp.city
|
||||
// item["Q"] = resp.district
|
||||
// item["S"] = resp.name
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// item["Y"] = new Date().format("yyyy-MM-dd")
|
||||
// if (!item["R"]) {
|
||||
// item["R"] = resp.address
|
||||
// }
|
||||
// }
|
||||
// item["T"] = searchAddress.buildingNumber
|
||||
// item["U"] = searchAddress.unitNumber
|
||||
// item["V"] = searchAddress.roomNumber
|
||||
// if(city == "湘潭"){
|
||||
// String roomNumber = DataUtil.match(addr, Pattern.compile("\\d+(号|(号房))\$"))
|
||||
// if(roomNumber){
|
||||
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
|
||||
// }
|
||||
// roomNumber = DataUtil.match(addr, Pattern.compile("\\d{6,8}\$"))
|
||||
// if(roomNumber){
|
||||
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
|
||||
// }
|
||||
//
|
||||
// }
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
|
||||
new File("D:\\Êý¾Ý\\ѧУÊý¾Ý").eachFile {File file->
|
||||
ExcelUtils.read(file.getAbsolutePath(), 0).each {List item->
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.city = item["A"]
|
||||
dbo.region = item["B"]
|
||||
dbo.name = item["C"]
|
||||
dbo.address = item["E"]
|
||||
dbo.attribute = item["G"]
|
||||
dbo.property = item["H"]
|
||||
dbo.category = "Сѧ"
|
||||
dbo.location = [DataUtil.roundDouble(item["I"], 6), DataUtil.roundDouble(item["J"], 6)]
|
||||
school.insert(dbo)
|
||||
println(dbo)
|
||||
}
|
||||
ExcelUtils.read(file.getAbsolutePath(), 1).each {List item->
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.city = item["A"]
|
||||
dbo.region = item["B"]
|
||||
dbo.name = item["C"]
|
||||
dbo.address = item["E"]
|
||||
dbo.attribute = item["G"]
|
||||
dbo.property = item["H"]
|
||||
dbo.category = "ÖÐѧ"
|
||||
dbo.location = [item["I"], item["J"]]
|
||||
school.save(dbo)
|
||||
println(item)
|
||||
}, sheetIndex)
|
||||
}
|
||||
|
||||
|
||||
def getRoad(def s){
|
||||
def rex = ".*(路|街道|街|大道|巷|道|里).*\\d+(弄|号)\$"
|
||||
Pattern pattern = Pattern.compile(rex);
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list && list.size()>0){
|
||||
def s1 = list[0]
|
||||
def road = getRexText(s1, ".*(路|街道|街|大道|巷|道|里)")
|
||||
def num = getRexText(s1, "\\d+")
|
||||
return ["road":road, "roadNumber":num]
|
||||
}else{
|
||||
def road = getRexText(s, ".*(路|街道|街|大道|巷|道|里)")
|
||||
return ["road":road]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getRexText(s, regx){
|
||||
Pattern pattern = Pattern.compile(regx);
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
while (matcher.find()) {
|
||||
return matcher.group();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -1,9 +1,16 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.openqa.selenium.json.Json
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
|
||||
@@ -30,8 +37,6 @@ import org.apache.commons.lang3.StringUtils
|
||||
//}, 0,"结果")
|
||||
|
||||
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
|
||||
// if (item["T"] == null || ![1, 2, 4].contains(item["T"].toString().toDouble().toInteger())) {
|
||||
// return
|
||||
@@ -114,43 +119,80 @@ import org.apache.commons.lang3.StringUtils
|
||||
// println("${key} ${names.join(",")}")
|
||||
// }
|
||||
//}
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\µØÖ·±ê×¼»¯.xlsx",{List item->
|
||||
String address = "";
|
||||
if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
|
||||
address = ProvinceData.getDisplay(item["B"])
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\地址标准化.xlsx",{List item->
|
||||
// String address = "";
|
||||
// if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
|
||||
// address = ProvinceData.getDisplay(item["B"])
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["C"])){
|
||||
// String cityDisplay = CityData.getCityDisplay(item["C"]);
|
||||
// if(cityDisplay != null){
|
||||
// address = address + cityDisplay
|
||||
// }else{
|
||||
// address = address + item["C"] + "市"
|
||||
// }
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["D"])){
|
||||
// String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
|
||||
// if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
|
||||
// address = address + orgDistrict
|
||||
// }
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["E"])){
|
||||
// address = address + item["E"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["F"])){
|
||||
// address = address + item["F"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["G"])){
|
||||
// address = address + item["G"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["H"])){
|
||||
// address = address + item["H"] + "栋"
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["i"])){
|
||||
// address = address + item["i"] + "单元"
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["J"])){
|
||||
// address = address + item["J"] + "室"
|
||||
// }
|
||||
// item["K"] = address
|
||||
// println(address)
|
||||
//})
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia", "guiding_price")
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\深圳市住建局二手房指导价.xlsx", { List item ->
|
||||
if (item["G"]) {
|
||||
BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["G"], item["H"], item["I"]))
|
||||
BasicDBObject guid = new BasicDBObject()
|
||||
guid.guiding_type = 0
|
||||
guid.estate_type = seaweed.estate_type2
|
||||
guid.house_type = 0
|
||||
guid.c_date = new Date()
|
||||
guid.guiding_person = "深圳房管局"
|
||||
guid.guiding_date = Date.parse('yyyyMMdd', '20210411')
|
||||
guid.category = "district"
|
||||
guid.guiding_value = (int)item["E"]
|
||||
guid.name = item["D"]
|
||||
guid.city = item["G"]
|
||||
guid.region = item["H"]
|
||||
guid.name = item["i"]
|
||||
school.insert(guid)
|
||||
println(guid)
|
||||
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["C"])){
|
||||
String cityDisplay = CityData.getCityDisplay(item["C"]);
|
||||
if(cityDisplay != null){
|
||||
address = address + cityDisplay
|
||||
}else{
|
||||
address = address + item["C"] + "ÊÐ"
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["D"])){
|
||||
String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
|
||||
if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
|
||||
address = address + orgDistrict
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["E"])){
|
||||
address = address + item["E"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["F"])){
|
||||
address = address + item["F"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["G"])){
|
||||
address = address + item["G"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["H"])){
|
||||
address = address + item["H"] + "¶°"
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["i"])){
|
||||
address = address + item["i"] + "µ¥Ôª"
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["J"])){
|
||||
address = address + item["J"] + "ÊÒ"
|
||||
}
|
||||
item["K"] = address
|
||||
println(address)
|
||||
// if(!item["F"]){
|
||||
// JSONObject resp = OpenApiClient.addressMatch("深圳", item["D"], 'property', false, item["B"])
|
||||
// if(resp.id){
|
||||
// BasicDBObject seaweed = DbUtils.seaweed.findById(resp.id)
|
||||
// if(seaweed){
|
||||
// item["F"] = seaweed._id.toString()
|
||||
// item["G"] = seaweed.city
|
||||
// item["H"] = seaweed.region
|
||||
// item["I"] = seaweed.name
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
})
|
||||
+140
-73
@@ -1,82 +1,149 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.util.HouseValidator
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import net.sf.json.JSONObject
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\商务数据\\建设银行\\副本5市成交案例(测试).xlsx",{List item->
|
||||
// if(item["R"]){
|
||||
// return
|
||||
// }
|
||||
// def calcDate = null
|
||||
// if(item["E"]){
|
||||
// calcDate = item["E"].toString().substring(0, 10)
|
||||
// }else{
|
||||
// calcDate = item["B"].toString().substring(0, 10)
|
||||
// }
|
||||
// def floor = item["I"].toString().toDouble().toInteger().toString()
|
||||
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
|
||||
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
|
||||
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
|
||||
// def params =[:]
|
||||
// params.city = item["F"]
|
||||
// params.district = item["p"]
|
||||
// params.name = item['Q']
|
||||
// params.size = item["C"].toString()
|
||||
// params.floor = floor
|
||||
// params.maxFloor = maxFloor
|
||||
// params.direction = dir
|
||||
// params.decoration = fit
|
||||
// params.calDate = calcDate
|
||||
// params.__refresh_cache = "1"
|
||||
// params.scope = "1"
|
||||
// JSONObject evaResp = OpenApiClient.evaluate(params)
|
||||
// if(evaResp.result && evaResp.result.credit > 0.4){
|
||||
// item["R"] = evaResp.result.totalPrice
|
||||
// }
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
|
||||
//school.find(new BasicDBObject("source", "查学区")).each {BasicDBObject item->
|
||||
// item.category = item.attribute
|
||||
// item.attribute = null
|
||||
// school.save(item)
|
||||
// println(item)
|
||||
|
||||
// if(item["H"] != "中粮云景花园"){
|
||||
// return
|
||||
//}
|
||||
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\无经纬度学校-结果.xlsx").each {List item->
|
||||
// if(item["P"] != null){
|
||||
// def location = item["P"].split(",") as List
|
||||
// def data = new BasicDBObject()
|
||||
// data.map_name = item["N"]
|
||||
// data.map_region = item["R"]
|
||||
// data.map_address = item["N"]
|
||||
// data.map_type = item["S"]
|
||||
// data.location = [location[0].toString().toDouble(),location[1].toString().toDouble()]
|
||||
// school.update(new BasicDBObject("_id", new ObjectId(item["A"])), new BasicDBObject('$set', data))
|
||||
// println(data)
|
||||
// }
|
||||
def city = item["F"]
|
||||
def region = item["G"]
|
||||
def name = item["H"].toString()
|
||||
def addr = item["D"]
|
||||
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property" , false, region, name)
|
||||
if(resp.name){
|
||||
item["p"] = resp.district
|
||||
item['Q'] = resp.name
|
||||
//}
|
||||
regionCodeMap = [:]
|
||||
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
|
||||
for(def city : cityCodeMap.keySet()){
|
||||
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
|
||||
for(def d : datas){
|
||||
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
|
||||
}
|
||||
println(item)
|
||||
// def calcDate = null
|
||||
// if(item["E"]){
|
||||
// calcDate = item["E"].toString().substring(0, 10)
|
||||
// }else{
|
||||
// calcDate = item["B"].toString().substring(0, 10)
|
||||
// }
|
||||
// def floor = item["I"].toString().toDouble().toInteger().toString()
|
||||
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
|
||||
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
|
||||
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
|
||||
// def params =[:]
|
||||
// params.city = resp.city
|
||||
// params.district = resp.district
|
||||
// params.name = resp.name
|
||||
// params.size = item["C"].toString()
|
||||
// params.floor = floor
|
||||
// params.maxFloor = maxFloor
|
||||
// params.direction = dir
|
||||
// params.decoration = fit
|
||||
// params.calDate = calcDate
|
||||
// params.scope = "1"
|
||||
// JSONObject evaResp = OpenApiClient.evaluate(params)
|
||||
// if(evaResp.result && evaResp.result.credit > 0.4){
|
||||
// item["R"] = evaResp.result.totalPrice
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
}
|
||||
def data = Collections.synchronizedList([])
|
||||
data << ["ID","CITYID","DISTID","COMMNAME","LNG","LAT","ADDRESS","PS","PS_ADDR","PS_TYPE","PS_SEQNO","PS_LONGITUDE","PS_LATITUDE","MS","MS_ADDR","MS_TYPE","MS_SEQNO","MS_LONGITUDE","MS_LATITUDE","REMARK","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
cityCodeMap.keySet().each {def city->
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
|
||||
if(seaweed.lng){
|
||||
def item = []
|
||||
item << seaweed._id.toString()
|
||||
item << cityCodeMap.get(seaweed.city)
|
||||
item << regionCodeMap.get(seaweed.city + "-" + seaweed.region)
|
||||
item << seaweed.name
|
||||
def lng = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
|
||||
def lat = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
|
||||
item << lng
|
||||
item << lat
|
||||
item << seaweed.address
|
||||
BasicDBObject location = new BasicDBObject()
|
||||
BasicDBObject near = new BasicDBObject()
|
||||
BasicDBObject geometry = new BasicDBObject("type", "Point")
|
||||
geometry.append("coordinates", [lng, lat])
|
||||
near.append('$maxDistance', 1000)
|
||||
near.append('$geometry', geometry)
|
||||
location.append('$near', near)
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.append("location", location)
|
||||
query.append("attribute",new BasicDBObject('$in', ['省重点', '市重点', '区重点', '县重点', '省级示范']))
|
||||
query.append("category", new BasicDBObject('$in', ['中学']))
|
||||
BasicDBObject midSchool = school.findOne(query)
|
||||
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
|
||||
// query.append("location", location)
|
||||
// query.append("attribute", level)
|
||||
// midSchool = school.findOne(query)
|
||||
// if(midSchool){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(!midSchool){
|
||||
// query.removeField("attribute")
|
||||
// midSchool = school.findOne(query)
|
||||
// }
|
||||
BasicDBObject primarySchool=null
|
||||
query.append("category", new BasicDBObject('$in', ['小学']))
|
||||
primarySchool = school.findOne(query)
|
||||
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
|
||||
// query.append("location", location)
|
||||
// query.append("attribute", level)
|
||||
// primarySchool = school.findOne(query)
|
||||
// if(primarySchool){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(!primarySchool){
|
||||
// query.removeField("attribute")
|
||||
// primarySchool = school.findOne(query)
|
||||
// }
|
||||
if(primarySchool){
|
||||
item << primarySchool.name
|
||||
item << primarySchool.address
|
||||
item << primarySchool.attribute
|
||||
item << null
|
||||
item << primarySchool.location[0]
|
||||
item << primarySchool.location[1]
|
||||
}else {
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
}
|
||||
if(midSchool){
|
||||
item << midSchool.name
|
||||
item << midSchool.address
|
||||
item << midSchool.attribute
|
||||
item << null
|
||||
item << midSchool.location[0]
|
||||
item << midSchool.location[1]
|
||||
}else {
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
}
|
||||
item << null
|
||||
Date c_date = seaweed.c_date
|
||||
Date m_date = seaweed.m_date
|
||||
if(!c_date && !m_date){
|
||||
c_date = DateUtil.add(new Date(), Calendar.YEAR, -1)
|
||||
}
|
||||
if(!c_date){
|
||||
c_date = m_date
|
||||
}
|
||||
if(!m_date){
|
||||
m_date = c_date
|
||||
}
|
||||
item <<c_date.format("yyyy-MM-dd HH:mm:ss")
|
||||
item <<m_date.format("yyyy-MM-dd HH:mm:ss")
|
||||
item << 'fjw'
|
||||
println(item)
|
||||
if(primarySchool || midSchool){
|
||||
data << item
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
ExcelUtils.write("D:\\无学区房数据.xlsx", data, "结果")
|
||||
|
||||
|
||||
})
|
||||
|
||||
|
||||
+2
-14
@@ -1,20 +1,12 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.and
|
||||
import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
//
|
||||
//def data = [["城市", "区域", "小区名", "2018", "全部"]]
|
||||
//cityList = ["上海"]
|
||||
@@ -47,10 +39,6 @@ import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
//HttpRequest httpRequest = new HttpRequest()
|
||||
//println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8")
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
MongoCollection<Document> AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config")
|
||||
AVERAGER_CONFIG.find().each {Document doc->
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20201101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20201201", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
+3
-9
@@ -1,19 +1,13 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -61,7 +55,7 @@ import java.sql.Statement
|
||||
//println(bigMarkupCount)
|
||||
|
||||
def countHouses(String city, String region, String name){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-15
@@ -1,28 +1,16 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
def poiData = Collections.synchronizedList([])
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item ->
|
||||
@@ -79,7 +67,7 @@ def poiData = Collections.synchronizedList([])
|
||||
// println(item)
|
||||
//},1)
|
||||
def findHouseCount(String city, String region, String name, List alias) {
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user