Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_bohai

# Conflicts:
#	projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java
This commit is contained in:
2022-05-11 13:41:38 +08:00
122 changed files with 11488 additions and 880 deletions
@@ -13,7 +13,7 @@ import java.util.*;
*/
public class BaiDuApiUtils {
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
private static String AK = "ajgWIBd9rYsXed1hcDQqYvkX";
public static String[] AKS = new String[]{"hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
@@ -25,6 +25,7 @@ import java.util.zip.GZIPOutputStream;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.dictionary.py.Pinyin;
import com.sun.javafx.scene.transform.TransformUtils;
import net.sf.json.JSON;
import net.sf.json.JSONArray;
import net.sf.json.JSONNull;
@@ -1435,6 +1436,25 @@ public class DataUtil {
return null;
}
/**
* 匹配符合条件的字符串
*
* @param str
* @param pattern
* @return
*/
public static List<String> matchAll(String str, Pattern pattern) {
if (str == null) {
return null;
}
List list = new ArrayList();
Matcher matcher = pattern.matcher(str);
while (matcher.find()) {
list.add(matcher.group());
}
return list;
}
public static boolean isNumeric(String str) {
try {
Double.parseDouble(str);
@@ -1805,11 +1825,26 @@ public class DataUtil {
}
}
public static void main(String[] args) {
System.out.println(getFirstChar("文裕花园"));
System.out.println(getFirstChar("文怡花园"));
public static boolean isChineseChar(String c) {
if (StringUtils.isNotBlank(c)) {
String temp = String.valueOf(c);
Pattern pattern = Pattern.compile("[\\u4e00-\\u9fa5\\w]+");
Matcher matcher = pattern.matcher(temp);
return matcher.matches();
}
return false;
}
System.out.println(FourUpSixInto("258.393", 2));
public static void main(String[] args) {
System.out.println(isChineseChar("6文怡a花园"));
// System.out.println(getFirstChar("文怡花园"));
//
// System.out.println(FourUpSixInto("258.393", 2));
}
@@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils {
public static void main(String[] args) {
for (int i = 0; i < 1000; i++) {
System.out.println(random(0, 10));
System.out.println(random(-9, 9));
}
}
}
@@ -1,5 +1,9 @@
package com.fangjia.coresite.controllers;
import java.io.UnsupportedEncodingException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URLEncoder;
import java.util.Date;
import java.util.List;
@@ -7,6 +11,7 @@ import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.fangjia.core.util.DataUtil;
import org.apache.commons.lang.StringUtils;
import com.fangjia.coresite.model.JsonModel;
@@ -37,7 +42,7 @@ public class SaleLeadsController {
}
public Object doAddAction(SaleLeadsModel model, HttpServletRequest req,
HttpServletResponse res) {
HttpServletResponse res) throws UnsupportedEncodingException, NoSuchMethodException, IllegalAccessException, InvocationTargetException {
model.setViewType("redirect");
String refUrl = model.getBackUrl();
@@ -122,6 +127,8 @@ public class SaleLeadsController {
}
}
saleLeads = converValue(saleLeads);
saleLeadsService.save(saleLeads);
if(StringUtils.isNotBlank(saleLeads.getBusiness()) && saleLeads.getBusiness().contains("_")){
@@ -162,6 +169,30 @@ public class SaleLeadsController {
model.setViewParam("location", url);
return model;
}
private SaleLeads converValue(SaleLeads saleLeads) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, UnsupportedEncodingException {
//
Field[] fields = saleLeads.getClass().getDeclaredFields();
for(Field s : fields){
String name = s.getName();
String type = s.getGenericType().toString();
if(type.contains("java.lang.String")){
String fildName = name.substring(0,1).toUpperCase()+name.substring(1);
Method m = saleLeads.getClass().getMethod("get"+fildName);
String value = (String)m.invoke(saleLeads);
if(StringUtils.isNotBlank(value)){
String s1 = value.replaceAll("\\pP|\\pS", "");
if(!DataUtil.isChineseChar(s1)){
String newValue = new String(value.getBytes("ISO8859-1"), "UTF-8");
m = saleLeads.getClass().getMethod("set"+fildName,String.class);
m.invoke(saleLeads, newValue);
}
}
}
}
return saleLeads;
}
public Object addDownInfoAction(SaleLeadsModel model, HttpServletRequest req,
@@ -102,10 +102,10 @@ public class Loudong implements MongoEntity {
*/
private String nameInSeaweed;
/**
* 小区编号
* seaweed中的小区id
*/
private String idInSeaweed;
@Column(name = "remark")
public String getRemark() {
return remark;
@@ -38,6 +38,7 @@ public class OpenConfig extends Configuration {
private final String ipdatfile;
private final String baiduAk;
private final String defaultCity;
private final boolean isOpenAllAuthority;
@@ -54,6 +55,7 @@ public class OpenConfig extends Configuration {
baiduAk = configuration.get("fangjia.baidu.ak");
isOpenAllAuthority = configuration.getBoolean("fangjia.isOpenAllAuthority", false);
enabledAddressMatch = configuration.getBoolean("fangjia.addressMatch.enabled",false);
defaultCity = configuration.get("fangjia.default.city");
}
public static OpenConfig getConfig() {
@@ -134,4 +136,7 @@ public class OpenConfig extends Configuration {
return enabledAddressMatch;
}
public String getDefaultCity() {
return defaultCity;
}
}
@@ -89,6 +89,7 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService;
import com.fangjia.service.open.WaitCrawlKeywordsService;
import com.fangjia.service.sql.FjtEvaluationService;
import com.fangjia.service.web.AddressService;
import org.sothis.web.mvc.annotation.Param;
@Authority(role = Role.USER)
public class AddressController {
@@ -106,6 +107,7 @@ public class AddressController {
private HouseLoudongService houseLoudongService;
private FjtEvaluationService fjtEvaluationService;
private AddressMatchProxyService addressMatchProxyService;
private final static String AK = OpenConfig.getConfig().getBaiduAk();
private final static Logger LOGGER = LoggerFactory.getLogger(AddressController.class);
@@ -117,6 +119,8 @@ public class AddressController {
private static final String GRADE_NATION = "全国评分";
private static final String GRADE_CONSUME = "消费评分";
public final static OpenConfig CONFIG = OpenConfig.getConfig();
/**
* @deprecated see parse2Action
* @param model
@@ -1705,6 +1709,228 @@ public class AddressController {
return mav;
}
/**
* 地址标准化接口V2
*
* @param model
* @return
*/
public Object standard2Action(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 五级地址标准化接口
*
* @param model
* @return
*/
public Object fiveDivisionStructureAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
// 地址验证无效也输出标准化结果
params.put("flag", "true");
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 五级行政区划V2地址标准化接口
*
* @param model
* @return
*/
public Object fiveDivisionStructure2Action(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
// 地址验证无效也输出标准化结果
params.put("flag", "true");
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 详细地址标准化接口
*
* @param model
* @return
*/
public Object divisionStructureAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
// 地址验证无效也输出标准化结果
params.put("flag", "true");
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 地址标准化专用测试接口
* @param model
* @return
*/
public Object structureAndCompleteAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
params.put("address", model.getAddress());
// 地址验证无效也输出标准化结果
params.put("flag", "true");
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if(json.containsKey("result")){
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
public Object similarityAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
@@ -665,18 +665,6 @@ public class PropertyController {
return mav;
}
String userId = FangjiaApiUtil.parseUserId(model.getToken());
/**超级用户和上海交大账户不控制权限*/
if (!"1007497".equals(userId) && !"1007632".equals(userId) && !"1007548".equals(userId)
&& !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId)
&& !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId)
&& !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId)
&& !"1007747".equals(userId) && !"1007757".equals(userId)
&& model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制
mav.setModel(BasicModel.buildUnAuthorizedModel());
return mav;
}
District district = null;
if (StringUtils.isNotBlank(model.getId())){
if(ObjectId.isValid(model.getId())){
@@ -961,6 +949,12 @@ public class PropertyController {
return mav;
}
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
if(loudong != null && loudong.getFloors() >= model.getFloor()){
model.setMaxFloor(loudong.getFloors());
}
}
model.setRegion(district.getRegion());
model.setDistrictName(district.getName());
if (StringUtils.isBlank(model.getPropertyType())) {
@@ -0,0 +1,36 @@
package com.fangjia.open.controllers;
import com.fangjia.core.util.AnalyticsUtil;
import com.fangjia.open.interceptors.Authority;
import com.fangjia.open.interceptors.Role;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import javax.servlet.http.HttpServletRequest;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
public class requestController {
/**
* »ñÈ¡ÇëÇóµÄrequestºÍIPÐÅÏ¢
* @param request
* @return
*/
public Object requestAction(HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
String ip = AnalyticsUtil.getIpAddress(request);
Map<String, String> map = new HashMap<String, String>();
Enumeration headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String key = (String) headerNames.nextElement();
String value = request.getHeader(key);
map.put(key, value);
}
map.put("ip", ip);
mav.setModel(BasicModel.buildOKModel().setResult(map));
return mav;
}
}
@@ -100,4 +100,5 @@ fangjia.jms.email.queue=email.queue
fangjia.redis_nodes=192.168.0.79:6379,192.168.0.79:6380,192.168.0.79:6381,192.168.0.81:6382,192.168.0.81:6383,192.168.0.81:6384
fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2
fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2
fangjia.default.city = \u91CD\u5E86\u5E02
@@ -1,8 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<urlrewrite>
<global-params>
<param value="st" name="sort" type="attach"/>
<param value="pn" name="pager.currentPage" type="attach"/>
<!-- <param value="st" name="sort" type="attach"/>-->
<!-- <param value="pn" name="pager.currentPage" type="attach"/>-->
</global-params>
<rules>
<rule controller="city" action="condition2" channel="v2">
@@ -33,5 +33,15 @@
<ladder>/property/avgPrice</ladder>
</ladders>
</rule>
<rule controller="address" action="standard2" channel="v2">
<ladders>
<ladder>/address/standard</ladder>
</ladders>
</rule>
<rule controller="address" action="fiveDivisionStructure2" channel="v2">
<ladders>
<ladder>/address/fiveDivisionStructure</ladder>
</ladders>
</rule>
</rules>
</urlrewrite>
+27
View File
@@ -494,5 +494,32 @@
<artifactId>msgpack</artifactId>
<version>0.6.12</version>
</dependency>
<dependency>
<groupId>com.asprise.ocr</groupId>
<artifactId>java-ocr-api</artifactId>
<version>[15,)</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>4.1.0</version>
</dependency>
<dependency>
<groupId>net.sourceforge.tess4j</groupId>
<artifactId>tess4j</artifactId>
<version>3.4.0</version>
<exclusions>
<exclusion>
<groupId>com.sun.jna</groupId>
<artifactId>jna</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>
@@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("city", city)
q.put("status", 1)
// q.put("name","ÁÉÔ´Ò»´å")
def dbcursor = LOU_DONG_STAT.find(q)
def map = [:]
def set = new HashSet()
for(def db : dbcursor){
def list = map.get(db.region+"##"+db.name)
if(!list){
list = []
map.put(db.region+"##"+db.name, list)
}
list << db
if(db.house_num && db.house_num!="--"){
def key = db.region+"##"+db.name +"##"+db.house_num
if(!set.contains(key)){
set << key
list << db
}
}
}
return map
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){
def rets = []
def keys = []
distMap.each{k,dist->
def buildings = buildingMap.get(k)
if(buildings){
@@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{
def ret = []
def areaData = distCodeMap.get(obj.region)
if(areaData){
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << ""
ret << ""
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
def key = dist._id.toString() +"##"+obj.house_num
if(!keys.contains(key)){
keys << key
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << obj.house_num
ret << ""
}
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
if(arrs){
ret << arrs[0].trim()
}else{
ret << ''
}
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
ret << ""
}
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
if(arrs){
ret << arrs[0].trim()
}else{
ret << ''
}
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1){
// ret << "1"
// }else{
// ret << "0"
// }
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1) {
// ret << obj.elevators//µçÌÝÊý
// }else{
// ret << ""
// }
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
}
}
}
}
@@ -54,10 +54,10 @@ class BigdataDealBatch extends BaseBatch{
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
for(def city : citys){
addDealPriceMatch(city);
}
println citys.unique().size()
// for(def city : citys){
// addDealPriceMatch(city);
// }
// println citys.unique().size()
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
q.put("status", 0)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -149,8 +149,8 @@ class CcbCommonUtils {
public static def findTradesDetailById(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
// q.put("city", dist.city)
// q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
q.put("avg_price", new BasicDBObject('$gt', 0))
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
@@ -161,7 +161,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -367,7 +367,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","长隆小区")
// q.put("name","ÁÉÔ´Ò»´å")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["ĆŐͨץՏ","ÉĚץ"]))
@@ -488,4 +488,27 @@ class CcbCommonUtils {
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id), new BasicDBObject('$set',udata))
}
public static def findDistrictSeqno(){
def sql = "select cityid,commid,seqno from bigdata_position_seqno";
def map = new HashMap()
sqlInstance.eachRow(sql,{r->
def detailMap = map.get(r.cityid)
if(!detailMap){
detailMap = [:]
map.put(r.cityid, detailMap)
}
detailMap.put(r.commid, r.seqno)
});
return map
}
public static def findMaxSeqno(def cityid){
def sql = "select max(seqno) as max_seqno from bigdata_position_seqno where cityid='${cityid}'"
def maxSeqno = null;
sqlInstance.eachRow(sql,{r->
maxSeqno = r.max_seqno
});
return maxSeqno
}
}
@@ -0,0 +1,77 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jiangnanblank.SFTP
import com.jcraft.jsch.ChannelSftp
import org.apache.commons.codec.digest.DigestUtils
class CcbFtpTestUtils {
static def ftpServer = "192.168.0.86"
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb/output/"
static def ftpPath_rh = "/home/ccb_rh/output/"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
static def ftpUpload(type,outputPath_dist, month){
//上传到ftp
sftpChannel = reConnect();
if(type && type=="rh"){
sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel);
}else{
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
}
}
static def delete(path){
if(path.startsWith("/")){
sftp.deleteSFTP(path, sftpChannel)
}else{
sftp.deleteFile(path)
}
}
static def reConnect(){
if(sftpChannel == null){
println "sftpChannel 为空,需重连....."
if(sftp == null){
sftp = new SFTP();
}
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
}else{
println "sftpChannel 正常....."
}
return sftpChannel;
}
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
def data = new FileInputStream(new File(dataFileName))
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static main(args) {
}
}
@@ -25,8 +25,8 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()
task.delDealByPe()
task.deleteThirdPartyData()
// task.delDealByPe()
}
def delDealByPe(){
@@ -370,26 +370,26 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = ["东莞"] //,"杭州""广州",
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202110
def month = 202203
def rows = []
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202111)
def rets = findPeGteThirdParty(city, 202204)
def total = 0
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance.executeUpdate(sql)
sqlInstance236.executeUpdate(sql)
}
println city +" "+total
println city +" "+rets.size()
}
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='第三方' and cityname='${cityName}' and pe>20"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='µÚÈý·½' and cityname='${cityName}' and pe>50"
def rets = []
sqlInstance2.eachRow(sql,{r->
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.commid = r.commid
map.price = r.target_avg_price
@@ -1,15 +1,20 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -37,15 +42,74 @@ class DistrictDelete {
dataSource.setProperties(connectionProperties)
}
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new DistrictDelete()
// task.run()
// task.gzRun();
// task.deleteDistrict()
task.deleteCommPrice()
// task.deleteCommPrice()
// task.distHouseCountStat()
task.addThirdPartyStat()
// task.processRegion()
}
def processRegion(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d->
if(d['B']){
def region = RegionData.formatRegion("重庆",d['B'])
if(region){
d['B'] = region
}
}
},0)
}
def addThirdPartyStat(){
def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1)
def idMaps = [:]
for(def d : datas){
def id = d['A']
idMaps.put(id, d['D'])
}
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
def totalCount = new AtomicLong()
def map = findThirdPartyPrice("上海")
final CountDownLatch latch = new CountDownLatch(dists.size())
def thirdLists = Collections.synchronizedList(new ArrayList());
thirdLists << ["ID","区域","名称","总量"]
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = idMaps.get(d._id.toString())
if (!count) {
def dist = CommonDatautils.findDistrictById(d._id.toString())
def priceMap = map.get(dist.region + "##" + dist.name)
if (priceMap) {
thirdLists << [dist._id.toString(), dist.region, dist.name, priceMap.size().toString()]
println dist._id.toString()+"@"+ dist.region+"@"+ dist.name+"@"+ priceMap.size()
}
}
// println totalCount.get() + " " + d.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\重庆农商行\\上海第三方.xlsx",thirdLists, "房源")
}
def run(){
@@ -60,6 +124,40 @@ class DistrictDelete {
}
}
def distHouseCountStat(){
def city = "重庆"
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def distCount = districts.size()
def rets = Collections.synchronizedList(new ArrayList());
rets << ["ID","区域","名称","总量"]
for(def i=0;i<districts.size();i++) {
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def count = findHouseCount(city, dist.region, dist.name, dist.alias)
def count = findHouses(city, dist.region, "sell", 20210701,20210801, dist.name, dist.alias)
println totalCount.get() +" "+city +" "+dist.region+" "+dist.name+" "+count
if(count){
rets << [dist._id.toString(), dist.region, dist.name, count]
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆7月份的房源统计.xlsx",rets, "房源")
}
def checkDist(city,dists){
def rets = Collections.synchronizedList(new ArrayList());
@@ -259,4 +357,82 @@ class DistrictDelete {
sqlInstance.execute(sql)
println "ɾ³ý£º"+tableName+" "+commid
}
def findThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city",city)
q.put("s_date", new BasicDBObject('$gte', 202112).append('$lte',202112))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.region+"##"+db.name
def prices = map.get(key)
if(!prices){
prices = []
map.put(key, prices)
}
if( db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
prices << db.zhugezhaofang_esf_price
}
if(db.anjuke_esf_price && db.anjuke_esf_price>3000){
prices << db.anjuke_esf_price
}
if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
prices << db.get("58tc_esf_price")
}
if(db.soufang_esf_price && db.soufang_esf_price>3000){
prices << db.soufang_esf_price
}
if(db.lianjia_esf_price && db.lianjia_esf_price>3000){
prices << db.lianjia_esf_price
}
if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
prices << db.cityhouse_esf_price
}
if(db.maitian_esf_price && db.maitian_esf_price>3000){
prices << db.maitian_esf_price
}
if(db.beike_esf_price && db.beike_esf_price>3000){
prices << db.beike_esf_price
}
if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
prices << db.get("365house_esf_price")
}
if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
prices << db.zhongyuan_esf_price
}
if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
prices << db.tuboshi_esf_price
}
if( db.qfang_esf_price && db.qfang_esf_price>3000){
prices << db.qfang_esf_price
}
if(db.fangtan_esf_price && db.fangtan_esf_price>3000){
prices << db.fangtan_esf_price
}
if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
prices << db.fangduoduo_esf_price
}
if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
prices << db.get("5i5j_esf_price")
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
}
return map
}
}
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
def prices = CcbCommonUtils.findGuidePrice(city, "2022-03-1", "2022-04-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -21,26 +21,25 @@ class ExportOracleData {
}
def run() {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// //不会变
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
//
// exportGuidedPrice()//随人工改变
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
//
// //处理小区情况
// exportBigdataCommunity()
// exportCommunityStaging()
// exportCommunityBuileType()
//不会变
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
// exportThirdPartyPrice2(lastMonth)
// exportBigdataForsale("2021-12-19")
// exportBigdataDeal("2021-12-01")
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
// //处理小区情况
exportBigdataCommunity()
exportCommunityStaging()
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-03-19")
exportBigdataDeal("2022-03-01")
//计算结果相关
@@ -56,11 +55,11 @@ class ExportOracleData {
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// def curDate = curMonth.toString()+"28"
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//ok文件
exportLastFile()
// exportLastFile()
}
@@ -0,0 +1,416 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new ExportOracleEvaluationData()
task.run()
}
def run() {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
//计算结果相关
exportAvmEvaluationGJNB(curMonth)
exportAVMEvaluationItemGJNB(curMonth)
exportCityPrice(curMonth)
exportRegionPrice(curMonth)
exportCommunityBasePriceGj(curMonth)//已合下面的合并
exportCommunityBasePrice(curMonth)
exportCommunityPdescribeGj(curMonth)
exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// ok文件
exportOkFile()
}
def exportAvmEvaluationGJNB(yearMonth){
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMMUNITY_NUM
data << r.HAS_PRICE_COMMUNITY_NUM
data << r.PRICE_COVERAGE
data << r.ARTIFICIAL_CALC_NUM
data << r.ARTIFICIAL_CALC_COVERAGE
data << r.SAMPLE_CALC_NUM
data << r.SAMPLE_CALC_COVERAGE
data << r.ANN_CALC_NUM
data << r.ANN_CALC_COVERAGE
data << r.TEST_NUM
data << r.TEST_COMM_NUM
data << r.TEST_COMM_COVERAGE
data << r.PPE15_COVERAGE
data << r.RATE_MEDIAN
data << r.COEFFICIENT
data << r.DEAL_TEST_CASE_NUM
data << r.DEAL_TEST_COMM_NUM
data << r.DEAL_TEST_COMM_COVERAGE
data << r.DEAL_PPE15_COVERAGE
data << r.DEAL_COEFFICIENT
data << r.DEAL_RATE_MEDIAN
data << r.THIRD_TEST_CASE_NUM
data << r.THIRD_TEST_COMM_NUM
data << r.THIRD_TEST_COMM_COVERAGE
data << r.THIRD_PPE15_COVERAGE
data << r.THIRD_COEFFICIENT
data << r.THIRD_RATE_MEDIAN
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
}
def exportAVMEvaluationItemGJNB(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.CONFIDENCE
data << r.TARGET_AVG_PRICE
data << r.DEALDATE
data << r.UNIT_PRICE
data << r.BUILDAREA
data << r.FITMENT
data << r.HOUSEASPECT
data << r.FLOORNUM
data << r.FLOORCNT
data << r.DATA_TYPE
data << r.RATE
data << r.PE
data << r.CREATETIME
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
}
def exportCommunityBasePrice(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityBasePriceGj(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
}
def exportCityPrice(month){
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "CITY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "CITY_PRICE_GJ 上传Ftp完成......"
}
def exportRegionPrice(month){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
}
def exportCommunityPdescribeGj(month){
// def citys = CcbCommonUtils.citys;
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//
// for(def city : citys) {
// def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// }
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
}
def exportCommCaseDetailGj(month){
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.COMMUNITY_ID
data << r.NETLISTING_ID
data << r.PRICE_DATE
data << r.CREATE_DATE
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
});
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
println "comm_case_detail_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "comm_case_detail_gj 上传Ftp完成......"
}
def exportOkFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def dataCodeFileName = "/ccb/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
}
}
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -23,9 +23,10 @@ class LoadGuide {
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
}
def match(def filename){
@@ -48,15 +49,15 @@ class LoadGuide {
def datas = ExcelUtils.read(filePath,0, 1)
def rets = []
for(def d : datas){
if(d['E']){
if(d['D']){
def o = new BasicDBObject()
o.city = d['E']
o.city = d['D']
o.region = d['A']
o.name = d['B']
o.avg_price = d['C'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.match_city = d['D']
o.match_region = d['E']
o.match_name = d['F']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
@@ -224,18 +224,43 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
}
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
def testMap = findCaluateCityPPE(sqlInstance236, 202201)
ExcelUtils.modify(fileName,{d->
if(d['C']){
def oldcount = dealMap.get(d['C'])
d['T'] = oldcount
def casecount = testMap.get(d['C'])
d['U'] = casecount
println d
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202203)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['X'] = oldcount
// def casecount = testMap.get(d['C'])
// d['Y'] = casecount
// println d
// }
//},0)
def rows = findTradePpeData(sqlInstance236)
// rows.unique()
def map=findTradeDetail(sqlInstance236,null)
def datas=[]
map.each{key,row->
if(!rows.contains(key)){
datas<<row
}
},0)
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def datas=[]
//cityCodeDataMap.each{city,code->
// def rows = findTradePpeData(sqlInstance236, 202203,code)
// def alltrades=findTradeDetail(sqlInstance236,code)
// alltrades.each{it->
// def commid=it[0]
// if(!rows.contains(commid)){
// datas<<it
// }
// }
// println datas.size()
//}
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -271,19 +296,24 @@ ExcelUtils.modify(fileName,{d->
//},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d->
// if(d['A']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
// def flag = d['M']
// if(flag==0.0){
// if(d['D']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['A']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['D']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// }
// }
//
//},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
// if(d['A']){
// def region = d['B']
@@ -553,8 +583,30 @@ def findCaluateCityPPE(def sqlInstance, def yearMonth){
}
def findTradePpeData(def sqlInstance){
def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'";
def rows = new HashSet()
sqlInstance.eachRow(sql,{r->
rows<<r.commid+"##"+r.buildarea+"##"+r.target_avg_price+"##"+r.dealdate
});
return rows
}
def findTradeDetail(def sqlInstance, def cityid){
def sql = "select commid,cityname,distname,commname,buildarea,dealunitprice,to_char(dealdate,'yyyy-mm-dd') as dealdate from bigdata_deal_gj where dealdate>=to_date('2022-02-01','yyyy-MM-dd')";
def map = [:]
sqlInstance.eachRow(sql,{r->
def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate
def row=[]
row<<r.commid
row<<r.cityname
row<<r.distname
row<<r.commname
row<<r.buildarea
row<<r.dealunitprice
map.put(key,row)
});
return map
}
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
def count = 0
@@ -777,6 +829,23 @@ def getStreetInfo(def city, def glocation){
//}
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据")
//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1)
//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1)
//
//def keys = []
//for(def d : datas2){
// keys << d['A']+"##"+d['C']+"##"+d['B']
//}
//
//def index = 0
//for(def d : datas1){
// def k = d['A']+"##"+d['C']+"##"+d['B']
// if(!keys.contains(k)){
// index ++
// println index +" "+k
// }
//}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202110
def endDate = 202112
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202201
def endDate = 202203
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -9,6 +9,7 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import groovy.sql.Sql
import org.bson.types.ObjectId
@@ -48,6 +49,7 @@ class TradeStatLog {
// task.deleteDealData()
// task.deleteDealData2()
// task.statTableCount()
// task.filterTrade()
}
@@ -73,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-10-01"
def date = "2021-11-01"
def endDate = "2021-12-01"
def newDate = "2022-01-01"
def startDate = "2022-01-01"
def date = "2022-02-01"
def endDate = "2022-03-01"
def newDate = "2022-04-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -129,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -479,8 +481,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2021-12-01"
def endDate = "2022-01-01"
def startDate = "2022-03-01"
def endDate = "2022-04-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -561,7 +563,7 @@ class TradeStatLog {
}
println city +" "+rets.size()
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
}
def deleteDistrict(){
@@ -706,4 +708,24 @@ class TradeStatLog {
}
def filterTrade(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx",{d->
if(!d['J']){
def count = findTradeById(d['A'])
if(count){
d['J'] = "其它原因过滤"
}
}
},0)
}
def findTradeById(id){
def q = new BasicDBObject()
q.put("_id", new ObjectId(id))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
}
@@ -2,25 +2,243 @@ package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
class CcbDistrictToDB {
static main(args) {
def task = new CcbDistrictToDB();
// task.run()
task.run1()
// task.run1()
// task.meituanRun()
// task.meituanSqrentRun()
// task.updateDistrict()
task.updateDongguangRegion()
}
def updateDongguangRegion(){
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"])
for(def d : dist){
def t = d.c_date
if(t.getTime()>cDate.getTime()){
}
}
}
def updateDistrict(){
def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"]
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
def index = 0
for(def city : citys){
def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"])
for(def d : dist){
def t = d.c_date
if(t.getTime()>cDate.getTime()){
if(d.name.contains("(")){
def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1)
def newName = d.name.replace(delstr,"")
index++
println index+" "+d.city+" "+d.name+" "+newName
def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district")
if(!obj){
println "update"
def udata = new BasicDBObject()
udata.put("name", newName)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
d.name = newName
addPrice(d)
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0)
}else{
println "delete"
DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
}
}
}
}
}
}
def meituanSqrentRun(){
ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion) {
def name = d['H']
name = nameFileter(name)
if(name){
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
}
}
},0);
}
def meituanRun(){
def index = 0//美团需入库小区合并后.xlsx
def keys = new CopyOnWriteArrayList()
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['H']
name = nameFileter(name)
def formatAddress = null
if(d['E']){
formatAddress = AddressUtil.formatAddress(city, d['E'])
}
if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
def map = findStandardObject2(city,region, name, d['E'], rets);
if(map && map.address){
formatAddress = AddressUtil.formatAddress(city, map.address)
if(!formatAddress){
if(map.address.contains("交叉口")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
}else if(map.address.contains("交叉")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
}
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
def stop = false
while (!stop){
if(data._id.toString()){
addPrice(data)
stop = true;
}
}
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
},0);
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def run1(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
def city = d['A']
def region = d['B']
def newRegion = RegionData.formatRegion(city, region)
@@ -71,6 +289,7 @@ class CcbDistrictToDB {
if(name){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
@@ -221,4 +440,77 @@ class CcbDistrictToDB {
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
return name;
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -12,9 +12,90 @@ import com.fangjia.core.util.BaiDuApiUtils
class DistrictMarge {
static main(args) {
def task = new DistrictMarge()
task.run()
//task.run()
task.repeatAddress()
}
def repeatAddress(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "lng","address","name_cat"])
findRepead(c, districts)
}
}
def findRepead(city, districts){
def map = [:]
for(def d : districts){
if(d.name_cat == 0 && d.lng){
if(d.address && d.address.contains("号")){
def list = map.get(d.address)
if(!list){
list = []
map.put(d.address, list)
}
list << d
}
}
}
def newMap = [:]
map.each {k,vals->
if(vals.size()>=2){
vals.sort{o1,o2->
return o1.name.size() <=> o2.name.size()
}
newMap.put(k,vals)
}
}
newMap.each {k,vals->
def names = []
for(def d : vals){
def newKey = d.name
if(d.name.contains("小区")){
newKey = d.name.toString().replace("小区", "")
}
if(!names.contains(newKey)){
names << newKey
}else{
d.flag = "删除"
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId( d._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
def values = newMap.values()
def allRets = []
for(def list : values){
allRets.addAll(list)
}
def results = []
def delcount = 0
for(def r : allRets){
def rows = []
rows << r._id.toString()
rows << r.city
rows << r.name
rows << r.address
rows << r.name_cat
if(r.flag){
rows << r.flag
delcount++
}
results << rows
}
// ExcelUtils.write("d:\\数据\\小区\\删除重复地址(${city}).xlsx", results, "sheet")
println "${city} 共:"+map.size()+" 2个以上:"+newMap.size()+" delcount:"+delcount
}
def run(){
def rets = getDatas()
def deleteIndex = 0
@@ -1,24 +1,123 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import groovy.sql.Sql
DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43")
DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43")
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", ZS_MONGODB_URL)
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d->
if(d['B']){
//def avg_price =CommonDatautils.queryCityPrice(d['B'])
//d['C'] = avg_price
// def q = new BasicDBObject()
// q.put("area_name",new BasicDBObject( '$regex' , d['B'].replace("州","")+/.*$/))
// def obj = COMMUNITY_CONN.findOne(q)
// if(obj){
// d['D'] = obj.area_code
// println obj.area_name +" "+obj.area_code
// }
//def obj = findDistrictPriceCount(d['B'], DateUtils.parseFromTightByPattern("2022-03-13","yyyy-MM-dd"), CITY_INFO)
// d['D'] = CommonDatautils.findAllDistrictCount("district", d['B'])
// d['E'] = findHouseCount(d['B'], 20210101, 20211231)
//
// def startDate = DateUtils.parseFromTightByPattern("2021-01-01", "yyyy-MM-dd")
// def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
// def deals = findTradesDetailById(d['B'], startDate,endDate, DEAL_PRICE_COLL)
// d['F'] = deals.size()
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
d['G'] = findCcbDistCount(cityCodeDataMap.get(d['B']), sqlInstance)
}
},0)
def findCcbDistCount(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_community_gj where cityId='${cityid}' and status=0 ";
def total = null
sqlInstance.eachRow(sql,{r->
total = r.count
})
return total
}
public static def findTradesDetailById(city, startDate, endDate, DEAL_PRICE_COLL){
def q = new BasicDBObject()
q.put("city", city)
q.put("area", new BasicDBObject('$gte', 20))
q.put("avg_price", new BasicDBObject('$gt', 0))
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
lists << d
}
}
return lists
}
def findHouseCount(def city, def startDate, def endDate){
def aliasStr = null;
def total = 0
try{
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} "
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
})
}catch(Exception e){
e.printStackTrace()
}
return total
}
def findDistrictPriceCount(def city, def weekend, def CITY_INFO){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def obj = CITY_INFO.findOne(city)
return obj
}
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d ->
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// if(newRegion && !d['K']) {
@@ -433,23 +532,67 @@ def city = "
// }
//})
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
if(dist){
def alias = dist.mapping_alias
if(!alias){
alias = []
}
if(alias && !alias.contains(d['C'])){
index++
println index
alias << d['C']
def udata = new BasicDBObject()
udata.put("mapping_alias",alias)
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
}
}
}
})
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
// if (d['B']) {
// def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
// if(dist){
// def alias = dist.mapping_alias
// if(!alias){
// alias = []
// }
// if(alias && !alias.contains(d['C'])){
// index++
// println index
// alias << d['C']
// def udata = new BasicDBObject()
// udata.put("mapping_alias",alias)
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// }
// }
// }
//})
//ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
// if(d['A']){
// try{
// def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
// if(dist){
// d['B'] = dist.city
// d['C'] = dist.region
// d['D'] = dist.name
//
// def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
// d['E'] = price
// }
// }catch(Exception e){
// println d['A']
// }
//
// }
//
//})
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d ->
// if(d['A']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def mappingAlias = dist.mapping_alias
// if(mappingAlias){
// if(mappingAlias.contains(d['B'])){
// mappingAlias.remove(d['B'])
// DBObject udata = new BasicDBObject();
// udata.put("mapping_alias", mappingAlias);
// udata.put("m_date", new Date());
// DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata))
// }else{
// println d['A']+"无映射别名"
// }
// }else{
// println d['A']+"映射别名为空"
// }
// }else{
// println d['A']+"没找到"
// }
// }
//})
@@ -0,0 +1,89 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
class MapAddressMatch {
static main(args) {
def task = new MapAddressMatch()
task.run()
}
def run(){
def city = "北京"
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\北京地址重复数据.xlsx", { d ->
index++
println index +" "+ d['D']+" "+d['E']
if(!d['G']){
// def gdData = FangjiaClientUtils.getMapData(city, d['D'],d['E'], d['K'],"", "高德")
// if(gdData){
// d['F'] = gdData.district
// d['G'] = gdData.name
// d['H'] = gdData.address
// d['I'] = gdData.type
// d['J'] = "高德"
// }else{
def rets = OpenApiUtils.getGaoDeSearchLocation(city,d['D'],d['E'])
def map = findStandardObject(city,d['D'],d['E'], rets);
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(city,d['D'],d['E']);
map = findStandardObject(city,d['D'],d['E'], rets);
}
if(map){
if(map.adname){
d['F'] = map.adname
}else if(map.region){
d['F'] = map.region
}
d['G'] = map.name
d['H'] = map.address
if(map.type){
d['I'] = map.type
}else if(map.estateType){
d['I'] = map.estateType
}
d['J'] = map.source
}
// }
}
},0)
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
continue;
}
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
}
@@ -30,15 +30,63 @@ class MatchMapData {
static main(args) {
def task = new MatchMapData()
task.run()
// task.run()
// task.run1()
// task.run2()
// task.run3()
// task.exportHouse()
// task.addDistrictLevel()
task.run4()
}
def run4() {
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
def regionCode = null
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
regionCode = getRegionCode(newRegion)
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
d['F'] = "完全一致"
d['E'] = dist.name
d['D'] = dist.name_cat
d['G'] = dist.estate_type2
} else {
dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district")
if (dist) {
d['F'] = "映射别名"
d['E'] = dist.name
d['D'] = dist.name_cat
d['G'] = dist.estate_type2
} else {
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
if (rets) {
if(rets.size()>1){
d['F'] = "别名-多个"
}else{
d['F'] = "别名"
}
}
if (rets) {
dist = rets[0]
}
if (dist) {
d['F'] = "完全一致"
d['E'] = dist.name
d['D'] = dist.name_cat
d['G'] = dist.estate_type2
}
}
}
}
}
}, 0)
}
def run3(){
def files = ["ºÏ·ÊÈÚºÏÆ¥Åä_1215.xlsx"]
@@ -0,0 +1,198 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
class TradeDistrictToDB {
static main(args) {
def task = new TradeDistrictToDB();
// task.run()
task.run1()
}
def run1(){
def index = 0
def keys = []
ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
if(d['D']){
def key = d['A']+"##"+d['D']+"##"+d['C']
if(!keys.contains(key)){
def formatAddress = null
if(d['G']){
formatAddress = AddressUtil.formatAddress(d['A'], d['F'])
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['G']){
glat = d['H'].toString().toDouble() * 10000000000L
glng = d['G'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def name = d['C'].replace("·","").replace("II","二")
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}
keys << key
if(name){
def dist = CommonDatautils.findDistrict(d['A'], d['D'], name, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", d['A'])
data.put("region",d['D'])
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['D'], d['C'], 4);
println index+" "+data
}
}
}
}
},0)
}
def run() {
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d ->
def city = d['A']
def region = d['B']
def newRegion = RegionData.formatRegion(city, region)
if (newRegion) {
def map = findMapData(city, newRegion, d['C'])
if(map){
d['D'] = newRegion
d['E'] = map.name
d['F'] = map.address
d['G'] = map.lng
d['H'] = map.lat
d['I'] = map.lng2
d['J'] = map.lag2
}
}
}, 0);
}
def findMapData(def city, def region, name){
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
def map = findStandardObject2(city, region, name, null, rets);
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(city, region, name);
map = findStandardObject2(city, region, name, null, rets);
}
return map;
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -55,6 +55,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// addBigdataDistrictFJW()
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
def index = 0
for (def city : citys) {
index++
@@ -101,7 +103,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
println index +" "+city +" 5.小区费用表"
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 6.小区房屋用途表"
addBigdataPositionGJ(districts, cityCodeDataMap.get(city))
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
println index +" "+ city + " 7.地理位置表"
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
@@ -552,18 +554,17 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// }
}
def addBigdataPositionGJ(districts, def cityid){
def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){
def rets = []
def index = 0
for(def d : districts) {
index++
def ret = []
def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap);
ret << d.cityid
ret << "5"
ret << d.commid
ret << "1"
ret << index
ret << seqno
ret << d.lng
ret << d.lat
ret << ""
@@ -589,49 +590,29 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_POSITION_FJW 数据文件输出完成......"
}
// sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'")
//
// def sqls = []
// def index = 1
// for(def d : districts){
// def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark},"
// index++
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
// }else{
// sql = sql + "null,"
// }
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
// sqls << sql
//
// if (sqls.size() >= batchsize) {
// try {
// sqlInstance.withBatch(batchsize, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
//
// } catch (Exception e) {
// e.printStackTrace()
// }
// sqls.clear()
// }
//
// }
//
// if (sqls.size() > 0) {
// sqlInstance.withBatch(100, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
// }
def findSeqno(cityid, commid, citySeqnoMap){
def detailMap = citySeqnoMap.get(cityid)
if(detailMap){
def seqno = detailMap.get(commid)
if(seqno){
return seqno
}else{
seqno = CcbCommonUtils.findMaxSeqno(cityid)
if(seqno){
seqno = seqno + 1
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
def insertSql = "INSERT INTO bigdata_position_seqno(CITYID,COMMID,SEQNO,CREATETIME) VALUES ('${cityid}','${commid}',${seqno},to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'))"
sqlInstance.withBatch(batchsize, { stmt ->
stmt.addBatch insertSql
})
return seqno
}
}
}else{
println cityid+" ûÓиóÇÊеÄÐ¡ÇøÐòºÅ"
}
}
def addBigdataDistrictFJW(){
@@ -26,7 +26,7 @@ class ExportDistrictRongHe extends BaseBatch{
void process(Object args) {
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"))
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()
@@ -471,7 +471,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def allRows = []
if(curDate.toString().endsWith("01")){
if(curDate.toString().endsWith("28")){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
sqlInstance.eachRow(sql,{r->
def data = []
@@ -0,0 +1,747 @@
package com.fangjia.script.ccbevaluation.ronghe
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.sql.ResultSet
import java.sql.ResultSetMetaData
@BatchOwner(Watcher.GaoJuan)
class ExportDistrictRongHeTest extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new ExportDistrictRongHeTest(), args);
}
@Override
void process(Object args) {
def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}"
// if(CommonDatautils.isWindow()){
// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}"
// }
CcbFtpTestUtils.delete(fileName)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
exportBigdataCommunityGJRH()
exportBigdataBuildingFJW()
exportBigdataCommfeeGj()
exportBigdataCommbuildstruGJ()
exportBigdataCommhousetypeGJ()
exportBigdataCommbuildyearGj()
exportBigdataCommhouseuseGJ()
exportBigdataPositionGJ()
exportBigdataDistrictFJW()
exportBigdataAliasFJW()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportEvaluateRate()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
exportOk(curDate)
}
//
// def runByCity(){
// def citys = CcbCommonUtils.citys;
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//
//
//// //检查ok文件是否上传
//// exportAllBigdataDealFJW()
//// for(def city : citys) {
//// def cityCode = cityCodeDataMap.get(city)
////
// //按天导(一个文件夹下)
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
// //按天导(每天一个文件夹)
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
//
//// exportBigdataCommunityGJRH(cityCode)
//// exportBigdataBuildingFJW(cityCode)
//// exportBigdataCommfeeGj(cityCode)
////
//// exportBigdataCommbuildstruGJ(cityCode)
//// exportBigdataCommhousetypeGJ(cityCode)
//// exportBigdataCommbuildyearGj(cityCode)
//// exportBigdataCommhouseuseGJ(cityCode)
//// exportBigdataPositionGJ(cityCode)
//// exportBigdataDistrictFJW(cityCode)
//// exportBigdataAliasFJW(cityCode)
////
//// exportBigdataBuildprojFjw(cityCode)
//// exportBigdataCommlayoutpicFjw(cityCode)
//// }
//// exportEvaluateRate()
//
////
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
//
// exportOk()
// }
def exportBigdataCommunityGJRH(cityCode){
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMMNAME
data << r.COMMALIAS
data << r.DISTID
data << r.STREET
data << r.STREETNO
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.COMMTYPE
data << r.IFNEW
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << ''
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDSTRU
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
}
def exportBigdataCommhousetypeGJ(cityCode){
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildyearGj(){
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDYEAR
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
}
def exportBigdataCommfeeGj(){
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMPID
data << r.FEE
data << r.HOUSEUSE
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.FEETYPE
data << r.NOTE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
}
def exportBigdataCommhouseuseGJ(){
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSEUSE
data << r.HOUSECNT
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
}
def exportBigdataPositionGJ(){
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.OBJECTTYPE
data << r.OBJECTID
data << r.POSITIONTYPE
data << r.SEQNO
data << r.LONGITUDE
data << r.LATITUDE
data << r.ALTITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
}
def exportBigdataDistrictFJW(){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
}
def exportBigdataAliasFJW(){
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.city_id
data << r.ca_id
data << r.address_id
data << r.address_name
data << r.status
data << r.exp_dt
data << r.version
data << r.is_key_name
data << r.data_source
data << r.description
data << r.alias_grade
data << r.addr_type_cd
data << r.ifexamine
data << r.ifdisplay
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
}
def exportBigdataDealFJW(dealdate){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def stop = false
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def index = 0
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
while(!stop){
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
def allRows = []
if(index==1){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
}
if(curStartDateStr == curSndDateStr){
stop = true
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
def curEndStr = DateUtils.add(curStartStr, 1)
curStartDateStr = curEndStr
}
}
def exportBigdataDealFJW2(curDate, dealdate){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......"
}
def exportBigdataBuildingFJW(cityCode){
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.BUILDALIAS
data << r.DISTID
data << r.COMMID
data << r.BUILDYEAR
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.FLOORCNT
data << r.UNITCNT
data << r.HOUSECNT
data << r.IFELEVATOR
data << r.ELEVATORCNT
data << r.ELEVATORHOUSE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.VALID_FLAG
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
def exportEvaluateRate(){
def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.RATE_TYPE
data << r.RATE
data << r.RANK
data << r.CITY
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "EVALUATE_RATE_FJW 上传Ftp完成......"
}
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
}
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
}
}
@@ -0,0 +1,277 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
/**
* 重庆估值测试--2小时
*/
class EvaluationBatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new EvaluationBatch()
// task.run()
// task.run1();
task.run2()
}
def run2(){
def startDate = DateUtils.parseFromTightYYYYMMDD("2021-01-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-07")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V1.xlsx", { row ->
if(row['N'] && row['N'].toString().toDouble()>=20){
println row['F'] +" "+row['N']
def dist = CommonDatautils.findDistrictById(row['C'])
for(def weekend : weekends){
CommonUtils.recalculationAverage(row['D'], row['E'], row['F'], dist.estate_type2, weekend,false,0)
}
// FangjiaClientUtils.sendDistrictPriceCalculate(row['D'], row['E'], row['F'], 48);
}
},0)
}
def run1(){
def city = "重庆"
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V2.xlsx", { row ->
def region = row['E']
def name = row['F']
def id = row['C']
def lastWeekend = DateUtils.getFirstDayOfWeek(new Date()-30)
def lastYearWeekend = DateUtils.getFirstDayOfWeek(new Date()-365)
if (region && name) {
row['H'] = CommonDatautils.findNewSqmetreRent(city, region, name)
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
if(priceObj){
row['AB'] = priceObj.avg_price
}
def dist = CommonDatautils.findDistrictById(id)
if(dist) {
List<DBObject> list = DbUtils.findAroundDistrict(dist)
def map = getArroundAvgPriceMap(list)
if(map){
row['L'] = map.avg_price
row['Q'] = map.max_val
row['R'] = map.min_val
}
def lastArroundPrice = getArroundAvgPriceByWeekend(list, lastWeekend)
if(lastArroundPrice && priceObj){
def mod = (map.avg_price-lastArroundPrice)*100/lastArroundPrice
mod = DataHandleUtils.getDoubleRound(2, mod)
row['P'] = mod
}
def lastYearArroundPrice = getArroundAvgPriceByWeekend(list, lastYearWeekend)
if(lastYearArroundPrice && priceObj){
def mod = (map.avg_price-lastYearArroundPrice)*100/lastYearArroundPrice
mod = DataHandleUtils.getDoubleRound(2, mod)
row['O'] = mod
}
}
def lastWeekendObj = CommonDatautils.queryDistrictPrice(city, region, name, lastWeekend)
if(lastWeekendObj){
row['S'] = lastWeekendObj.avg_price
if(priceObj){
def mod = (priceObj.avg_price-lastWeekendObj.avg_price)*100/lastWeekendObj.avg_price
mod = DataHandleUtils.getDoubleRound(2, mod)
row['M'] = mod
}
}
def lastYearObj = CommonDatautils.queryDistrictPrice(city, region, name, lastYearWeekend)
if(lastYearObj){
row['T'] = lastYearObj.avg_price
if(priceObj){
def mod = (priceObj.avg_price-lastYearObj.avg_price)*100/lastYearObj.avg_price
mod = DataHandleUtils.getDoubleRound(2, mod)
row['N'] = mod
}
}
}
},0)
}
def run(){
def city = "重庆"
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆估值交易信息V2.xlsx", { row ->
def region = row['E']
def name = row['F']
def id = row['C']
def area = row['G'].toDouble()
if(region && name){
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
if(priceObj){
row['J'] = priceObj.avg_price
if(row['C']){
row['I'] = (area*priceObj.avg_price).toInteger()
}
}
def dist = CommonDatautils.findDistrictById(id)
if(dist){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
row['N'] = getArroundAvgPrice(list)
def alias = dist.alias
if(!alias){
alias = []
}
alias << dist.name
def houseAvgPrice = findSimAreaHousePrice(dist.city, dist.region, alias,20220101,20220401, area-10, area+10)
if(houseAvgPrice){
row['K'] = houseAvgPrice
}
def tradeAvgPrice = findTrades(dist, DateUtils.parseFromTightYYYYMMDD("2021-12-01"), area-10, area+10)
row['L'] = tradeAvgPrice
def avg_price = findAvgPrice(dist._id.toString())
if(row['J'] && avg_price){
def weekendPrice = row['J'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['P'] = ppe15
}
}
}
});
}
def findTrades(dist, startDate, startArea, endArea){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
// q.put("source", "链家在线")
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate))
q.put("area", new BasicDBObject('$gte', startArea).append('$lt',endArea))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def prices = []
for(def d : dbcorsor){
if(d.avg_price){
prices << d.avg_price
}
}
def price = null
if(prices){
price = (prices.sum()/prices.size()).toInteger()
}
return price
}
def getArroundAvgPriceMap(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
prices.sort()
def map = [:]
map.min_val = prices[0]
map.max_val = prices[prices.size()-1]
def avg_price = (prices.sum()/prices.size()).toInteger()
map.avg_price = avg_price
return map;
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
def getArroundAvgPriceByWeekend(def list, def weekend){
def prices = []
for(def l : list){
def priceObj =CommonDatautils.queryDistrictPrice(l.city, l.region, l.name, weekend)
if(priceObj){
prices << priceObj.avg_price
}
}
def avg_price = null;
if(prices){
avg_price = (prices.sum()/prices.size()).toInteger()
}
return avg_price;
}
def findSimAreaHousePrice(def city, def region, def alias, def startDate, def endDate, def startArea, def endArea){
def aliasStr = null;
def total = 0
def prices = []
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and area>=${startArea} and area<=${endArea}"
ElasticUtils.executeSqlQuery(sql,{ data->
prices << data.avg_price
})
}catch(Exception e){
e.printStackTrace()
}
def price = null
if(prices){
price = (prices.sum()/prices.size()).toInteger()
}
return price
}
public String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -0,0 +1,84 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class BlankMatch {
static main(args) {
def task = new BlankMatch()
task.run()
// task.run1()
// task.run2()
// task.run3()
}
def run() {
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
//def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx";
def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['C']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['C'] = jsonObject.get("id")
row['D'] = jsonObject.get("city")
row['E'] = jsonObject.get("district")
row['F'] = jsonObject.get("name")
row['G'] = jsonObject.get("address")
}
}
});
}
def run1() {
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx", { row ->
if(row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['F'], "property", false, row['E'], null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['C'] = jsonObject.get("id")
row['D'] = jsonObject.get("city")
row['J'] = jsonObject.get("name")
}
}
// def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
// if(jsonObject != null && jsonObject.precise!=-1) {
// row['C'] = jsonObject.get("id")
// row['D'] = jsonObject.get("city")
// row['E'] = jsonObject.get("district")
// row['F'] = jsonObject.get("name")
// row['G'] = jsonObject.get("address")
// }
});
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
if(dist){
row['C'] = dist._id.toString()
}
}
},0)
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
for(def d : datas){
if(d['B'] == row['B']){
row['C'] = d['C']
row['D'] = d['D']
row['E'] = d['E']
row['F'] = d['J']
break;
}
}
}
},0)
}
}
@@ -0,0 +1,77 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import org.apache.lucene.queryparser.classic.QueryParser
import org.bson.types.ObjectId
class FieldAddValue {
static main(args) {
def task = new FieldAddValue()
task.run()
}
def run(){
//16991 14360
def districts = CommonDatautils.findAllDistrictByCity("ÖØÇì", "district", ["region","name","alias","is_elevator"])
def index = 0
def updateIndex = 0
for(def d : districts){
if(!d.is_elevator){
index++
println index
def alias = d.alias
if(!d.alias){
alias = []
}
alias << d.name
def maxHeight = findHouseHeight("ÖØÇì", d.region, alias, 20220101)
if(maxHeight && maxHeight>6){
updateIndex++
def udata = new BasicDBObject()
udata.put("is_elevator", true)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId( d._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
println index+" "+updateIndex
}
def findHouseHeight(def city, def region, def alias, def startDate){
def aliasStr = null;
def total = 0
def heights = null
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select max(height) as height from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name in ('${aliasStr}') "
ElasticUtils.executeSqlQuery(sql,{ data->
if(data.height){
heights = data.height
}
})
}catch(Exception e){
e.printStackTrace()
}
// def max_height = null
// if(heights){
// heights.sort()
// max_height = heights[heights.size()-1]
// }
return heights
}
public String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
}
@@ -0,0 +1,239 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
class MatchEvaluateRegionTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new MatchEvaluateRegionTest()
task.run()//小区匹配
task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
task.addAuthor()
}
def run(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("name")
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def run1(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("name")
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def addPrice(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['L']){
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
if(priceObj){
row['M'] = priceObj.avg_price
row['N']= (int)(priceObj.avg_price * row['E'])
}
def avg_price = findAvgPrice(row['M'])
if(row['M'] && avg_price){
def weekendPrice = row['M'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def addAuthor(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def total = 747
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(index < avg){
row['P'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
});
}
def addDistrict(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def flag = d['M']
if(!flag){
def city = d['E']
def region = d['F']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['G']
name = nameFileter(name)
def formatAddress = null
def address = d['H']
if(address){
formatAddress = AddressUtil.formatAddress(city, address)
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
addPrice(data)
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
}
},0)
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name.contains("(")){
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
name = name.replace(delstr,"")
}
return name;
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -0,0 +1,280 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import com.fangjia.script.gj.district.FangjiaClientUtils
class MatchEvaluateTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx"
def total = 1500
static main(args) {
def task = new MatchEvaluateTest()
// task.run()//小区匹配
// task.run1()//地图匹配
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
// task.addPrice()
// task.addArroundPrice()
// task.addAuthor()
task.stat()
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A'] && !row['K']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['K'] = priceObj.unitPrice
row['L'] = (int) (priceObj.unitPrice * row['C'])
row['O'] = "周边价格"
}
}
});
}
def run(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("address")
row['I'] = jsonObject.get("propertyType")
row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def run1(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("address")
row['I'] = jsonObject.get("propertyType")
row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def addPrice(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['J'] && row['E']){
def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G'])
if(priceObj){
row['K'] = priceObj.avg_price
row['L']= (int)(priceObj.avg_price * row['C'])
}
def avg_price = findAvgPrice(row['D'])
if(row['K'] && avg_price){
def weekendPrice = row['K'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def addAuthor(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['N'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def addDistrict(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def flag = d['M']
if(!flag){
def city = d['E']
def region = d['F']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['G']
name = nameFileter(name)
def formatAddress = null
def address = d['H']
if(address){
formatAddress = AddressUtil.formatAddress(city, address)
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
addPrice(data)
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
}
},0)
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name.contains("(")){
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
name = name.replace(delstr,"")
}
return name;
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
def stat(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1)
ExcelUtils.multiThreadingModify(path, { d ->
if(d['B']){
for(def row : datas){
if(d['B']==row['B']){
if(row['K']){
d['P'] = row['K']
break
}
}
}
}
});
}
}
@@ -218,7 +218,6 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","˝đťÔ˛ŽžôÉ˝")
DBObject keys = new BasicDBObject();
if(fields){
@@ -829,6 +828,9 @@ class CommonDatautils {
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
keys.put("sid", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -916,6 +918,23 @@ class CommonDatautils {
}
return ret;
}
public static def findDistrictsByCityAndName(String city, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("cat", cat);
q.put("name", name);
q.put("visible", 0);
def list = []
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
list << obj;
}
}
return list;
}
public static def findDistrict(String city, String name, String cat){
@@ -3134,5 +3153,36 @@ class CommonDatautils {
}
return dbo;
}
static def findNewSqmetreRent(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
def dbcursor = SQMETRE_RENT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1)
def avg_price = null
for(def d : dbcursor){
avg_price = d.avg_price;
}
return avg_price;
}
static def findCityAvgRentRadio(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = SQMETRE_RENT_COLL.find(q)
def avg_price = null
def radios = []
for(def d : dbcursor){
if(d.price_to_rent){
radios << d.price_to_rent.toInteger()
}
}
def avg_radios = (int)(radios.sum()/radios.size())
return avg_radios;
}
}
@@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils;
class DataHandleUtils {
static main(args){
}
static def getStandardRegion(def regionMap, def region){
def newRegion = null;
@@ -94,6 +92,8 @@ class DataHandleUtils {
BigDecimal deSource = new BigDecimal(dSource);
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
}
static def trimChar(String c) {
try {
@@ -20,11 +20,11 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
// task.regionPriceTrendRun()
task.regionPriceTrendRun()
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:]
@@ -53,7 +53,7 @@ class FzDataResultStat {
}
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
}
//ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
@@ -94,7 +94,7 @@ class FzDataResultStat {
}
def prices = exportRegion(city, months)
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据")
ExcelUtils.write("D:\\数据\\福州\\福州6区均价半年走势_${curMonth}.xlsx", prices, "数据")
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -143,8 +143,8 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
}
if(!retMap.anjuke_esf_price && d.anjuke_esf_price){
retMap.anjuke_esf_price = d.anjuke_esf_price
if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){
retMap.anjuke_esf_price = d.get("58tc_esf_price")
}
if(!retMap.soufang_esf_price && d.soufang_esf_price){
retMap.soufang_esf_price = d.soufang_esf_price
@@ -152,6 +152,9 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.lianjia_esf_price && d.lianjia_esf_price){
retMap.lianjia_esf_price = d.lianjia_esf_price
}
if(!retMap.lianjia_esf_price && d.beike_esf_price){
retMap.lianjia_esf_price = d.beike_esf_price
}
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
retMap.cityhouse_esf_price = d.cityhouse_esf_price
@@ -0,0 +1,278 @@
package com.fangjia.script.fuzhou
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import java.util.concurrent.CountDownLatch
class ThirdTest {
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new ThirdTest()
// task.run()
// task.run1()
task.run3()
// task.run4()
}
def run4(){
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
if (!d['C']) {
// def name = d['A'].replace("(钢混)","").replace("(非钢混)","").replace("(混合)","")
// def lists = CommonDatautils.findDistrictsByCityAndName("福州",name, "district")
// if(lists){
// for(def o : lists){
//// if(d['B'] && d['B'].toString().contains(o.address)){
// d['C'] = o.region
// d['D'] = o.name
// d['E'] = "名称去括号中的字"
//// }
// }
// }
def rMap = SeaweedDataMatchProcess.doMatch("福州", null, d['A'], d['B']);
if(rMap){
d['C'] = rMap.region
d['D'] = rMap.name
d['E'] = "匹配接口"
}
}
},0)
}
def run3() {
// ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
// if(!d['F']){
// def price = CommonDatautils.queryDistrictPrice("福州",d['C'], d['D'])
// d['F'] = price
// }
// },0)
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202202.csv"))
// def map1 = [:]
// for(def s : list1){
// def row = s.split("@\\|@")
// def key = row[2]+"##"+row[3]+"##"+row[4]
// map1.put(key, row[5])
// }
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv"))
def map1 = [:]
for(def str : list1){
def s = str.split("@\\|@")
def key = s[2]+"##"+s[3]+"##"+s[4]
def priceMap = [:]
map1.put(key, priceMap)
if(s.size()>=7 && s[6]){
priceMap.put("zhugezhaofang", s[6])
}
if(s.size()>=8 && s[7]){
priceMap.put("anjuke", s[7])
}
if(s.size()>=9 && s[8]){
priceMap.put("soufang", s[8])
}
if(s.size()>=10 && s[9]){
priceMap.put("lianjia", s[9])
}
if(s.size()>=11 && s[10]){
priceMap.put("cityhouse", s[10])
}
if(s.size()>=12 && s[11]){
priceMap.put("fangtan", s[11])
}
if(s.size()>=13 && s[12]){
priceMap.put("fangdoudou", s[12])
}
if(s.size()>=14 && s[13]){
priceMap.put("maitian", s[13])
}
try{
if(s.size()>=15 && s[14]){
priceMap.put("58tc", s[14])
}
}catch(Exception e){
e.getMessage()
}
}
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d ->
// if (!d['F']) {
// def lists = CommonDatautils.findDistrictsByCityAndName("福州", d['A'], "district")
// if(lists){
// for(def o : lists){
// if(o.address==d['B']){
// d['C'] = o.region
// }
// }
// }
// }
def key = "福州##"+d['C']+"##"+d['D']
def priceMap = map1.get(key)
if(priceMap){
if(priceMap.zhugezhaofang){
d['F'] = priceMap.zhugezhaofang
}
if(priceMap.anjuke){
d['G'] = priceMap.anjuke
}
if(priceMap.soufang){
d['H'] = priceMap.soufang
}
if(priceMap.lianjia){
d['I'] = priceMap.lianjia
}
if(priceMap.cityhouse){
d['J'] = priceMap.cityhouse
}
if(priceMap.fangtan){
d['K'] = priceMap.fangtan
}
if(priceMap.fangdoudou){
d['L'] = priceMap.fangdoudou
}
if(priceMap.maitian){
d['M'] = priceMap.maitian
}
if(priceMap.get("58tc")){
d['N'] = priceMap.get("58tc")
}
}
}, 0)
}
def run1(){
ExcelUtils.modify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供-结果.xlsx",{d->
if(d['B'] && !d['D']){
if(d['B'].contains("号")){
// def price = searchContent2("福州","https://fz.58.com", d['B'], 304)
// if(price){
// d['C'] = price
// d['F'] = "58同城"
// }
def price = thirdPartyColl("福州", d['B'])
if(price){
d['D'] = price;
d['F'] = "麦田"
}
}
}
},0)
}
def thirdPartyColl(def city, def address){
def q = new BasicDBObject()
q.put("city", city)
q.put("address", address)
q.put("source","麦田")
q.put("s_date",202202)
def obj = thirdPartyColl.findOne(q)
if(obj){
return obj.avg_price
}
return null;
}
//查看此关键字是否有数据
def searchContent2(def city,def url,def keyword, def cityid){
def newUrl = url+"/esf-ajax/community/pc/autocomplete/?city_id=${cityid}&kw="+URLEncoder.encode(keyword, "utf-8")+"&type=2"
def document = CrawalUtil.getDocumentByABuYun(newUrl, "访问过于频繁", "utf-8", null, true);
def price = null
if(document){
println "【"+keyword +"】 查询"
def jsonObj = JSONObject.fromObject(document)
if(jsonObj && jsonObj.status=="ok"){
if(jsonObj.data.size()>0){
def datas = jsonObj.data
def d = datas[0]
price = d.price.toInteger()
}
}else{
println "【"+keyword +"】 无结果"
}
}
return price;
}
def run(){
def map = findThirdPrice("福州")
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供.xlsx",{d->
if(!d['F']){
def lists = CommonDatautils.findDistrictsByCityAndName("福州",d['A'], "district")
if(!lists){
lists = CommonDatautils.findDistrictByAlias("福州",null, d['A'], "district")
}
for(def o : lists){
// if(o.address == d['B']){
def priceMap = map.get(o.region+"##"+o.name)
d['C'] = priceMap.anjuke_esf_price
d['D'] = priceMap.maitian_esf_price
d['E'] = priceMap.beike_esf_price
d['F'] = o.region
// }
}
}
},0)
}
def findThirdPrice(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', 202109))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor) {
def key = db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.get("lianjia_esf_price") && db.lianjia_esf_price>3000){
dataMap.put("beike_esf_price", db.lianjia_esf_price)
}
}
return map;
}
}
@@ -45,7 +45,7 @@ class EsHouseStat {
public static void main(String[] args){
def task = new EsHouseStat();
//task.run();
task.run();
// task.testRun("ÉîÛÚ")
// task.exportOpenLog();
// task.exportMongoLog()
@@ -59,7 +59,7 @@ class EsHouseStat {
// task.szRentHouseStat();
// task.addPrice();
task.statNoHouseCCBData();
// task.statNoHouseCCBData();
}
def addPrice(){
@@ -337,21 +337,6 @@ class EsHouseStat {
return total;
}
def findHouse(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def total = 0;
while (rs.next()){
total = rs.getInt("count")
}
return total;
}
def findHouseSource(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT * FROM \"seawater-20*\" WHERE 1=1")
@@ -373,6 +358,21 @@ class EsHouseStat {
return statMap;
}
def findHouse(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def total = 0;
while (rs.next()){
total = rs.getInt("count")
}
return total;
}
def testRun(def city){
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
address = address.replace("(","(").replace(")",")").replace(" ","")
address = address.replace("(","(").replace(")",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
@@ -149,6 +149,7 @@ class OpenApiUtils {
ret.lng2 = gmap.get("lng")
ret.lat2 = gmap.get("lat")
ret.estateType = ret.type
ret.source = "高德"
}catch (Exception e){
}
@@ -250,14 +251,14 @@ class OpenApiUtils {
}
if(type && type.contains("出入口")){
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
}else{
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
poiList.add(jo);
}
}else{
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
}
}
}else{
@@ -302,7 +303,7 @@ class OpenApiUtils {
def distname = formatName(ret.get("name"))
address = ret.get("address")
if((distname && distname.contains(name)) || (distname && name.contains(distname)) || (address && address.contains(name)) || ( address && name.contains(address))){
if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
// if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
lng2 = location.lng
lat2 = location.lat
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
@@ -311,7 +312,7 @@ class OpenApiUtils {
def mapRegion = ret.get("area")
distRegion = mapRegion
districtName = distname
if(address && address.contains(mapRegion)){
if(address && mapRegion && address.contains(mapRegion)){
address = address.substring(address.indexOf(mapRegion)+mapRegion.size());
}
if(ret.get("detail_info")){
@@ -330,6 +331,7 @@ class OpenApiUtils {
map.name = districtName
map.region = distRegion
map.estateType = estateType
map.source = "百度"
rets << map
lat = null
@@ -340,12 +342,18 @@ class OpenApiUtils {
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
}
}else{
println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
}
// }else{
// println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
// }
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
def score1 = 0
if(distname && name){
score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
}
def score2 = 0
if(address && name){
score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
}
if(score1 || score2){
rets << map
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】(score:"+score1+"#"+score2+")"
@@ -31,7 +31,7 @@ class RegionPointCrawl {
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
@@ -6,6 +6,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.qqmap.TengXunApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
@@ -42,4 +43,29 @@ println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk"
// }
//
//}
//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据")
//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据")
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d->
if(d['B']){
def json = GaoDeApiUtils.geo("重庆", d['B'])
if(json && json.info=="OK"){
def datas = json.geocodes
if(datas){
def row = datas[0]
println row
d['C'] = row.country
d['D'] = row.province
d['E'] = row.city.toString().replace("[]","")
d['F'] = row.citycode.toString().replace("[]","")
d['G'] = row.district.toString().replace("[]","")
d['H'] = row.street.toString().replace("[]","")
d['I'] = row.number.toString().replace("[]","")
d['J'] = row.adcode.toString().replace("[]","")
d['K'] = row.location
d['L'] = row.level
}
}
}
})
@@ -7,11 +7,11 @@ email.setSSL(true);
email.setHostName("smtp.exmail.qq.com");
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码
email.addTo("gaojuan@fangjia.com"); //接收方
email.setFrom("service@fangjia.com"); //发送方
email.setSubject("test"); //标题
email.setContent("你?", "text/html");
email.setContent("测试", "text/html");
//email.setHostName("smtp.exmail.qq.com");
//email.setSSL(true);
@@ -7,13 +7,17 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.script.util.DbUtils
/**
* 模糊地址经纬度更新
@@ -44,13 +48,16 @@ class VagueAddressLocationBatch extends BaseBatch {
void process(Object args) {
//"北京","常州",
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
def citys =["上海"] //CommonDatautils.findAllCity();
def citys =["重庆"] //CommonDatautils.findAllCity();
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
//市谊大厦 华三新村 阳光家园
for (def city : citys) {
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat2", "lat", "lng2", "lng"]);
def rets = findVagueAddress(city, districts);
findAndUpdateLocation(rets);
writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
// writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
writeCSV(results, "d:\\数据\\结果\\" + city + "地址模糊.csv")
// ExcelUtils.write("d:\\数据\\结果\\" + city + "地址模糊.xlsx", rets, "数据")
}
@@ -58,6 +65,7 @@ class VagueAddressLocationBatch extends BaseBatch {
def findVagueAddress(def city, def districts) {
def rets = []
def rows = []
for (def district : districts) {
def address = district.address;
if (address) {
@@ -66,10 +74,13 @@ class VagueAddressLocationBatch extends BaseBatch {
//没地址
if (!address) {
rets << district;
// rows << [district.city, district.region, district.name, district.address]
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
} else {
if (!findPattern(address)) {
rets << district;
// rows << [district.city, district.region, district.name, district.address]
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
} else {
println district.city + " " + district.address + " 精确地址"
@@ -97,7 +108,7 @@ class VagueAddressLocationBatch extends BaseBatch {
} else if (address.indexOf("路") == -1 && address.indexOf("街") == -1 && address.indexOf("大道") == -1 && address.indexOf("巷") == -1 && address.indexOf("北里") == -1 && address.indexOf("南里") == -1 && address.indexOf("东里") == -1 && address.indexOf("西里") == -1) {
return null;
} else {
pattern = Pattern.compile("[路|街|大道|巷]\\d+");
pattern = Pattern.compile("[路甲|路|街|大道|巷|弄|里]\\d+");
matcher = pattern.matcher(address)
if(matcher.find()){
return s;
@@ -123,16 +134,14 @@ class VagueAddressLocationBatch extends BaseBatch {
for (def ret : rets) {
index++;
if(ret.name!="药师新村"){
continue
}
def locationMap = findLatAndLng(ret);
if (locationMap) {
updateLocation(locationMap, ret);
} else {
println "[没找到] " + ret.city + " " + ret.region + " " + ret.name + " " + ret.address
}
println "【${index}/${rets.size()}】************************************************************************************"
println "【${index}/${rets.size()}】**************【${ret.region}---${ret.name}----${ret.address}】**********************************************************************"
}
}
@@ -148,14 +157,14 @@ class VagueAddressLocationBatch extends BaseBatch {
msgMap.db_lng2 = ret.lng2
BasicDBObject udata = new BasicDBObject()
if (ret.lat2 != locationMap.lat2) {
udata.put('lat2', locationMap.lat2)
udata.put('lng2', locationMap.lng2)
if (ret.lat2 && ret.lat2 != locationMap.lat2) {
msgMap.m_lat2 = locationMap.lat2
msgMap.m_lng2 = locationMap.lng2
def distance = GeoHelper.GetDistance(ret.lat2, ret.lng2, locationMap.lat2, locationMap.lng2)
msgMap.updateLngFlag = true
msgMap.distance = distance
udata.put('lat2', locationMap.lat2)
udata.put('lng2', locationMap.lng2)
}
if (ret.lat != locationMap.lat) {
@@ -171,31 +180,39 @@ class VagueAddressLocationBatch extends BaseBatch {
if (udata) {
println ret.city + " " + ret.region + " " + ret.name + " [ 修改其经纬度!]"
udata.put("m_date", new Date())
// DBUtils.seaweed.update(new BasicDBObject('_id',locationMap._id),new BasicDBObject('$set',udata))
DbUtils.seaweed.update(new BasicDBObject('_id',locationMap.id),new BasicDBObject('$set',udata))
}
results << msgMap
}
def findStandardObject(def city, def name, def rets) {
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() > 1) {
//相似里找有精确地址
for (def mapObj : rets) {
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
if (mapObj.address && findPattern(mapObj.address)) {
map = mapObj;
break;
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name.contains(name) || name.contains(mapObj.name)){
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
if (mapObj.address && findPattern(mapObj.address)) {
map = mapObj;
break;
}
}
}
}
if(!map){
//交带有交叉口地址
for (def mapObj : rets) {
if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) {
if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){
map = mapObj;
break;
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name.contains(name) || name.contains(mapObj.name)) {
if (mapObj.address && (mapObj.address.contains("交汇") || mapObj.address.contains("交叉"))) {
if (!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")) {
map = mapObj;
break;
}
}
}
}
}
@@ -211,18 +228,18 @@ class VagueAddressLocationBatch extends BaseBatch {
def findLatAndLng(ret) {
//https://restapi.amap.com/v3/place/text?keywords=
def rets = OpenApiUtils.getGaoDeSearchLocation(ret.city, ret.region, ret.name)
def map = findStandardObject(ret.city, ret.name, rets);
def map = findStandardObject(ret.city, ret.region, ret.name, rets);
def locationMap = [:]
if (!map || !map.lat) {
//http://api.map.baidu.com/place/v2/search
rets = OpenApiUtils.getBiDuLocation(ret.city, ret.region, ret.name);
map = findStandardObject(ret.city, ret.name, rets);
map = findStandardObject(ret.city, ret.region, ret.name, rets);
}
if (!map || !map.lat) {
//https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap
def code = codeMap.get(ret.city);
rets = OpenApiUtils.searchBaiDuMap(ret.name, ret.region, code.toInteger());
map = findStandardObject(ret.city, ret.name, rets);
map = findStandardObject(ret.city, ret.region, ret.name, rets);
}
// if((!map || !map.lat)){
@@ -237,7 +254,7 @@ class VagueAddressLocationBatch extends BaseBatch {
locationMap.lng2 = (map.lng2.toDouble() * 10000000000).longValue()
locationMap.id = ret._id
locationMap.city = ret.city
if (ret.address.toString().endsWith("路")) {
if (map.address) {
locationMap.address = map.address.replace("(","").replace(")","");
}
@@ -37,22 +37,22 @@ class YouDaDataParser {
public static final DBCollection youda_newhouse_coll = MongoUtils.getCollectionByName2("business", "res_newdealprice", offlineUrl)
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
//线下:228 线上:45
//线下:228 线上:45 测试
static def new_targetUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc"
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc"
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc"
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-12-01"
static def statDate = "2022-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -490,8 +490,9 @@ class YouDaDataParser {
if(s){
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
if(hits && hits.total instanceof Integer){
total = hits.total
}
}
return total
@@ -565,7 +566,7 @@ class YouDaDataParser {
if(obj.premisesName && obj.premisesName!="null"){
dbo.premisesName = obj.premisesName
}
dbo.price = obj.price
dbo.price = obj.price.toInteger()
dbo.region = obj.region
dbo.resId = obj.ydId.toString()
dbo.totalPrice = obj.totalPrice
@@ -0,0 +1,61 @@
package com.fangjia.script.gj.apiLog
import com.alibaba.excel.util.DateUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.Calendar;
@BatchOwner(Watcher.GaoJuan)
class ApiTimeOutAnalyBatch extends BaseBatch{
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
static main(args) {
runBatch(new ApiTimeOutAnalyBatch(), args);
}
@Override
public void process(Object args) {
def uriCountMap = findTimeOutUri()
sendMessage(uriCountMap)
}
def sendMessage(uriCountMap){
def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:"
def total = message.length()
uriCountMap.each{uri,count->
if(count>=5){
message = message + uri +"\n"
}
}
if(message.length()>total){
println message
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
DingDingMessageUtil.sendTextMessage(access_token, message)
}
}
def findTimeOutUri(){
def q = new BasicDBObject()
q.put("time", new BasicDBObject('$gte', 3000))
def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5)
q.put("c_date", new BasicDBObject('$gte', startTime))
def dbcursor = LOG.find(q)
def uriCountMap = [:]
for(def d : dbcursor){
def count = uriCountMap.get(d.request_uri)
if(!count){
count = 0
}
uriCountMap.put(d.request_uri, count + 1)
}
return uriCountMap
}
}
@@ -0,0 +1,111 @@
package com.fangjia.script.gj.apiLog
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
@BatchOwner(Watcher.GaoJuan)
class ApiUserCountAnalyBatch extends BaseBatch{
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
static main(args) {
runBatch(new ApiUserCountAnalyBatch(), args);
}
@Override
public void process(Object args) {
def rets = findAllUsers()
sendMessage(rets)
}
def sendMessage(def rets){
def message = "【Api接口使用提醒】\n"
def total = message.length()
for(def r : rets){
def type = "正式帐户"
if(r.type!=0){
type = "测试帐户"
}
message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n"
}
if(message.length()>total){
println message
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
DingDingMessageUtil.sendTextMessage(access_token, message)
}
}
def findAllUsers(){
def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'"
def rows = dbFangjia.rows(sql)
def rets = []
for(def row : rows){
def obj = findResources(row.id)
if(obj){
def map = obj.total_count_map
def uris = []
map.each{k,v->
//def visit_count = findVisitCount(row.id, k)
def total_visit_count = findAllVisitCount(row.id, k)
println row.id +" "+k+" "+v+" "+total_visit_count
if(v && total_visit_count){
def precent = v*0.8;
if(total_visit_count>=precent){
uris << k
}
}
}
if(uris){
rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test]
}
}
}
return rets;
}
def findAllVisitCount(userId, uri){
def q = new BasicDBObject()
q.put("user_id", userId.toString())
q.put("uri", uri)
q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date())))
def dbcursor = VISIT_COLL.find(q)
def total = 0
for(def obj : dbcursor){
total = total +obj.visit_count
}
return total
}
def findVisitCount(userId, uri){
def q = new BasicDBObject()
q.put("user_id", userId.toString())
q.put("uri", uri)
q.put("visit_date", DateUtils.truncateDay(new Date()))
def obj = VISIT_COLL.findOne(q)
if(obj){
return obj.visit_count
}
return null
}
def findResources(def userId){
def q = new BasicDBObject()
q.put("user_id", userId)
return AUTHORITY_COLL.findOne(q)
}
}
@@ -62,12 +62,12 @@ class CcbCodeUtils {
return null
}
// 高层塔楼 0101 增加“超高层” 0108
// 高层板楼 0102 增加“老洋房” 0109
// 高层塔楼 0101 增加“超高层” 0109
// 高层板楼 0102 增加“老洋房” 0113
// 小高层板楼 0103 增加“花园洋房” 0110
// 多层板楼 0104 增加“四合院” 0111
// 独栋别墅 0105 增加“老式别墅洋房” 0112
// 联排别墅 0106
// 联排别墅 0106 增加“双拼别墅” 0108
// 叠拼别墅 0107
// 其他建筑形式 0199
@@ -84,7 +84,10 @@ class CcbCodeUtils {
types << "0110"
}
if(buildType.contains("老洋房")){
types << "0109"
types << "0113"
}
if(buildType.contains("双拼")){
types << "0108"
}
if(buildType.contains("叠拼")){
types << "0107"
@@ -99,7 +102,7 @@ class CcbCodeUtils {
types << "0104"
}
if(buildType.contains("超高层")){
types << "0108"
types << "0109"
}
if(buildType.contains("小高层")){
types << "0103"
@@ -25,11 +25,11 @@ class DataMatch {
static main(args) {
def task = new DataMatch()
// task.run1()
task.run2()
// task.run2()
// task.run3()
// println task.findAddress("²ıÖݽֵÀÌÄÏã·")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
}
def run1(){
@@ -0,0 +1,163 @@
package com.fangjia.script.gj.chongqingblank
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import com.fangjia.core.util.DataUtil
import org.apache.commons.lang.math.RandomUtils
class DistrictSelectHouse {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new DistrictSelectHouse();
// task.run()
// task.run2()
task.run3()
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本V1 -跑地址.xlsx",{d->
if(d['A']) {
def dist = CommonDatautils.findDistrict(d['A'],d['B'], d['C'], "district")
if(dist){
d['L'] = dist.address
}
}
},0)
}
def run2(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本.xlsx",{d->
if(d['N']){
def floorInfo = d['J']
def height = d['N']
if(floorInfo && height){
height = height.toInteger()
def mod = (int)(height/3)
def h_start = (height - mod).toInteger()
def random_floor = null
if(floorInfo.contains("低楼层")){
random_floor = random(1, mod)
}else if(floorInfo.contains("中楼层")){
random_floor = random(mod+1, h_start-1)
}else if(floorInfo.contains("高楼层")){
random_floor = random(h_start, height)
}
d['M'] = random_floor
}
}
},0)
}
int random(int i1, int i2) {
int max = Math.max(i1, i2);
int min = Math.min(i1, i2);
return min + RandomUtils.nextInt(max - min + 1);
}
def run(){
def city = "重庆"
def weekend = "2022-03-06"
def thirdPriceMap = findNearThirdPartyPrice(city)
def distStats = findCreditHightDistrict2(city, weekend, thirdPriceMap)
findAllHouse(distStats);
}
def findAllHouse(distStats){
def houses = []
distStats.each{k,v->
def arrs = k.split("#")
def list = findHouses(arrs[0],arrs[1], arrs[2], 20220201,20220401)
if(list){
houses.addAll(list)
}
}
if(houses){
def rets = []
for(def h : houses){
def row = []
row << h.city
row << h.region
row << h.district_name
row << h.area
row << h.avg_price
row << h.total_price
row << h.direction
row << h.estate_type
row << h.fitment
row << h.floor_info
row << h.room
row << h.hall
row << h.floor
row << h.height
row << h.p_date
rets << row
}
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆样本.xlsx", rets, "数据")
}
}
def findHouses(def city, def region, def name, def startDate, def endDate){
String sql = "select * from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and region='${region}' and district_name='${name}' and p_date >= ${startDate} and p_date<${endDate}"
def houses = []
ElasticUtils.executeSqlQuery(sql,{ data->
houses << data
})
if(houses.size()>10){
houses = houses[0..9]
}
return houses
}
def findNearThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', 202108))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
def findCreditHightDistrict2(def city, def weekend, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice) {
def precent = (thirdPrice - d.avg_price) * 100 / d.avg_price
if (Math.abs(precent) >= 0 && Math.abs(precent) <= 10) {
retMaps.put(d.city +"#"+d.region+"#"+d.district_name, d.avg_price)
}
}
}
return retMaps
}
}
@@ -0,0 +1,246 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeDetailCrawl {
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
static main(args) {
def task = new AnjukeDetailCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://chongqing.anjuke.com/community/?from=navigation"
WebDriver driver = new ChromeDriver()
def regions = getRegions(driver, url)
getAllRegions(driver, regions)
}
def getAllRegions(driver, regions){
for(def url : regions){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def total = getTotal(doc)
if(total) {
if (total > 1250) {
} else {
def links = getAllLinks(url, driver)
parse(links,driver)
}
}
}
}
def getAllLinks(url, driver){
def stop = false
def urls = []
while(!stop) {
def doc = Jsoup.parse(driver.getPageSource());
def details = JQueryUtils.select(".list-cell a.li-row", doc)
for (def d : details) {
urls << d.attr("href")
}
def nextEle = JQueryUtils.select(".next", doc)
if (nextEle) {
driver.findElement(By.cssSelector(".next")).click()
}else{
stop = true
}
if(urls.size()>20){
stop = true
}
}
return urls
}
def getTotal(doc){
def totalText = JQueryUtils.selectText(".total-info", doc)
if(totalText){
def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim()
if(total && total.isNumber()){
return total.toInteger()
}
}
return null;
}
def getRegions(driver, url){
driver.get(url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
// if(loginIfm){
// def href = loginIfm[0].attr("src")
// driver.get(href)
// driver.findElementById("pwdTab").click()
// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框
// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框
// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。
// elemUsername.sendKeys("17740818975");
// elemPassword.sendKeys("12qwaszx");
// btn.submit(); // 点击行为;
//
// def regionEles = JQueryUtils.select(".region-parents li a", doc)
// def urls = []
// for(def e : regionEles){
// urls << e.attr("href")
// }
// return urls;
// }
def regionEles = JQueryUtils.select(".region-parents li a", doc)
def urls = []
for(def e : regionEles){
urls << e.attr("href")
}
return urls;
}
def parse(def links, driver){
def rets = []
for(def link : links){
def district = parsing(link, driver)
if(district){
rets << district
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
rets = []
}
}
}
if (rets.size()> 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
def parsing(url, driver){
def district = [:]
def stop = false
def count = 0
while(!stop){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def headers = JQueryUtils.select(".crumbs a", doc)
if(headers){
if (headers && headers.size() >= 3) {
district.city = headers[1].text().replace("小区", "")
district.region = headers[2].text().replace("小区", "")
district.name = headers[headers.size() - 1].text().replace("二手房", "")
}
district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc)
def priceStr = JQueryUtils.selectText(".average", doc)
if (priceStr && priceStr.isNumber()) {
district.avg_price = priceStr.toInteger()
}
def infoDiv = JQueryUtils.select(".info-list", doc)
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
if (houseCountStr) {
houseCountStr = houseCountStr.replace("户", "")
if (houseCountStr) {
district.house_count = houseCountStr.toInteger()
}
}
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
if(carSiteStr){
if(carSiteStr.contains("(")){
carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("("))
}
if(carSiteStr && carSiteStr.isNumber()){
district.car_site = carSiteStr
}
}
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.url = url
district.c_date = new Date()
district.m_date = new Date()
district.root = "anjuke.com"
stop = true;
}else{
Thread.sleep(3000)
count++
}
if(count>10){
stop = true;
}
}
return district
}
String getValue(String s){
if(s && s.contains("暂无")){
return null
}
return s
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.gj.crawl
import org.jsoup.Jsoup
class CommonCrawl {
static def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
}
@@ -55,7 +55,7 @@ class AddDistrictFieldValue {
// }
// })
// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
def citys = ["重庆"]//CcbCommonUtils.citys;
//第三方均价数据匹配
// for(def city : citys){
// def rets = findAllDataByCity(city)
@@ -63,11 +63,11 @@ class AddDistrictFieldValue {
// }
//第三方均价数据填补
// for(def city : citys){
//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
// def rMap = getMatchDistrict(city)
// createResultValue(dists, rMap)
// }
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
def rMap = getMatchDistrict(city)
createResultValue(dists, rMap)
}
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
@@ -103,52 +103,52 @@ class AddDistrictFieldValue {
// }
def citys = ["上海"]
def updateIndex = 0
def rets = []
rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
def total = dists.size()
for(def dist : dists){
if(dist.household_count){
def ret = []
ret << dist._id.toString()
ret << dist.city
ret << dist.region
ret << dist.name
ret << dist.household_count
ret << dist.loudong_count
def details = findDistrictDetail(dist)
if(details){
for(def d : details){
def region = RegionData.formatRegion(city, d.region)
if(d.region == region){
if(dist.household_count && dist.household_count==d.house_count){
ret << d.loudong_count
ret << "链家"
// updateIndex++
// def udata = new BasicDBObject()
// udata.put("household_count", d.house_count.toInteger())
// udata.put("m_date", new Date())
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}else{
ret << ""
ret << ""
}
rets << ret
}
}
ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
}
// def citys = ["上海"]
// def updateIndex = 0
// def rets = []
// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
// def total = dists.size()
//
// for(def dist : dists){
// if(dist.household_count){
// def ret = []
// ret << dist._id.toString()
// ret << dist.city
// ret << dist.region
// ret << dist.name
// ret << dist.household_count
// ret << dist.loudong_count
// def details = findDistrictDetail(dist)
// if(details){
// for(def d : details){
// def region = RegionData.formatRegion(city, d.region)
// if(d.region == region){
// if(dist.household_count && dist.household_count==d.house_count){
// ret << d.loudong_count
// ret << "链家"
//// updateIndex++
//// def udata = new BasicDBObject()
//// udata.put("household_count", d.house_count.toInteger())
//// udata.put("m_date", new Date())
//// def q = new BasicDBObject()
//// q.put("_id", dist._id)
//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }
// }
// }
// }else{
// ret << ""
// ret << ""
// }
//
// rets << ret
// }
// }
// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
// }
}
@@ -303,6 +303,14 @@ class AddDistrictFieldValue {
def createResultValue(dists, rMap){
def updateIndex = 0
def total = dists.size()
def addressIndex = 0
def allAreaIndex = 0
def buildAreaIndex = 0
def virescence_percent_index = 0
def cubage_percent_index = 0
def finish_building_date_index = 0
def household_count_index = 0
def car_site_count_index = 0
for(def d : dists){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
@@ -311,12 +319,14 @@ class AddDistrictFieldValue {
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
addressIndex++
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
allAreaIndex++
udata.put("all_area", lastValue)
}
}
@@ -324,6 +334,7 @@ class AddDistrictFieldValue {
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
buildAreaIndex++
udata.put("build_area", lastValue)
}
}
@@ -331,6 +342,7 @@ class AddDistrictFieldValue {
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
virescence_percent_index ++
udata.put("virescence_percent", lastValue)
}
}
@@ -338,6 +350,7 @@ class AddDistrictFieldValue {
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
cubage_percent_index ++
udata.put("cubage_percent", lastValue)
}
}
@@ -345,6 +358,7 @@ class AddDistrictFieldValue {
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
finish_building_date_index ++
udata.put("finish_building_date", lastValue)
}
}
@@ -352,6 +366,7 @@ class AddDistrictFieldValue {
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
household_count_index ++
udata.put("household_count", lastValue)
}
}
@@ -359,6 +374,7 @@ class AddDistrictFieldValue {
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
car_site_count_index ++
udata.put("car_site_count", lastValue)
}
}
@@ -373,6 +389,15 @@ class AddDistrictFieldValue {
}
}
}
println "addressIndex "+ addressIndex
println "allAreaIndex "+ allAreaIndex
println "buildAreaIndex "+ buildAreaIndex
println "virescence_percent_index "+ virescence_percent_index
println "cubage_percent_index "+ cubage_percent_index
println "finish_building_date_index "+ finish_building_date_index
println "household_count_index "+ household_count_index
println "car_site_count_index "+ car_site_count_index
}
def getValuesByField(sourceMap, field, estateType){
@@ -12,7 +12,19 @@ class TempMatchDistrict {
static main(args) {
def task = new TempMatchDistrict()
// task.run()
task.regionConver()
// task.regionConver()
task.stat()
}
def stat(){
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
def total = 0
for(def city : citys){
def count = CommonDatautils.findAllDistrictCount("district", city)
total = total + count
println city +" "+count
}
println "共:"+total
}
def regionConver(){
@@ -29,7 +41,7 @@ class TempMatchDistrict {
}
def run(){
def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据"
def files = ["合肥.xlsx","湖北.xlsx","陕西.xlsx","上海.xlsx","深圳.xlsx","苏州.xlsx","天津.xlsx","雄安.xlsx","浙江.xlsx","重庆.xlsx"]
def files = ["杭州.xlsx"]
for(def f : files){
def filePath = path+"\\"+f;
match(filePath);
@@ -10,7 +10,7 @@ class FangjiaClientUtils {
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
JSONObject jsonObj = null
try {
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
if(address){
url = url + "&address="+address
@@ -38,8 +38,8 @@ class FangjiaClientUtils {
}
public static def sendDistrictPriceCalculate(def city, def region, def name, def count){
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
def content = null;
try{
content = new HttpRequest().newGet(url, null, "utf-8")
@@ -83,6 +83,23 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100"
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static Map<String, Object> JsonToMap(JSONObject json) {
Map<String, Object> columnValMap = new HashMap<String, Object>();
@@ -0,0 +1,340 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import com.fangjia.core.util.BaiDuApiUtils
import java.sql.ResultSet
class ZsNoExistsToDB {
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
def task = new ZsNoExistsToDB()
task.run()
// task.addDistrictToDB()
}
def addDistrictToDB(){
def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1)
def keys = []
def addIndex = 0
for(def d : datas){
if(d['J'] && d['J']=="新增"){
def city = d['A']
def region = d['B']
def name = d['F'].toString().replace("·","")
def address = d['G']
def dist = CommonDatautils.findDistrict(city,region,name,"district")
if(!dist){
dist = CommonDatautils.findDistrict(city,region,d['C'],"district")
}
if(!dist){
keys << city+"##"+region+"##"+name
def alias = []
def str = d['H']
if(str){
def arrs = str.split(",")
for(def s : arrs){
if(!alias.contains(s)){
alias << s
}
}
}
if(address){
address = AddressUtil.formatAddress(city, address)
}
def lng = d['M']
def lat = d['L']
def lng2 = null
def lat2 = null
long lng_new = 0
long lat_new = 0
long lng2_new = 0
long lat2_new = 0
if(lng){
lng = lng.toString().toDouble()
lat = lat.toString().toDouble()
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
lng2 = baidu.x
lat2 = baidu.y
lng_new = lng *10000000000L
lat_new = lat *10000000000L
if(lng2){
lng2_new = lng2 *10000000000L
lat2_new = lat2 *10000000000L
}
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", address)
if(lng_new){
data.put("lat",lat_new)
data.put("lng",lng_new)
data.put("lat2",lat2_new)
data.put("lng2",lng2_new)
}
if(alias){
data.put("alias",alias)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
DbUtils.seaweed.save(data)
FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12);
addIndex++
println "[新增] "+addIndex +" "+ data
}else{
println dist.city +" "+dist.region+" "+dist.name+" 小区已存在"
}
}
}
}
def run(){
def rets = findOpenLog2()
// ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
println rets.size()
// matchMap2()
}
def matchMap2() {
ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d ->
if(!d['E']){
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject(d['A'],d['B'],d['C'], rets);
def locationMap = [:]
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
map = findStandardObject(d['A'],d['B'],d['C'], rets);
}
if(map){
println map
if(map.adname){
d['E'] = map.adname
}else if(map.region){
d['E'] = map.region
}
d['F'] = map.name
d['G'] = map.address
d['K'] = map.estateType
d['L'] = map.lat
d['M'] = map.lng
d['N'] = map.lat2
d['O'] = map.lng2
d['P'] = map.source
d['Q'] = RegionData.formatRegion(d['A'], d['E'])
}
}
})
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
def matchMap(){
ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data->
def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德")
if(!gdData){
gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德")
}
if(gdData){
data['E'] = gdData.district
data['F'] = gdData.name
data['G'] = gdData.address
data['H'] = gdData.type
data['I'] = gdData.lng
data['J'] = gdData.lat
data['K'] = gdData.score
}
def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度")
if(!bdData){
bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度")
}
if(bdData){
data['L'] = bdData.district
data['M'] = bdData.name
data['N'] = bdData.address
data['O'] = bdData.type
data['P'] = bdData.lng
data['Q'] = bdData.lat
data['R'] = bdData.score
}
},0)
}
def findOpenLog2(){
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18")
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate)
def rets = []
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def index = 0
def count = 0
def keys = []
for(def d : datas){
if(d.code == 400){
index++
def msg = d.msg
if(msg.contains("该楼盘不存在")){
def arrs = msg.split(" ")
if(arrs){
def rMap = [:]
for(def s : arrs){
if(s.contains(":")){
def ars = s.split(":")
if(ars.size()==2){
rMap.put(ars[0], ars[1])
}else{
rMap.put(ars[0], null)
}
}
}
if(rMap){
count++
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
if(!keys.contains(key)){
keys << key
def params = d.parameter_map
def address = null
if(params && params.address){
address = params.address
}
def r = [rMap.city, rMap.region, rMap.name, address]
if(!rets.contains(r)){
rets << r
}
}
}
}
}else{
println msg
}
}
}
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
return rets;
}
def findOpenLog(){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1")
sqlBuffer.append(" and user_id = '1007569'")
sqlBuffer.append(" and request_uri = '/czbank/evaluation'")
sqlBuffer.append(" and c_date >= '2022-01-01'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def rets = []
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
def index = 0
def count = 0
while (rs.next()){
def code = rs.getInt("code")
if(code == 400){
index++
def msg = rs.getString("msg")
if(msg && msg.contains("该楼盘不存在")){
def arrs = msg.split(" ")
if(arrs){
def rMap = [:]
for(def s : arrs){
if(s.contains(":")){
def ars = s.split(":")
if(ars.size()==2){
rMap.put(ars[0], ars[1])
}else{
rMap.put(ars[0], null)
}
}
}
if(rMap){
count++
def params = rs.getObject("parameter_map")
println params
def r = [rMap.city, rMap.region, rMap.name]
if(!rets.contains(r)){
rets << r
}
}
}
}else{
println msg
}
}
}
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
return rets;
}
}
@@ -1,15 +1,18 @@
package com.fangjia.script.gj.level
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil
@@ -18,7 +21,11 @@ import groovy.sql.Sql
import net.sf.json.JSONObject
import org.apache.commons.httpclient.util.DateUtil
import org.apache.commons.lang.StringUtils
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
import org.apache.commons.math3.stat.descriptive.moment.Mean
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -44,13 +51,15 @@ class DistrictIndexBatch extends BaseBatch{
dataSource.setProperties(connectionProperties)
}
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc"
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM")
static main(args) {
runBatch(new DistrictIndexBatch(), args);
}
@@ -63,9 +72,8 @@ class DistrictIndexBatch extends BaseBatch{
def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = CommonDatautils.findAllCity();
citys.remove("上海")
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
def code = cityCodeDatas.get(city)
def city_avg_total_price = null
if(city!="ÉϺ£"){
@@ -78,7 +86,9 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
calcute(city, dists, code, city_avg_total_price)
// calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
println city +" "+city_avg_total_price
}
@@ -110,48 +120,28 @@ class DistrictIndexBatch extends BaseBatch{
def total_price = getHousePrice(d, dists, priceMap, citycode);
def avg_cost = getAvgCost(d, costDatas)
def total_price_index = 0
if (total_price) {
total_price_index = 286.0134196 * (total_price - city_avg_total_price) / 313.3693508
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
}
def avg_charge_index = 0
if (avg_charge) {
avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
}
def rank_index = 0
if (rank) {
rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
}
def avg_cost_index = 0
if (avg_cost) {
avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
}
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name)
def map = [:]
map.id = d._id.toString()
map.city = d.city
map.region = d.region
map.name = d.name
map.avg_total_price = total_price_index
map.avg_estate_charge = avg_charge
map.price_rank = rank
map.avg_cost = avg_cost
map.comp_value = potentiality
map.month = "2022-01"
map.total_price = total_price
//map.comp_value = potentiality
//map.avg_total_price = total_price_index
map.month = month
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
def newUrl = targetUrl +"/"+map.id
map.sell_avg_price = avg_price
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price
println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price+""
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -163,16 +153,288 @@ class DistrictIndexBatch extends BaseBatch{
latch.await();
def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
def fileName = "/home/gj/potentiality_${city}.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
// def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"]
// def fileName = "/home/gj/potentiality_${city}.dat"
// if(CommonDatautils.isWindow()){
// fileName = "D:\\数据\\评级\\${city}综合潜力.dat"
// }
// FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
// // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
}
def calcuteIndex(city, dists, citycode, city_avg_total_price){
def detailMap = findAllDistrictDatas(city)
def avgMap = calcuteAvgValues(detailMap);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def avg_total_price = avgMap.avg_price
def avg_charge = avgMap.avg_charge
def avg_cost = avgMap.avg_cost
def avg_rank = avgMap.avg_rank
def sd_avg_price = avgMap.sd_avg_price
def sd_avg_charge = avgMap.sd_avg_charge
def sd_avg_cost = avgMap.sd_avg_cost
def sd_avg_rank = avgMap.sd_avg_rank
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def rMap = detailMap.get(d._id.toString())
if(rMap){
def total_price_index = 0
if (rMap.total_price) {
// total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / 313.3693508
total_price_index = 286.0134196 * (rMap.total_price - avg_total_price) / sd_avg_price
total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index)
}
def avg_charge_index = 0
if (rMap.avg_estate_charge) {
// avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / 0.772635241
avg_charge_index = 139.1374524 * (rMap.avg_estate_charge - avg_charge) / sd_avg_charge
avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index)
}
def rank_index = 0
if (rMap.price_rank) {
// rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / 0.261907636
rank_index = 21.8427926 * (rMap.price_rank - avg_rank) / sd_avg_rank
rank_index = DataHandleUtils.getDoubleRound(6, rank_index)
}
def avg_cost_index = 0
if (rMap.avg_cost) {
// avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / 31.3526959
avg_cost_index = 67.0195871700 * (rMap.avg_cost - avg_cost) / sd_avg_cost
avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index)
}
def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248
potentiality = DataHandleUtils.getDoubleRound(6, potentiality)
def year = CommonUtils.getBuildYear(d.finish_building_date)
def sqmetre_rent = CommonDatautils.findNewSqmetreRent(d.city,d.region,d.name)
def map = [:]
map.id = d._id.toString()
map.city = d.city
map.region = d.region
map.name = d.name
map.avg_estate_charge = rMap.avg_estate_charge
map.price_rank = rMap.price_rank
map.avg_cost = rMap.avg_cost
map.total_price = rMap.total_price
map.comp_value = potentiality
map.avg_total_price = total_price_index
map.month = month
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
map.finish_date = year
map.sell_avg_price = rMap.sell_avg_price
if(sqmetre_rent){
map.sqmetre_rent = sqmetre_rent
}
if(rMap.sell_avg_price && sqmetre_rent){
map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger()
}
if(d.household_count){
map.household_count = d.household_count
}
if(d.lat){
def lMap = [:]
lMap.lon = d.lng/10000000000.0
lMap.lat = d.lat/10000000000.0
map.location = lMap
}
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
//修改多了会报错
// def newUrl = targetUrl +"_update_by_query"
// def must = []
// def term1 = ["match_phrase":["id":d._id.toString()]]
// must << term1
// def source = "ctx._source.comp_value="+potentiality+";ctx._source.avg_total_price="+total_price_index+";ctx._source.month='2022-03'"+";ctx._source.m_date='"+DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")+"'"
// def params = ["script":["source":source,"lang": "painless"], "query":["bool":["must":must]]]
// httpSaveData(newUrl, params);
println totalCount.get() +" "+d.city+" "+d.region+" "+d.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown();
}
}
})
}
latch.await();
}
def findAllDistrictDatas(def city){
def detailMap = [:]
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
for(def rs : houses){
def id = rs.id
def rMap = detailMap.get(id)
if(!rMap){
rMap = [:]
detailMap.put(id, rMap)
}
rMap.put("total_price", rs.total_price)
rMap.put("price_rank", rs.price_rank)
rMap.put("avg_cost", rs.avg_cost)
rMap.put("avg_estate_charge", rs.avg_estate_charge)
rMap.put("sell_avg_price", rs.sell_avg_price)
}
return detailMap;
}
def findDistrictDatasByCity(def city){
def must = []
def term1 = ["term":["city":city]]
must << term1
// def params = ["query":["bool":["must":must]]]
def params = ["query":["match":["city":city]]]
def s = httpSaveData(targetUrl+"/_search", params);
def total = 0;
if(s){
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
total = hits.total
}
}
}
def calcuteAvgValues(detailMap){
def prices = []
def ranks = []
def costs = []
def charges = []
detailMap.each{k,v->
if(v.total_price){
prices << v.total_price
}
if(v.price_rank){
ranks << v.price_rank
}
if(v.avg_cost){
costs << v.avg_cost
}
if(v.avg_estate_charge){
charges << v.avg_estate_charge
}
}
FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false);
// ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据")
def avg_price = null
def dx_avg_price = null
def sd_avg_price = null
if(prices){
def v = prices.sum()/prices.size()
avg_price = DataHandleUtils.getDoubleRound(6, v)
double pow_sum = 0
prices.each {p->
def sod = Math.abs(p - avg_price)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_price = DataHandleUtils.getDoubleRound(6,pow_sum/prices.size())
sd_avg_price = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_price))
}
def avg_rank = null
def dx_avg_rank = null
def sd_avg_rank = null
if(ranks){
def v = ranks.sum()/ranks.size()
avg_rank = DataHandleUtils.getDoubleRound(6, v)
def pow_sum = 0;
ranks.each {p->
def sod = Math.abs(p - avg_rank)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_rank = DataHandleUtils.getDoubleRound(6,pow_sum/ranks.size())
sd_avg_rank = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_rank))
}
def avg_cost = null
def dx_avg_cost = null
def sd_avg_cost = null
if(costs){
def v = costs.sum()/costs.size()
avg_cost = DataHandleUtils.getDoubleRound(6, v)
double pow_sum = 0;
costs.each {p->
def sod = Math.abs(p - avg_cost)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_cost = DataHandleUtils.getDoubleRound(6,pow_sum/costs.size())
sd_avg_cost = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_cost))
}
def avg_charge = null
def dx_avg_charge = null
def sd_avg_charge = null
if(charges){
def v = charges.sum()/charges.size()
avg_charge = DataHandleUtils.getDoubleRound(6, v)
def pow_sum = 0;
charges.each {p->
def sod = Math.abs(p - avg_charge)
pow_sum = Math.pow(sod, 2) + pow_sum
}
dx_avg_charge = DataHandleUtils.getDoubleRound(6,pow_sum/charges.size())
sd_avg_charge = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_charge))
}
def avgMap = [:]
avgMap.put("avg_rank", avg_rank)
avgMap.put("avg_price", avg_price)
avgMap.put("avg_cost", avg_cost)
avgMap.put("avg_charge", avg_charge)
avgMap.put("dx_avg_rank", dx_avg_rank)
avgMap.put("dx_avg_price", dx_avg_price)
avgMap.put("dx_avg_cost", dx_avg_cost)
avgMap.put("dx_avg_charge", dx_avg_charge)
avgMap.put("sd_avg_rank", sd_avg_rank)
avgMap.put("sd_avg_price", sd_avg_price)
avgMap.put("sd_avg_cost", sd_avg_cost)
avgMap.put("sd_avg_charge", sd_avg_charge)
return avgMap;
}
//方差s^2=[(x1-x)^2 +...(xn-x)^2]/n 或者s^2=[(x1-x)^2 +...(xn-x)^2]/(n-1)
public double Variance(def x) {
int m=x.size();
double sum=0;
for(int i=0;i<m;i++){//求和
sum+=x[i];
}
double dAve=sum/m;//求平均值
double dVar=0;
for(int i=0;i<m;i++){//求方差
dVar+=(x[i]-dAve)*(x[i]-dAve);
}
return dVar/m;
}
def getAvgTotalPrice(city, startDate, endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT avg(total_price) as avg_total_price FROM \"seawater-20*\" WHERE 1=1")
@@ -309,7 +571,7 @@ class DistrictIndexBatch extends BaseBatch{
return datas
}
def sortByDistance(d, list){
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
@@ -496,6 +758,7 @@ class DistrictIndexBatch extends BaseBatch{
def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
@@ -508,7 +771,7 @@ class DistrictIndexBatch extends BaseBatch{
String param = null;
BufferedWriter mWriter = null;
InputStreamReader inStream = null;
StringBuilder buffer = new StringBuilder();
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
@@ -547,7 +810,7 @@ class DistrictIndexBatch extends BaseBatch{
} catch (Exception e) {
e.printStackTrace()
}
return null;
return buffer.toString();
}
}
@@ -0,0 +1,215 @@
package com.fangjia.script.util
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class DsElasticUtils {
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
connectionProperties.setProperty("user","developer");
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
dataSource.setProperties(connectionProperties);
return dataSource;
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
* @return
*/
static createBulkProcessor(String host, RestHighLevelClient client = null) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
};
if(!client){
client = getClient(host);
}
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
final SearchHit hit = searchHits[k];
executorService.submit(new Runnable() {
@Override
void run() {
try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
e.printStackTrace()
}
}
})
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def results = []
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
SearchHit searchHit = searchHits[k]
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
client.close()
return results;
}
}
@@ -1,11 +1,20 @@
package com.fangjia.script.gj.location
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.codehaus.jackson.sym.NameN
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import com.fangjia.script.gj.district.FangjiaClientUtils
class DistrictLocationUpdate {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
@@ -19,20 +28,214 @@ class DistrictLocationUpdate {
static main(args) {
def task = new DistrictLocationUpdate()
task.run()
// task.run()
task.mapMatch()
// task.updateLocationToDB()
}
def run(){
def citys = CommonDatautils.findAllCity()
def citys = ["福州"]//CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
updateLocation(city, dists);
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"])
def rets = []
rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"]
for(def d : dists){
def lng = null
def lat = null
if(d.lng){
lng = d.lng/10000000000.0
lat = d.lat/10000000000.0
}
rets << [d.city, d.region, d.name, d.address, lng,lat]
}
ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据")
}
}
def mapMatch(){
def index = 0
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
if(!d['H'] && d['C']=="华侨城龙居御景二期"){
index++
// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德")
// println d['A']+" "+d['B']+" "+d['C']
// if(gdData){
// d['G'] = gdData.district
// d['H'] = gdData.name
// d['I'] = gdData.address
// d['J'] = gdData.type
// d['K'] = gdData.lng
// d['L'] = gdData.lat
// d['M'] = gdData.score
// if(d['E'] && d['K']){
// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
// d['N'] = distance
// }
// }
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
}
if(map){
if(map.adname){
d['G'] = map.adname
}else if(map.region){
d['G'] = map.region
}
d['H'] = map.name
d['I'] = map.address
if(map.type){
d['J'] = map.type
}else if(map.estateType){
d['J'] = map.estateType
}
d['K'] = map.lng
d['L'] = map.lat
d['M'] = map.score
if(d['E'] && d['K']){
def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
d['N'] = distance
}
}
}
},0)
}
def updateLocationToDB(){
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
if(d['N'] && d['N']>1000){
if(d['M']>=0.4 && d['K']){
def query = new BasicDBObject()
query.put("city", d['A'])
query.put("region", d['B'])
query.put("name", d['C'])
def udata = new BasicDBObject()
udata.put("m_date", new Date())
Long lng0 = d['K'].toString().toDouble()*10000000000L
Long lat0 = d['L'].toString().toDouble()*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
def gmap = GeocodeUtils.getGaodeToBaidu(d['K'].toString().toDouble(), d['L'].toString().toDouble())
Long lng2 = gmap.get("lng")*10000000000L
Long lat2 = gmap.get("lat")*10000000000L
udata.put("lat2", lat2)
udata.put("lng2", lng2)
DbUtils.seaweed.update(query, new BasicDBObject('$set', udata))
}
}
},0)
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
continue;
}
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
def queryPoiMap(def city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
@@ -40,9 +40,9 @@ class LocationTest {
// System.out.println(str);
def task = new LocationTest()
// task.run()
task.run()
// task.run2()
task.run3()
// task.run3()
// task.run4()
// task.run5()
}
@@ -0,0 +1,354 @@
package com.fangjia.script.gj.meituan
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.bson.types.ObjectId
class MatchTest {
static main(args) {
def task = new MatchTest()
// task.run()
// task.addNum()
task.dongguang()
// task.meituanDel()
}
def meituanDel(){
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
def index = 0
ExcelUtils.modify(path, { row ->
def flag = row['O']
if(flag){
def city = row['F']
def region = row['G']
def name = nameFileter(row['H'])
def obj = null
if(city == "东莞"){
obj = CommonDatautils.findDistrict(city, null, name, "district")
}else{
obj = CommonDatautils.findDistrict(city, region, name, "district")
}
if(obj){
index++
println index +" "+ city +' '+ region+" "+name
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(obj._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
},0);
}
def dongguang(){
def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx
ExcelUtils.modify(path, { row ->
def region = null
if(row['B'] && row['B']=="NULL" ){
region = null
}
def name = nameFileter(row['H'])
def address = row['E']
if (name && name.contains("湖畔新家")) {
def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district")
if(obj){
def json = GaoDeApiUtils.geo(row['F'], address)
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
def twinshop = getStreetInfo(m.location)
if(twinshop){
def newRegion = RegionData.formatRegion("东莞",twinshop)
if(newRegion){
def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district")
if(dbObj){
println "delete"
DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id))
}else{
println "update"
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
}
}
}
// println address +" "+twinshop
}
}
}
}
},0)
}
def getStreetInfo(def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
}
}
}
return township;
}
def addNum(){
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx
def index = 0
ExcelUtils.modify(path, { row ->
index++
row['A'] = index
},0)
}
def run() {
def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
def radioMap = [:]
for(def city : citys){
// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
// radioMap.put(city, avg_radio)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
ExcelUtils.multiThreadingModify(path, { row ->
def region = null
if(row['C'] && row['C']=="NULL" ){
region = null
}
def name = row['D']
def address = row['F']
def city = RegionData.formatCity(row['B'])
if(name && !row['N']){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
if(rMap.level){
row['P'] = rMap.level
}
}
}
}
if (name && row['N']) {
//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name)
// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null)
// if (jsonObject != null && jsonObject.precise != -1) {
// row['F'] = jsonObject.get("city")
// row['G'] = jsonObject.get("district")
// row['H'] = jsonObject.get("name")
// row['I'] = jsonObject.get("propertyType")
// row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
// }
if(row['G'] && row['H'] && row['I']){
// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H'])
// if(priceObj){
// row['M'] = priceObj.avg_price
// }
// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H'])
// if(sqmetreRentPrice){
// row['N'] = sqmetreRentPrice
// }
if(row['N'] && !row['O']){
def avg_rent_radio = radioMap.get(city)
row['O'] = (int)(row['N']/avg_rent_radio)
}
}
}
})
}
def getArroundDistrictPriceByAddress(def city, def address, def levels){
def json = GaoDeApiUtils.geo(city, address)
def location = null
def detail = new BasicDBObject()
def level = null
def rank = null
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
location = m.location
level = m.level
}
}
if(location){
def arrs = location.split(",")
detail.lng = arrs[0].toString().toDouble()*10000000000L
detail.lat = arrs[1].toString().toDouble()*10000000000L
}
if(detail.lng){
detail.city = city
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
if(list && list.size()>10){
list = sortByDistance(detail, list);
list = list[0..9]
}
def arroundAvgPrice = getArroundAvgPrice(list)
if(arroundAvgPrice){
def mod = Integer.MAX_VALUE;
for(def d : levels){
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
if(newMod<mod){
mod = newMod
rank = d.level
}
}
}
}
def m = [:]
m.level = level
m.rank = rank
return m
}
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
}
//排序
Collections.sort(list, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return list;
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
return datas;
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
return null
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
return name;
}
}
@@ -0,0 +1,403 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.gj.paimai.ChromeDriverPool
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new BeiKeCrawlByChrome();
task.run()
// task.run1()
task.addDealRegion();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def citys = getAllCityUrls(driver)
citys = citys.reverse()
def start = 0
// for(def i=0;i<citys.size();i++){
// def m = citys[i]
// println i+" "+ m.city
// if(m.city=="天津"){
// start = i
// break;
// }
//
// }
// citys = citys[(start+1)..(citys.size()-1)]
getLogin(driver)
crawlData(driver, citys)
}
def addDealRegion(){
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
for(def d : dbcursor){
keys << ["city":d.city, "name":d.district_name]
}
for(def m : keys){
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
if(list && list.size()==1){
def dist = list[0]
def query = new BasicDBObject()
query.put("city", m.city)
query.put("region", null)
query.put("district_name", m.name)
// query.put("source", "链家在线")
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
def data = new BasicDBObject()
data.put("region", dist.region)
def ds = DEAL_PRICE_COLL.count(query)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
}
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def m : citys){
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, m.city, m.url)
println m.city +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
def getPageUrls(driver, cityUrl){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
stop = true;
break;
}
}
page++;
}else{
stop = true;
}
}
return detailUrls
}
def getPageDeail(driver, city, cityUrl){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "/chengjiao/pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
}else{
stop = true;
break;
}
}
if(page>=100){
stop = true;
break;
}
page++;
}else{
stop = true;
}
}
return detailUrls
}
def parse(city,e){
try {
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble()
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "贝壳找房"
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.url = e.select(".title a").attr("href")
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}catch(Exception ex){
ex.printStackTrace()
println e.select(".title a").attr("href")
}
}
def findRegionByThirdPary(city, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", district_name)
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
def dbo = null;
for(def d : dbcursor){
dbo = d
}
if(dbo){
return dbo.region
}
return null;
}
def getContent(driver, detailUrl){
driver.get(detailUrl)
def str = driver.getPageSource()
if(str && str.contains("<title>登录</title>")){
getLogin(driver)
return getContent(driver, detailUrl)
}
return str;
}
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
if(detailUrls){
for(def i=0;i<detailUrls.size();i++){
def detailUrl = detailUrls[i]
try{
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
} catch (Exception e) {
e.printStackTrace()
}
}
}
}
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
def content = getContent(driver, detailUrl)
def doc = Jsoup.parse(content);
def eles = JQueryUtils.select(".deal-bread a", doc)
def detail = new BasicDBObject()
if(eles && eles.size()>=4){
detail.city = eles[0].text().replace("房产网","")
detail.region = eles[2].text().replace("二手房成交","")
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
}
if(detail.district_name) {
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
def dbo = findTrade(DEAL_PRICE_COLL, detail)
if (!dbo) {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.url = detailUrl
detail.c_date = new Date()
detail.m_date = new Date()
DEAL_PRICE_COLL.save(detail)
}
}
return detail;
}
def findTrade(def detail){
def q = new BasicDBObject()
q.put("city", detail.city)
q.put("region", detail.region)
q.put("district_name", detail.district_name)
q.put("area", detail.area)
q.put("total_price", detail.total_price)
q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo
}
def getAllCityUrls(driver){
driver.get("https://www.ke.com/city/")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
def list = []
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("重庆")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
}
}
}
return list
}
def getLogin(driver){
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
}
}
@@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import javax.swing.JApplet
import java.text.DateFormat
import java.util.function.Consumer
@@ -41,45 +42,138 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
System.setProperty("http.nonProxyHosts", "localhost");
def doc = getDocument();
def detailUrls = getDetailLinks(doc);
def rets = batchParse(detailUrls);
saveToDB(rets);
WebDriver driver = new ChromeDriver()
def proviceMap = getDocument(driver);
def citys = getCity(driver,proviceMap)
def detailUrls = getDetailLinks(driver, citys);
def getDetailLinks(doc){
def urls = []
def getDetailLinks(driver, cityList){
def zxcitys = ["天津市","北京市","上海市","重庆市"]
for(def cityMap : cityList){
def citys = cityMap.citys
citys.each{city,id->
def urls = []
if(!zxcitys.contains(cityMap.provice)){
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
getDocument(url, driver)
}else{
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
getDocument(url, driver)
}
def doc = Jsoup.parse(driver.getPageSource());
def count = JQueryUtils.selectText(".count", doc)
def totalPage = getTotalPage(count, 12)
if(totalPage){
for(def i=1;i<=totalPage;i++){
doc = Jsoup.parse(driver.getPageSource());
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
if(eles){
for(def el :eles){
def href = el.attr("href")
def map = [:]
map.province = cityMap.provice
map.city = city
map.url = "https://paimai.caa123.org.cn"+href
urls << map
}
}
try{
driver.findElement(By.className("next")).click()
}catch(Exception e){
Thread.sleep(8000)
driver.findElement(By.className("next")).click()
}
}
def rets = batchParse(urls, driver);
saveToDB(rets);
println city +" "+urls.size()
}
}
println cityMap.provice +" 完成"
}
//return urls;
}
def getTotalPage(def total, def pageSize){
if(total){
def mode = total.toInteger()/pageSize
if(mode > mode.toInteger()){
mode = mode.toInteger() +1
}
return mode;
}
return 0;
}
def getCity(driver, proviceMap){
def rows = []
def zxcitys = ["天津市","北京市","上海市","重庆市"]
proviceMap.each {k,v->
if(!zxcitys.contains(k)){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
def doc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc)
def cityMap = [:]
for(def c : cityEles){
def cityid = c.attr("data-cityid")
def city = c.text()
if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
cityMap.put(city, cityid)
}
}
def map = [:]
map.provice = k
map.provice_id = v
map.citys = cityMap
rows << map
}else{
def map = [:]
map.provice = k
map.provice_id = v
def cityMap = [:]
cityMap.put(k, v)
map.citys = cityMap
rows << map
}
}
return rows
}
def getDocument(driver){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport=");
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
def doc = Jsoup.parse(driver.getPageSource());
def provinceMap = [:]
if(doc){
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
if(eles){
for(def el :eles){
def href = el.attr("href")
urls << "https://paimai.caa123.org.cn"+href
def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc)
for(def ele : proviceEles){
def provice = ele.text()
def value = ele.attr("data-proviceid")
if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){
provinceMap.put(provice, value)
}
}
}
return urls;
return provinceMap;
}
def getDocument(){
WebDriver driver = new ChromeDriver()
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
def batchParse(def urls, driver){
def doc = Jsoup.parse(driver.getPageSource());
return doc;
}
def batchParse(def urls){
WebDriver driver = new ChromeDriver()
def ret = []
for(def url :urls){
driver.get(url);
def doc = Jsoup.parse(driver.getPageSource());
def res = parse(url, doc);
for(def map :urls){
def doc = getDocument(map.url, driver)
def res = parse(map.province, map.city, map.url, doc);
if(res){
ret << res
}
@@ -87,111 +181,167 @@ def batchParse(def urls){
return ret;
}
def parse(url,doc){
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def parse(province, city, url,doc){
if(doc){
def result = [:]
// def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
// result.address = address
if(result.address){
def rMap = OpenApiUtils.matchDistrict(result.address)
if(rMap) {
result.matchCity = rMap.matchCity
result.matchRegion = rMap.matchRegion
result.matchName= rMap.matchName
result.matchAddress = rMap.matchAddress
result.province = province
result.city = city
def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
result.address = address
if(result.address && !result.address.contains("测试")) {
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
if(finishDateStr) {
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss");
}
result.status = JQueryUtils.selectText([".bd_state"], doc)
result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc)
result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc)
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",","")
result.tradePrice = curPriceStr.toInteger()
}
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
if (finishDateStr) {
try{
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
}catch(Exception e){
}
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
if(evaluationPriceStr) {
evaluationPriceStr = evaluationPriceStr.replace(",", "")
if(evaluationPriceStr.isInteger()) {
result.evaluationPrice = evaluationPriceStr.toInteger()
}
}
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
if(depositPriceStr) {
depositPriceStr = depositPriceStr.replace(",", "")
if(depositPriceStr.isInteger()) {
result.depositPrice = depositPriceStr.toInteger()
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
if (curPriceStr) {
curPriceStr = curPriceStr.replace(",", "")
result.tradePrice = curPriceStr.toDouble().toInteger()
}
}
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
if(addPriceStr) {
addPriceStr = addPriceStr.replace(",", "")
if(addPriceStr.isInteger()) {
result.addPrice = addPriceStr.toInteger()
}
}
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
if(startPriceStr) {
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}
}
result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc)
result.retain = JQueryUtils.selectText(["#retain"], doc)
def imgs =JQueryUtils.select(".sf-pic-slide img", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("src")
if(href.contains("http")) {
imgLinks << href
}else {
imgLinks << url.substring(0, url.indexOf(":")+1)+href
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
if (evaluationPriceStr) {
evaluationPriceStr = evaluationPriceStr.replace(",", "")
if (evaluationPriceStr.isNumber()) {
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
}
}
result.imgUrls = imgLinks
}
def desc = JQueryUtils.selectText(["#J_desc"], doc)
if(!desc){
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
}
if(desc) {
if(desc.contains("别墅")) {
result.address = desc.substring(0, desc.indexOf("别墅")+2)
}else if(desc.contains("号楼")) {
result.address = desc.substring(0, desc.indexOf("号楼")+2)
}else if(desc.contains("房屋")) {
result.address = desc.substring(0, desc.indexOf("房屋")+2)
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
if (depositPriceStr) {
depositPriceStr = depositPriceStr.replace(",", "")
if (depositPriceStr.isNumber()) {
result.depositPrice = depositPriceStr.toDouble().toInteger()
}
}
}
def descr = JQueryUtils.selectText(["body"], doc)
if(descr) {
def area = getAreaNumber(descr, 5000);
if(area){
result.area = area.toDouble()
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
if (addPriceStr) {
addPriceStr = addPriceStr.replace(",", "")
if (addPriceStr.isNumber()) {
result.addPrice = addPriceStr.toDouble().toInteger()
}
}
}
if(result.area && result.tradePrice){
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
}
result.content = DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
result.url = url
return result;
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
if (startPriceStr) {
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
if (startPriceStr.isNumber()) {
result.startPrice = startPriceStr.toDouble().toInteger()
}
}
result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc)
result.retain = JQueryUtils.selectText(["#retain"], doc)
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc)
if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
}
try{
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim())
}catch(Exception e){
}
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
if (imgs) {
def imgLinks = []
for (def img : imgs) {
def href = img.attr("src")
if (href.contains("http")) {
imgLinks << href
} else {
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
}
}
result.imgUrls = imgLinks
}
def desc = JQueryUtils.selectText(["#J_desc"], doc)
if (!desc) {
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
}
if (desc) {
if (desc.contains("别墅")) {
result.address = desc.substring(0, desc.indexOf("别墅") + 2)
} else if (desc.contains("号楼")) {
result.address = desc.substring(0, desc.indexOf("号楼") + 2)
} else if (desc.contains("房屋")) {
result.address = desc.substring(0, desc.indexOf("房屋") + 2)
}
}
def descr = JQueryUtils.selectText(["body"], doc)
if (descr) {
def area = getAreaNumber(descr, 5000);
if (area) {
result.area = area.toDouble()
}
}
if (result.area && result.tradePrice) {
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
if(result.startDate){
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
}
result.content = DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
result.url = url
return result;
}
}
return null
}
@@ -207,11 +357,13 @@ def saveToDB(rets){
ret.each{k,v->
obj.put(k,v)
}
obj.source= "中拍网"
if(dbo){
obj.remove("url")
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
paimaiResultColl.save(obj)
}
}
@@ -229,16 +381,25 @@ def getAreaNumber(input, maxArea) {
def area = null;
for (def val : list) {
if(val.toString().contains(" ")){
val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size())
}
if(val.contains("室")){
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
}
def v = val.replace("ƽ·½Ã×", "").replace("©O", "").replace("ƽ", "").replace("·½Ã×", "").replace(",", "").replace("m2", "");
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
if(v && v.toString().isNumber()){
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
break;
}
} else {
area = v
break;
}
} else {
area = v
break;
}
}
if (area) {
@@ -0,0 +1,311 @@
package com.fangjia.script.gj.paimai
import org.apache.commons.codec.binary.Base64
import org.apache.commons.lang.StringUtils
import sun.font.Font2D
import sun.font.Font2DHandle
import sun.font.TrueTypeFont
import sun.font.TrueTypeGlyphMapper
import java.awt.Font
import java.awt.font.FontRenderContext
import java.awt.font.GlyphVector
import java.awt.geom.AffineTransform
import java.lang.reflect.Field
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataUtil {
static def checkMinDate(curDate, checkDate){
if(curDate){
def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
if(curDate.getTime()<minDate.getTime()){
return null
}
}
return curDate
}
static def checkMaxDate(curDate, checkDate){
if(curDate){
def maxDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
if(curDate.getTime()>maxDate.getTime()){
return null
}
}
return curDate
}
static def getAreaNumber(def input, def maxArea) {
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)')
Matcher matcher = pattern.matcher(input)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() > 0) {
def area = null
for (def val : list) {
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "")
if (maxArea) {
if (v.isDouble()) {
if (v.toDouble() < maxArea) {
area = v
break
}
}
} else {
area = v
break
}
}
if (area) {
return area
}
}
return null
}
static def getPaiMaiType(def s) {
if (s) {
if (s.toString().contains("结束")) {
return "已完成"
} else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) {
return "进行中"
} else if (s.toString().contains("即将开始")) {
return "即将开始"
} else if (s.toString().contains("流拍")) {
return "流拍"
} else if (s.toString().contains("撤回")) {
return "撤回"
} else if (s.toString().contains("暂缓")) {
return "暂缓"
} else if (s.toString().contains("中止")) {
return "中止"
} else if (s.toString().contains("失败")) {
return "失败"
}
}
return s
}
static def getDateRange(def s) {
s = s.replace("起", "")
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
def str = matcher.group()
list.add(str)
// if(str) {
// def indx1 = str.indexOf("年")
// def indx2 = str.indexOf("月")
// def month = str.substring(indx1+1, indx2);
// if(month.toInteger()<12) {
// list.add(str);
// }
// }
}
if (list.size() >= 1) {
String s1 = list.get(0)
s1 = s1.replace("法院将于", "").replace(" ", "")
def array = s1.split("至")
return array
}
return null
}
static def getStartPrice(def s) {
Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() >= 1) {
String s1 = list.get(0)
if (s1.contains("万元")) {
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","")
return (int) (s1.toDouble() * 10000)
} else {
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","")
if (s1.isInteger()) {
return s1.toInteger()
} else {
return (int) (s1.toDouble())
}
}
}
return null
}
static def getEstateType(def s) {
Pattern pattern = Pattern.compile('房屋类型:.*;')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() >= 1) {
String s1 = list.get(0)
if (s1) {
s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";"))
return s1.replace(")", "").replace("】", "")
}
}
return null
}
static def findEstateType(def s) {
if (s) {
def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]
for (def type : villTypes) {
if (s.contains(type)) {
return "别墅"
}
}
def districtTypes = ["住宅", "公寓"]
for (def type : districtTypes) {
if (s.contains(type)) {
return "普通住宅"
}
}
}
return null
}
static def filterChar(def detailMap) {
def newDetailMap = [:]
detailMap.each { def k, v ->
if (k.toString().contains(".")) {
newDetailMap.put(k.toString().replace(".", "\\u002e"), v)
} else {
newDetailMap.put(k, v)
}
}
return newDetailMap
}
/**
* 匹配符合条件的字符串
*
* @param str
* @param pattern
* @return
*/
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
/**
* font-secret字符串专用解密工具
*
* @param key 密匙
* @param encodeString 加密后的字符串
* @return 解密后的字符串
*/
static String decodeString(String key, String encodeString) {
if (!key) {
return ""
}
try {
//base64解码,初始化字体
Base64 base64 = new Base64()
byte[] ss = base64.decodeBase64(key)
InputStream inputStream = new ByteArrayInputStream(ss)
Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream)
FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false)
GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "")
//获取font中字形的映射关系,字段为private,使用反射
Class<?> clazz = Font.class
Field[] fs = clazz.getDeclaredFields()
Font2DHandle font2DHandle = null
for (int i = 0; i < fs.length; i++) {
fs[i].setAccessible(true)// 将目标属性设置为可以访问
if (fs[i].getName().equals("font2DHandle")) {
font2DHandle = (Font2DHandle) fs[i].get(dynamicFont)
}
}
//得到映射关系
Font2D font2D = font2DHandle.font2D
TrueTypeFont trueTypeFont = (TrueTypeFont) font2D
TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper()
//开始解密,encodeString为加密后的字符串
StringBuffer buffer = new StringBuffer()
char[] chars = encodeString.toCharArray()
for (int i = 0; i < chars.length; i++) {
buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1)
}
return buffer.toString()
} catch (Exception e) {
e.printStackTrace()
}
return ""
}
static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"]
static String urlToSiteName(String url) {
if (StringUtils.isEmpty(url)) {
return null
}
for (String suffix : SITE_SUFFIXS) {
if (url.indexOf(suffix) > -1) {
String s = StringUtils.substringBefore(url, suffix)
if (s.indexOf(".") > -1) {
s = StringUtils.substringAfterLast(s, ".")
if (StringUtils.isNotBlank(s)) {
return s + suffix
}
} else if (s.indexOf("://") > -1) {
s = StringUtils.substringAfterLast(s, "://")
if (StringUtils.isNotBlank(s)) {
return s + suffix
}
} else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
return s + suffix
}
return null
}
}
return null
}
static double getRound(int scale, double dSource) {
BigDecimal deSource = new BigDecimal(dSource)
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue()
}
static def filterEmpty(def s){
if(s && s.contains("暂无")){
return null
}
return s
}
}
@@ -514,8 +514,9 @@ class GaoDeUtils {
// "2a1fb75990f2b39f6221e50892dd3988"
// ]
//35925efd7333f1df52145e51b8e4e56a 15021384995
def KeyList = [
"10bcb421d2ee4e51ed9268d6dca58ad1"
"10bcb421d2ee4e51ed9268d6dca58ad1"//17740818975
]
def count = RandomUtils.nextInt(KeyList.size())
return KeyList.get(count);
@@ -0,0 +1,348 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import javafx.scene.input.DataFormat
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class IcbcCrawl {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new IcbcCrawl();
task.run();
}
def run(){
def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver.get(url)
driver.findElementById("category_SM310000000000000006").click()
Thread.sleep(3000)
def urls = getAllDetailLinks(driver)
println "共:"+urls.size()
detailCrawling(urls, driver)
}
def getAllDetailLinks(driver){
def stop = false
def urls = []
while (!stop){
def doc = Jsoup.parse(driver.getPageSource());
def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc)
if(eles){
for(def e : eles){
def href = e.select(".li_2_name a").attr("href")
urls << "https://gf.trade.icbc.com.cn"+href
}
def nextPage = JQueryUtils.select("a:contains(下一页)", doc)
if(nextPage){
driver.findElement(By.linkText("下一页")).click();
}else{
stop = true
}
}else{
stop = true
}
// if(urls.size()>=10){
// stop = true
// }
}
return urls;
}
def detailCrawling(def urls, driver){
def rets = []
for(def url : urls){
driver.get(url)
Thread.sleep(5000)
def doc = Jsoup.parse(driver.getPageSource());
def result = parser(doc, url)
if (result) {
rets << result
println rets.size()
}
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
rets = []
}
}
if(rets.size()>0){
saveData(rets)
print "完"
}
}
def parser(doc, url){
def result = [:]
if(doc){
def title = JQueryUtils.selectText("#prodnameb", doc)
def id = url.substring(url.indexOf("="), url.length())
if(title){
def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc)
if(cityStr && cityStr.contains("...")){
def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc)
if(cityEles){
cityStr = cityEles[0].attr("title")
}
}
if(cityStr && cityStr.contains("省")){
result.province = cityStr.substring(0, cityStr.indexOf("省"))
result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length())
}else{
if(cityStr.contains("自治区")){
result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3)
result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length())
}else{
result.city = cityStr.substring(0, cityStr.indexOf("市"))
}
}
if(!result.city){
return null
}
result.title = title
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","")
def t1 = JQueryUtils.selectText("#pricep", doc)
if(t1 && t1.contains("原因")){
result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length())
}
result.type = JQueryUtils.selectText("dl:contains(类  型:) dd", doc)
def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc)
if(enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def lookCount = JQueryUtils.selectText(["#visitorCount"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
}
def callCount = JQueryUtils.selectText(["#remindPPNum"], doc)
if(callCount) {
result.callCount = callCount.toInteger()
}
def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim()
if(curPriceStr.isInteger()) {
result.curPrice = curPriceStr.toInteger()
}else {
result.curPrice = (int)(curPriceStr.toDouble())
}
}
def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc)
if(startPriceEles) {
def startPriceStr = startPriceEles[0].attr("title")
if(startPriceStr){
startPriceStr = startPriceStr.replace(",", "").replace("¥","")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}else {
result.startPrice = (int)(startPriceStr.toDouble())
}
}
}
def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
if(addPriceEles){
def addPriceStr = addPriceEles[0].attr("title")
if(addPriceStr){
addPriceStr = addPriceStr.replace(",", "").replace("¥","")
if(addPriceStr) {
result.addPrice = addPriceStr.toDouble().toInteger()
}
}
}
def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc)
if(evaluationPriceEles){
def evaluationPriceStr = evaluationPriceEles[0].attr("title")
if(evaluationPriceStr){
evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","")
if(evaluationPriceStr.isInteger()) {
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
}else {
result.evaluationPrice = (int)(evaluationPriceStr.toDouble())
}
}
}
def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
if(depositPriceEles){
def v = depositPriceEles[0].attr("title")
if(v){
v = v.replace(",", "").replace("¥","")
if(v.isInteger()) {
result.depositPrice = v.toDouble().toInteger()
}else {
result.depositPrice = (int)(v.toDouble())
}
}
}
result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc)
result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc)
result.court = JQueryUtils.selectText("#goToCourt", doc)
def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc)
if(mobileEles){
result.mobile = mobileEles[0].attr("title")
}
def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc)
if(s1) {
def start = s1.indexOf("法院将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
}
def dateRangs = PaimaiDataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
}
}
def trs = JQueryUtils.select([".rygtable table tr"], doc)
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
if(d_tds[2].text() && d_tds[2].text()!="/"){
detailMap.put(d_tds[1].text(), d_tds[2].text())
}
}else {
if(d_tds[1].text() && d_tds[1].text()!="/") {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}
}
}
if(detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if(!result.address) {
result.address = detailMap.get("拍品名称")
if(!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs =JQueryUtils.select("#scroll li img", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("src")
if(href.contains("icbc.com")) {
imgLinks << "https:"+href
}
}
result.imgUrls = imgLinks
}
def detailStr = JQueryUtils.selectText(".detail_list", doc)
if(detailStr){
def area = DataUtil.getAreaNumber(detailStr, 5000)
if(area) {
result.area = area.toDouble()
if(result.tradePrice) {
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
}
}
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = detailStr.toString()
result.url = url
result.mDate = new Date()
result.source = "融e够"
return result;
}
}
return null
}
def getCallCount(id){
def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id
def s = ABuYunUtil.doPost(url, null, null, "utf-8", true)
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
println jsonObject
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
}
@@ -0,0 +1,533 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
import javax.sound.midi.MidiSystem
import java.util.regex.Pattern
class JdPaimaiCrawl {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new JdPaimaiCrawl();
task.run();
}
def run(){
def existsIds = findAllIds()
println "已存在:"+existsIds.size()
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def links = getList(driver)
def ids = getDetailLinks(links, driver)
println "共:"+ids.size()
parserList( existsIds,ids, driver)
}
def getList(driver){
def list = []
for(def i=1;i<=32;i++){
if(i==8){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
}else if(i==5){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
}else if(i==17){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
}else if(i==19){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
}else{
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
}
}
def links = []
for(def url : list){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
//def document = ABuYunUtil.doPost(url, null, null, "utf-8", true)
if(doc){
def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")"))
JSONObject obj = JSONObject.fromObject(json);
if(obj && obj.total){
def total = obj.total
def page = total/40
def p = page.toInteger()
if(page > p){
p = p +1
}
for(def i=1;i<=p;i++){
links << url+"&page="+i
}
// if(links){
// break;
// }
}
}
}
return links;
}
def getDetailLinks(links, driver){
def rets = []
def citys = ["北京","上海","天津","重庆"]
for(def link : links){
driver.get(link)
def doc = Jsoup.parse(driver.getPageSource());
if(doc) {
def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")"))
JSONObject obj = JSONObject.fromObject(json);
if (obj && obj.ls) {
for(def o : obj.ls){
def map = [:]
if(citys.contains(o.province)){
map.province = o.province
map.city = o.province
map.region = o.city
}else{
map.province = o.province
map.city = o.city
}
map.id = o.id.toString()
rets << map
}
}
}
}
return rets
}
def parserList(existsIds,list, driver){
def rets = []
for(def map : list) {
if(!existsIds.contains(map.id)){
def url = "https://paimai.jd.com/" + map.id
def doc = getDocument(url, driver)
if (doc) {
def result = parsering(doc, url, map.province, map.city, map.region)
if (result) {
rets << result
}
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
rets = []
}
}
}
}
if(rets.size()>0){
saveData(rets)
print "完"
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
def parsering(doc, def url, def province, def city, def region){
def result = [:]
if(doc) {
result.proince = province
result.city = city
if(region){
result.region = region
}
def title = JQueryUtils.selectText([".pm-name"], doc)
result.title = title
result.address = JQueryUtils.selectText([".pm-location em"], doc)
if(result.address) {
//调用获取城市
// if(!city) {
// city = RegionData.formatCity(cityMap.name)
// }
if(city) {
def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
}
result.status = JQueryUtils.selectText([".pm-summary .state"], doc)
def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc)
if(enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
}
def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc)
if(callCount) {
result.callCount = callCount.toInteger()
}
def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "")
if(curPriceStr.isInteger()) {
result.curPrice = curPriceStr.toInteger()
}else {
result.curPrice = (int)(curPriceStr.toDouble())
}
}
def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc)
if(startPriceStr) {
startPriceStr = startPriceStr.replace(",", "")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}else {
result.startPrice = (int)(startPriceStr.toDouble())
}
}
def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc)
if(tradePriceStr) {
tradePriceStr = tradePriceStr.replace(",", "")
if(tradePriceStr.isInteger()) {
result.tradePrice = tradePriceStr.toInteger()
}else {
result.tradePrice = (int)(tradePriceStr.toDouble())
}
}
def finishDateStr = JQueryUtils.selectText([".endtime"], doc)
if(finishDateStr && finishDateStr.contains("结束时间")) {
finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" "))
result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss")
}
def lis = JQueryUtils.select(".description ul li", doc)
if(lis && lis.size()) {
for(def li : lis) {
def str = li.select("span").text()
if(str.contains("起拍")) {
def v= li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
if(v.isInteger()) {
result.startPrice = v.toDouble().toInteger()
}else {
result.startPrice = (int)(v.toDouble())
}
}
}else if(str.contains("评估价")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v.isInteger()) {
result.evaluationPrice = v.toDouble().toInteger()
}else {
result.evaluationPrice = (int)(v.toDouble())
}
}else if(str.contains("加价幅度")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
result.addPrice = v.toDouble().toInteger()
}
}else if(str.contains("保证金")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
if(v.isInteger()) {
result.depositPrice = v.toDouble().toInteger()
}else {
result.depositPrice = (int)(v.toDouble())
}
}
}else if(str.contains("延时周期")) {
result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "")
}else if(str.contains("优先竞买权人")) {
result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "")
}
}
}
def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc)
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
detailMap.put(d_tds[1].text(), d_tds[2].text())
}else {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}else {
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
}
}
}
if(detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if(!result.address) {
result.address = detailMap.get("拍品名称")
if(!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("href")
if(href.contains("http")) {
imgLinks << href
}else {
imgLinks << url.substring(0, url.indexOf(":")+1)+href
}
}
result.imgUrls = imgLinks
}
}
//解析面积
def article = JQueryUtils.selectText([".floor-content"], doc)
if(article) {
def area = DataUtil.getAreaNumber(article, 5000)
if(!area) {
article = JQueryUtils.selectText(["body"], doc)
if(article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if(area) {
result.area = area.toDouble()
if(result.tradePrice) {
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
}
}
}
def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc)
if(s1) {
def start = s1.indexOf("法院将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
}
def dateRangs = DataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
for(def j=0;j<dateRangs.size();j++) {
def date = dateRangs[j]
def pattern = null
if(date.endsWith("日时")||date.endsWith("日")) {
pattern = "yyyy年MM月dd日"
}else {
def ind = date.indexOf("日")
if(date.indexOf("时")==-1 && date.toString().length()>ind+1) {
date = date +"时"
}
pattern = "yyyy年MM月dd日HH时"
}
if(j==0) {
try{
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
}catch(Exception e){
}
}else {
if(!result.finishDate) {
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
}
}
}
}
if(!result.startPrice) {
result.startPrice = DataUtil.getStartPrice(s1)
}
def estateType = DataUtil.findEstateType(s1)
if(estateType) {
result.matchEstateType = estateType
}
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
if(result.startDate){
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
result.url = url
result.city = city
result.mDate = new Date()
result.source = "京东"
return result;
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def findAllIds(){
def q = new BasicDBObject()
q.put("url", Pattern.compile(".*jd.com.*"))
def dbcursor = paimaiResultColl.find(q)
def ids = []
for(def d : dbcursor){
def url = d.url
if(url){
def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length())
ids << id
}
}
return ids;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
@@ -34,8 +35,9 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
task.run()
// task.run1()
// task.run1()
task.addDealRegion();
task.standardRegion()
}
def run1(){
@@ -94,22 +96,44 @@ public class LianjiaCrawlByChrome {
}
def run(){
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
// System.setProperty("http.nonProxyHosts", "localhost");
// WebDriver driver = new ChromeDriver()
// def cityMap = getAllCityUrls(driver)
// getLogin(driver)
// crawlData(driver, cityMap)
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
getLogin(driver)
crawlData(driver, cityMap)
}
def standardRegion(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
if(d.region){
def newRegion = RegionData.formatRegion(d.city, d.region)
if(newRegion && newRegion!=d.region){
def query = new BasicDBObject()
query.put("city", d.city)
query.put("region", d.region)
def data = new BasicDBObject()
data.put("region", newRegion)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
}
}
def addDealRegion(){
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["Á´¼ÒÔÚÏß","±´¿ÇÕÒ·¿"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
@@ -131,6 +155,7 @@ public class LianjiaCrawlByChrome {
def data = new BasicDBObject()
data.put("region", dist.region)
println m.city+ " "+m.name +" "+dist.region
def ds = DEAL_PRICE_COLL.count(query)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
@@ -162,7 +187,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -193,7 +218,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -202,6 +227,10 @@ public class LianjiaCrawlByChrome {
break;
}
}
if(page>=100){
stop = true;
break;
}
page++;
}else{
stop = true;
@@ -219,6 +248,9 @@ public class LianjiaCrawlByChrome {
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
@@ -388,10 +420,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["³¤´º","¼ªÁÖ"]
if(cityEles){
for(def e : cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if(!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.paimai
import java.text.SimpleDateFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
@@ -94,32 +95,32 @@ class PaimaiDataUtil {
}
public static def getDateRange(def s) {
s = s.replace("起","")
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
Matcher matcher = pattern.matcher(s);
List<String> list = new ArrayList();
while (matcher.find()) {
def str = matcher.group()
list.add(str);
// if(str) {
// def indx1 = str.indexOf("年")
// def indx2 = str.indexOf("月")
// def month = str.substring(indx1+1, indx2);
// if(month.toInteger()<12) {
// list.add(str);
// }
// }
}
if(list.size()>=1) {
String s1 = list.get(0)
s1 = s1.replace("法院将于", "").replace(" ","")
def array = s1.split("至");
return array
}
return null;
}
// public static def getDateRange(def s) {
// s = s.replace("起","")
// Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)');
// Matcher matcher = pattern.matcher(s);
//
// List<String> list = new ArrayList();
// while (matcher.find()) {
// def str = matcher.group()
// list.add(str);
//// if(str) {
//// def indx1 = str.indexOf("年")
//// def indx2 = str.indexOf("月")
//// def month = str.substring(indx1+1, indx2);
//// if(month.toInteger()<12) {
//// list.add(str);
//// }
//// }
// }
// if(list.size()>=1) {
// String s1 = list.get(0)
// s1 = s1.replace("法院将于", "").replace(" ","")
// def array = s1.split("至");
// return array
// }
// return null;
// }
public static def getStartPrice(def s) {
@@ -159,4 +160,30 @@ class PaimaiDataUtil {
return newDetailMap;
}
public static def getDateRange(def s) {
s = s.replace("起","")
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d+月\\d+日\\d+时)至(\\d+年\\d+月\\d+日\\d+时)');
Matcher matcher = pattern.matcher(s);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>=1) {
String s1 = list.get(0)
s1 = s1.replace("法院将于", "")
def array = s1.split("至");
return array
}
return null;
}
public static Date parseFromTightByPattern(String s, String pattern){
if (s == null || s.length() == 0){
return null
}
SimpleDateFormat format = new SimpleDateFormat(pattern);
return format.parse(s)
}
}
@@ -7,34 +7,174 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
import java.text.SimpleDateFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
import java.util.zip.GZIPOutputStream
import org.apache.commons.lang.StringUtils;
import java.text.SimpleDateFormat;
class PaimaiTest {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new PaimaiTest();
task.run();
}
def run(){
def url = "http://www.gpai.net/sf/item2.do?Web_Item_ID=23134"
def con = find(url);
parser(con, url);
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def url = "https://s.gpai.net/sf/search.do?at=376"
def provinces = getProvice(url, driver)
def cityUrls = getCity(provinces, driver)
getDetailUrls(cityUrls, driver)
}
def getDetailUrls(cityUrls, driver){
for(def cMap : cityUrls){
driver.get("https:"+cMap.url)
def doc = Jsoup.parse(driver.getPageSource());
def total = JQueryUtils.selectText("span:contains(共有拍品) label", doc)
def parser(s, url){
Document doc = Jsoup.parse(s);
if(total){
def totalPage = getTotalPage(total.toInteger(), 20)
def detailDatas = []
for(def i=1;i<=totalPage;i++){
driver.get("https:"+cMap.url+"&page="+i)
doc = Jsoup.parse(driver.getPageSource());
def detailEles = JQueryUtils.select(".block-wrap:contains(共有拍品) .filt-result-list li .item-tit", doc)
for(def ele : detailEles){
def dMap = [:]
dMap.proince = cMap.proince
dMap.city = cMap.city
dMap.url = ele.select("a").attr("href")
detailDatas << dMap
}
}
def results = parser(detailDatas, driver, cMap.proince, cMap.city)
saveData(results)
println cMap.proince+" "+cMap.city+" "+results.size()
}
}
}
def getTotalPage(def total, def pageSize){
if(total){
def mode = total.toInteger()/pageSize
if(mode > mode.toInteger()){
mode = mode.toInteger() +1
}
return mode;
}
return 0;
}
def getCity(datas, driver){
def citys = ["上海","北京","天津","重庆"]
def cityList = []
for(def data : datas){
driver.get("https:"+data.href)
def doc = Jsoup.parse(driver.getPageSource());
if(citys.contains(data.proince)){
def links = JQueryUtils.select(".condition .s-subtab a", doc)
for(def ele : links) {
def cityMap = [:]
cityMap.proince = data.proince
cityMap.city = data.proince
cityMap.region =ele.text()
cityMap.url = data.href
cityList << cityMap
}
}else{
def links = JQueryUtils.select(".condition .s-subtab a", doc)
for(def ele : links){
if(!["乌鲁木齐市","克拉玛依市","合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){
def cityMap = [:]
cityMap.proince = data.proince
cityMap.city = ele.text()
cityMap.url = ele.attr("href")
cityList << cityMap
}
}
}
}
return cityList
}
def getProvice(url, driver){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def datas = []
def keys = [ "天津","上海","重庆","北京","浙江","河北","陕西","山西","新疆","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","西藏","宁夏","甘肃","广东","安徽","广西","海南","四川","贵州","云南"]
if(doc){
def eles = JQueryUtils.select(".condition a", doc)
for(def e : eles){
def data = [:]
if(!keys.contains(e.text())){
data.proince = e.text()
data.href = e.attr("href")
datas << data
}
}
}
return datas
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def parser(detailDatas, driver, proince, city){
def results = []
for(def m : detailDatas){
// try{
driver.get("https:"+m.url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
def obj = parsering(doc, m.url)
if(obj){
obj.proince = proince
obj.city = city
results << obj
}
// }catch(Exception e){
// println m.url+" 解析有问题!"
// }
}
return results
}
def parsering(doc, url){
def result = [:]
if(doc) {
def title = JQueryUtils.selectText([".d-m-title"], doc)
if(!title) {
title = JQueryUtils.selectText([".DivItemName"], doc)
@@ -48,6 +188,9 @@ class PaimaiTest {
}
result.title = title
result.status = status
if(title.contains("北京市朝阳区东三环中路7号4号楼28层3201")){
println title
}
if(title) {
def newStr =title
@@ -76,7 +219,7 @@ class PaimaiTest {
if(curPriceStr.contains("成交价:")) {
def tradePriceStr = JQueryUtils.selectText([".d-m-price .price-red"], doc).replace("元","")
if(tradePriceStr) {
result.tradePrice = tradePriceStr.toInteger()
result.tradePrice = tradePriceStr.toDouble().toInteger()
}
}
}
@@ -85,23 +228,28 @@ class PaimaiTest {
curPriceStr = JQueryUtils.selectText(["#ItemArea .red"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "").replace("元","").replace("--","")
result.tradePrice = curPriceStr.toInteger()
result.tradePrice = curPriceStr.toDouble().toInteger()
}
}
def enrollCount = JQueryUtils.selectText([".peoples-infos span:eq(0) b"], doc)
if(enrollCount) {
def enrollCount = JQueryUtils.selectText([".peoples-infos span:contains(人报名) b"], doc)
if(enrollCount && enrollCount.isNumber()) {
result.enrollCount = enrollCount.toInteger()
}
def callCount = JQueryUtils.selectText([".peoples-infos span:eq(1) b"], doc)
if(callCount) {
def callCount = JQueryUtils.selectText([".peoples-infos span:contains(人设置提醒) b"], doc)
if(callCount && callCount.isNumber()) {
result.callCount = callCount.toInteger()
}
def lookCount = JQueryUtils.selectText([".peoples-infos span:eq(2) b"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
def lookCount = JQueryUtils.selectText([".peoples-infos span:contains(人围观) b"], doc)
if(lookCount && lookCount.isNumber()) {
try{
result.lookCount = lookCount.toInteger()
}catch(Exception e){
println lookCount
}
}
def finishDateStr = JQueryUtils.selectText(["p:contains(拍卖竞价确认时间:)"], doc)
@@ -140,7 +288,7 @@ class PaimaiTest {
}else if(str.contains("保证金")) {
def v = str.replace("保证金:", "").replace("元", "").replace(",", "").replace("--", "")
if(v) {
result.depositPrice = v.toInteger()
result.depositPrice = v.toDouble().toInteger()
}
}else if(str.contains("延时周期")) {
@@ -184,6 +332,47 @@ class PaimaiTest {
}
}
}
if(detailMap && !detailMap.get("拍品名称")){
detailMap = [:]
for(def tr : trs) {
if(tr.text().contains("拍品名称")){
def td1 = tr.select("td:contains(拍品名称)+td").text()
detailMap.put("拍品名称", td1)
}
if(tr.text().contains("权利来源")){
def td1 = tr.select("td:contains(权利来源)+td").text()
detailMap.put("权利来源", td1)
}
if(tr.text().contains("权证情况")){
def td1 = tr.select("td:contains(权证情况)+td").text()
detailMap.put("权证情况", td1)
}
if(tr.text().contains("拍品所有人")){
def td1 = tr.select("td:contains(拍品所有人)+td").text()
detailMap.put("拍品所有人", td1)
}
if(tr.text().contains("房屋情况")){
def td1 = tr.select("td:contains(房屋情况)+td").text()
detailMap.put("房屋情况", td1)
}
if(tr.text().contains("权利限制情况及瑕疵情况")){
def td1 = tr.select("td:contains(权利限制情况及瑕疵情况)+td").text()
detailMap.put("权利限制情况及瑕疵情况", td1)
}
if(tr.text().contains("拍品介绍")){
def td1 = tr.select("td:contains(拍品介绍)").text()
detailMap.put("拍品介绍", td1)
}
}
}
result.detailMap = detailMap
if(!result.address) {
result.address = detailMap.get("拍品名称")
@@ -248,17 +437,18 @@ class PaimaiTest {
}
}
result.content = DataUtil.zip(s)
result.content = DataUtil.zip(doc.toString())
def el = JQueryUtils.select([".d-block table:eq(1)"], doc)
if(!el) {
el = JQueryUtils.select([".d-title:contains(标的物介绍)+div"], doc)
}
result.content = DataUtil.zip(s)
// result.content = DataUtil.zip(s)
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
result.url = url
result.mDate = new Date();
return result;
}
def find(url){
@@ -374,4 +564,32 @@ class PaimaiTest {
}
return null;
}
def saveData(def results){
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
obj.source = "公拍网"
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
}
}
@@ -1,12 +1,4 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.gj.paimai.PaimaiDataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -14,66 +6,266 @@ import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.JavascriptExecutor
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.WebDriverWait
import org.openqa.selenium.support.ui.ExpectedCondition
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import com.fangjia.script.gj.OpenApiUtils
import java.util.concurrent.TimeUnit
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.core.util.DataUtil
import com.fangjia.script.crawl.JQueryUtils
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://www1.rmfysszc.gov.cn/projects.shtml?dh=3&gpstate=1&wsbm_slt=1"
WebDriver driver = new ChromeDriver()
driver.get(url);
driver.findElementById("1").click()
run(url, driver)
//driver.get(url);
//driver.findElementById("1").click()
//driver.findElementById("search_sub").click()
//
//def provinces = getAllProvinceValue(driver.getPageSource())
//for(def m : provinces){
// def city_setting = driver.findElementById("city")
// //${m.province_id}
// city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
//
////driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
////WebDriverWait wait = new WebDriverWait(driver,20)
////wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
// Thread.sleep(10000)
// def doc = Jsoup.parse(driver.getPageSource());
// def citys = getAllCity(doc)
// getDetailByCity(m.province, citys, driver)
//}
def city_setting = driver.findElementById("city")
city_setting.findElement(By.xpath("//option[@value='31']")).click()
driver.findElementById("search_sub").click()
//driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
//WebDriverWait wait = new WebDriverWait(driver,20)
//wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1"))))
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def links = getLinks(doc)
def run(url, driver){
def values = getAllCityRegionValue(url, driver)
def finishCity = ["北京市","天津市","张家口市","邯郸市","保定市","邢台市","石家庄市","唐山市","秦皇岛市"]
for(def m : values){
if(!finishCity.contains(m.city)){
driver.get(url);
driver.findElementById("1").click()
Thread.sleep(3000)
def pro_setting = driver.findElementById("city")
pro_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
Thread.sleep(3000)
def city_setting = driver.findElementById("city1")
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
Thread.sleep(4000)
def region_setting = driver.findElementById("city2")
region_setting.findElement(By.xpath("//option[@value='${m.region_id}']")).click()
driver.findElementById("search_sub").click()
Thread.sleep(10000)
getDetailByPage(m.province, m.city, m.region, driver)
}
def eles = driver.findElementsByClassName("pagecur1")
for(def ele : eles){
ele.click();
Thread.sleep(10000)
doc = Jsoup.parse(driver.getPageSource());
links.addAll(getLinks(doc))
}
def rets = []
for(def durl : links){
driver.get(durl);
doc = Jsoup.parse(driver.getPageSource());
def res = parse(durl, doc);
if(res){
rets << res
}
}
saveToDB(rets);
def getAllCityRegionValue(def url, def driver){
driver.get(url);
driver.findElementById("1").click()
driver.findElementById("search_sub").click()
def provinces = getAllProvinceValue(driver.getPageSource())
def values = []
for(def m : provinces){
def city_setting = driver.findElementById("city")
city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click()
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def citys = getAllCity(doc)
def cityRegions = getAllCityByRegion(m, citys, driver)
values.addAll(cityRegions)
//getDetailByCity(m.province, citys, driver)
}
return values
}
def getDetailByCity(province, citys, driver){
for(def m : citys){
def city_setting = driver.findElementById("city1")
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = getAllRegion(doc)
getDetailByRegion(province, m.city, regions, driver)
}
}
def getAllCityByRegion(provinceMap, citys, driver){
def datas = []
for(def m : citys){
def city_setting = driver.findElementById("city1")
city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click()
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = getAllRegion(provinceMap, m, doc)
datas.addAll(regions)
}
return datas
}
def getDetailByRegion(province, city, regions, driver){
def links = []
for(def r : regions){
def city_setting = driver.findElementById("city2")
city_setting.findElement(By.xpath("//option[@value='${r.region_id}']")).click()
driver.findElementById("search_sub").click()
Thread.sleep(10000)
def stop = false;
while(!stop){
def doc = Jsoup.parse(driver.getPageSource());
links.addAll(getLinks(r.region, doc))
def eles = JQueryUtils.select("a:contains(下一页)", doc)
if(eles){
driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
Thread.sleep(3000)
}else{
stop = true;
}
}
}
def datas = parseDatas(province, city, links, driver)
saveToDB(datas);
println r.region +" "+ links.size()
}
def getDetailByPage(province, city, region, driver){
def links = []
def stop = false;
while(!stop){
def doc = Jsoup.parse(driver.getPageSource());
links.addAll(getLinks(doc))
def eles = JQueryUtils.select("a:contains(下一页)", doc)
if(eles){
driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
Thread.sleep(3000)
}else{
stop = true;
}
}
def datas = parseDatas(province, city, region, links, driver)
println province +" "+city+" "+region +" "+ links.size()
saveToDB(datas);
}
def getAllProvinceValue(doc){
def optionsEls = JQueryUtils.select("#city option", doc)
def rets = []
for(def eles : optionsEls){
def val = eles.attr("value")
if(val!="" && val!="0"){
def province = eles.text()
rets << ["province":province, "province_id":val]
def parse(url, doc){
}
}
return rets
}
def getAllCity(doc){
def optionsEls = JQueryUtils.select("#city1 option", doc)
def rets = []
for(def eles : optionsEls){
def val = eles.attr("value")
if(val!=""){
def province = eles.text()
rets << ["city":province, "city_id":val]
}
}
return rets
}
def getAllRegion(provinceMap, cityMap,doc){
def optionsEls = JQueryUtils.select("#city2 option", doc)
def rets = []
for(def eles : optionsEls){
def val = eles.attr("value")
if(val!=""){
def region = eles.text()
def regionMap = [:]
regionMap.province = provinceMap.province
regionMap.province_id = provinceMap.province_id
regionMap.city = cityMap.city
regionMap.city_id = cityMap.city_id
regionMap.region = region
regionMap.region_id = val
rets << regionMap
}
}
return rets
}
def parseDatas(province, city, region,links, driver){
def datas = []
for(def url: links){
def doc = getDocument(url, driver)
def result = parse(city, region, url, doc)
if(result){
result.province = province
result.city = city
result.region = region
datas << result
}
}
return datas
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def parse(city, region, url, doc){
def result=[:]
if(doc) {
def title = JQueryUtils.selectText(["#Title"], doc)
result.title = title;
result.address = title
if(result.address){
def rMap = OpenApiUtils.matchDistrict("ÉϺ£",result.address)
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
if(rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
@@ -99,6 +291,7 @@ def parse(url, doc){
def enrollCount = JQueryUtils.selectText([".bmnumber"], doc)
if(enrollCount) {
enrollCount = enrollCount.replace("人报名","")
if(enrollCount.isInteger()){
result.enrollCount = enrollCount.toInteger()
}
@@ -107,6 +300,7 @@ def parse(url, doc){
def callCount = JQueryUtils.selectText([".sc"], doc)
if(callCount) {
callCount =callCount.replace("人收藏","")
if(callCount.isInteger()){
result.callCount = callCount.toInteger()
}
@@ -114,6 +308,7 @@ def parse(url, doc){
def lookCount = JQueryUtils.selectText([".hits"], doc)
if(lookCount) {
lookCount = lookCount.replace("次围观","")
if(lookCount.isInteger()){
result.lookCount = lookCount.toInteger()
}
@@ -123,8 +318,8 @@ def parse(url, doc){
def curPriceStr = JQueryUtils.selectText(["#price div:eq(0)"], doc)
if(curPriceStr && curPriceStr.contains("起拍价")) {
curPriceStr = curPriceStr.replace("起拍价", "").replace("¥","")
if(curPriceStr.contains("ÍòÔª")){
curPriceStr = curPriceStr.replace("ÍòÔª","")
if(curPriceStr.contains("万")){
curPriceStr = curPriceStr.replace("万元","").replace("万","")
if(curPriceStr.isInteger()){
result.startPrice = curPriceStr.toInteger()*10000
}else{
@@ -144,6 +339,44 @@ def parse(url, doc){
}
}
def tableEles = JQueryUtils.select("#bg1", doc)
if(tableEles){
def evaluatePriceStr = tableEles.select("td:contains(值:)").text()
if(evaluatePriceStr && evaluatePriceStr.contains(":")){
def arrs = evaluatePriceStr.split(":")
if(arrs.size()==2){
evaluatePriceStr = arrs[1]
}
}
def depositPriceStr = tableEles.select("td:contains(金:)").text()
if(depositPriceStr && depositPriceStr.contains(":")){
def arrs = depositPriceStr.split(":")
if(arrs.size()==2){
depositPriceStr = arrs[1]
}
}
if(depositPriceStr){
depositPriceStr = depositPriceStr.replace("万元","")
if(depositPriceStr.toString().isNumber()){
result.depositPrice = depositPriceStr.toDouble()*10000
}
}
def courtStr = tableEles.select("td:contains(院:)").text()
if(courtStr && courtStr.contains(":")){
result.court = courtStr.substring(courtStr.indexOf(":")+1, courtStr.size())
}
def personStr = tableEles.select("td:contains(人:)").text()
if(personStr && personStr.contains(":")){
result.person = personStr.substring(personStr.indexOf(":")+1, personStr.size())
}
def countStr = tableEles.select("td:contains(拍卖阶段:)").text().replace("拍卖阶段:","")
if(countStr && countStr.contains("第") && countStr.contains("次")){
result.count = countStr.substring(countStr.indexOf("第")+1, countStr.indexOf("次"))
}
}
def lis = JQueryUtils.select("#bg1 div:eq(0) table tr", doc)
if(lis && lis.size()) {
@@ -151,18 +384,23 @@ def parse(url, doc){
def str = li.select("span").text()
if(str.contains("评估值")) {
def v = li.select("span:eq(1)").text().replace("万元", "")
if(v.isInteger()) {
result.evaluationPrice = v.toInteger()*10000
}else{
result.evaluationPrice = (int)(v.toDouble()*10000)
if(v.isNumber()){
if(v.isInteger()) {
result.evaluationPrice = v.toInteger()*10000
}else{
result.evaluationPrice = (int)(v.toDouble()*10000)
}
}
}else if(str.contains("保证金")) {
def v = li.select("span:eq(1)").text().replace("万元", "")
if(v) {
if(v.isInteger()) {
result.depositPrice = v.toInteger()*10000
}else {
result.depositPrice = (int)(v.toDouble()*10000)
if(v.isNumber()) {
if (v) {
if (v.isInteger()) {
result.depositPrice = v.toInteger() * 10000
} else {
result.depositPrice = (int) (v.toDouble() * 10000)
}
}
}
}else if (str.contains("公告日期:")){
@@ -177,13 +415,32 @@ def parse(url, doc){
def trs = tables[1].select("tr")
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(tr.text().contains("调查情况表")){
continue
}
if(d_tds.size()>1){
def lastEleStr = d_tds[d_tds.size()-1].toString()
if(lastEleStr.contains('height="42"')){
d_tds.remove(d_tds[d_tds.size()-1])
}
}
if(d_tds.size()>3){
continue
}
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
detailMap.put(d_tds[1].text(), d_tds[2].text())
if(d_tds[1].text() && d_tds[2].text()){
detailMap.put(d_tds[1].text(), d_tds[2].text())
}
}else {
detailMap.put(d_tds[0].text(), d_tds[1].text())
if(d_tds[0].text() && d_tds[1].text()){
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}
}else {
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
@@ -202,8 +459,6 @@ def parse(url, doc){
result.introduce = tables[1].toString()
}
def imgs =JQueryUtils.select("#bdjs11 img", doc)
if(imgs) {
def imgLinks = []
@@ -232,6 +487,20 @@ def parse(url, doc){
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
}
def s1 = JQueryUtils.selectText(["#pmgg"], doc)
if(s1) {
def start = s1.indexOf("法院将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
}
def dateRangs = PaimaiDataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
}
}
result.content = DataUtil.zip(doc.toString())
result.url = url
@@ -246,7 +515,7 @@ def getLinks(doc){
if(eles){
for(def el :eles){
def href = el.attr("href")
urls << "https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
urls <<"https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length())
}
}
}
@@ -264,13 +533,19 @@ def saveToDB(rets){
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.remove("url")
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
paimaiResultColl.save(obj)
}
obj.source= "人民法院诉讼资产网"
try{
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
paimaiResultColl.save(obj)
}
}catch(Exception e){
println "error:"+obj.url
}
}
}
@@ -287,14 +562,16 @@ def getAreaNumber(input, maxArea) {
for (def val : list) {
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "");
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
if(v && v.isNumber()){
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
break;
}
} else {
area = v
break;
}
} else {
area = v
break;
}
}
@@ -1,7 +1,12 @@
package com.fangjia.script.gj.paimai
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.DateUtils
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class TaobaoPaimaiCrawl {
static main(args) {
@@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl {
task.run();
}
def run(){
def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1"
def regionUrls = regionCrawl(seedUrl);
def pageUrls = pageUrlCrawl(regionUrls)
detailCrawl(pageUrls)
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1"
def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1"
WebDriver driver = new ChromeDriver()
getAllCityRegions(driver, url);
}
def regionCrawl(def seedUrl){
def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk");
def regionUrls = []
if(document){
def eles = JQueryUtils.select(".small-subcondion a", document)
if(eles){
for(def el : eles){
def reiongUrl = el.attr("href").trim()
if(reiongUrl.startsWith("//")){
regionUrls << "http:"+reiongUrl
def getAllCityRegions(driver, url){
def provinces = getAllProvince(driver, url)
def citys = ["天津","上海","北京","重庆"]
for(def province : provinces){
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
def doc = Jsoup.parse(driver.getPageSource());
def cityEles= JQueryUtils.select(".unfold .J_FilterCity a")
for(def cityEle : cityEles){
def city = cityEle.text()
if(citys.contains(city)){
driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click()
doc = Jsoup.parse(driver.getPageSource());
driver.findElement(By.cssSelector(".J_FilterCity a")).click()
doc = Jsoup.parse(driver.getPageSource());
def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a")
for(def r : regionEles){
def m = [:]
m.province = province
m.city = city
m.region = r.text()
def urls = crawlLinks(m, driver)
parse(m, driver, urls)
}
}else{
def m = [:]
m.province = province
m.city = city
def urls = crawlLinks(m, driver)
parse(m, driver, urls)
}
}
}
return regionUrls;
}
def pageUrlCrawl(regionUrls){
def crawlLinks(def m, def driver){
def stop = false
def urls = []
for(def url : regionUrls){
def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk");
if(document){
def pages = JQueryUtils.selectText(".page-total",document)
if(pages){
pages = pages.toInteger()
for(def page=1;page<pages;page++){
urls << url+"&page="+page
while(!stop){
def doc = Jsoup.parse(driver.getPageSource());
def details = JQueryUtils.select(".sf-pai-item-list li a", doc)
for(def d : details){
urls << "https:"+d.attr("href")
}
def nextEle = JQueryUtils.select(".next", doc)
if(nextEle){
driver.findElement(By.cssSelector(".next")).click()
}
}
return urls
}
def getAllProvince(driver, url){
driver.get(url);
Thread.sleep(100000)
def doc = Jsoup.parse(driver.getPageSource());
def eles= JQueryUtils.select("li:contains(标的物类型) .condition li a", doc)
def provinces = []
for(def e : eles){
def province = e.text()
provinces << province
}
return provinces
}
def parse(m, driver, urls){
def results = []
for(def url : urls) {
driver.get(url);
def doc = Jsoup.parse(driver.getPageSource());
def result = [:]
if (doc) {
def title = JQueryUtils.selectText([".pm-main h1"], doc)
def statusDesc = JQueryUtils.selectText([".bid-fail"], doc)
result.province = m.province
result.city = m.city
if (m.region) {
result.region = m.region
}
if (title) {
def addr = title.toString().replace("本场拍卖已", "").replace(" !", "")
if (addr && addr.indexOf("室") > -1) {
result.address = addr.substring(0, addr.indexOf("室") + 1)
} else {
result.address = addr
}
if (result.city) {
if (result.address.contains(city + "市")) {
result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1)
} else {
result.address = result.address.substring(result.address.indexOf(city) + city.length())
}
}
if (result.address.indexOf("】") > -1) {
result.address = result.address.substring(result.address.indexOf("】") + 1)
}
if (result.city) {
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
}
if (statusDesc) {
def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1)
result.status = DataUtil.getPaiMaiType(status)
result.statusDesc = statusDesc
}
def statusEle = JQueryUtils.select(".status-tip", doc)
if (statusEle) {
def sEle = JQueryUtils.select(".status-ing", doc)
if (sEle) {
result.status = "进行中"
} else {
result.status = "即将开始"
}
}
def curPrice = JQueryUtils.selectText(["#sf-price"], doc)
if (curPrice) {
if (curPrice.contains("当前价")) {
curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (curPrice) {
curPrice = curPrice.replace(",", "")
if (curPrice.isInteger()) {
result.curPrice = curPrice.toInteger()
} else {
result.curPrice = (int) (curPrice.toDouble())
}
}
} else if (curPrice.contains("成交价")) {
def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (tradePrice) {
tradePrice = tradePrice.replace(",", "")
if (tradePrice.isInteger()) {
result.tradePrice = tradePrice.toInteger()
} else {
result.tradePrice = (int) (tradePrice.toDouble())
}
}
} else if (curPrice.contains("起拍价")) {
def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc)
if (startPrice) {
startPrice = startPrice.replace(",", "")
if (startPrice.isInteger()) {
result.startPrice = startPrice.toInteger()
} else {
result.startPrice = (int) (startPrice.toDouble())
}
}
}
}
def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc)
if (finishStr) {
if (finishStr.contains("结束时间")) {
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
if (finishStr) {
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
}
} else if (finishStr.contains("成交时间")) {
finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text()
if (finishStr) {
result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss")
}
}
}
def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc)
if (enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc)
if (callCount) {
result.callCount = callCount.toInteger()
}
def lookCount = JQueryUtils.selectText(["#J_Looker"], doc)
if (lookCount) {
result.lookCount = lookCount.toInteger()
}
def tds = JQueryUtils.select(["#J_HoverShow td"], doc)
if (tds && tds.size() > 0) {
for (def td : tds) {
def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "")
if (str) {
if (str.contains("变卖价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.sellOffPrice = p.toInteger()
} else {
result.sellOffPrice = (int) (p.toDouble())
}
}
} else if (str.contains("加价幅度")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.addPrice = p.toInteger()
} else {
result.addPrice = (int) (p.toDouble())
}
}
} else if (str.contains("类型")) {
result.type = td.select(".pay-type").text()
} else if (str.contains("保证金")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.depositPrice = p.toInteger()
} else {
result.depositPrice = (int) (p.toDouble())
}
}
} else if (str.contains("变卖周期")) {
result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("竞价周期")) {
result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("优先购买权")) {
result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("变卖预缴款")) {
def str1 = td.select("span:eq(1) .J_Price").text()
def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.sellOffAdvancePay = p.toInteger()
} else {
result.sellOffAdvancePay = (int) (p.toDouble())
}
}
} else if (str.contains("评估价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.evaluationPrice = p.toInteger()
} else {
result.evaluationPrice = (int) (p.toDouble())
}
}
} else if (str.contains("延时周期")) {
result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "")
} else if (str.contains("起拍价")) {
def str1 = td.select(".J_Price").text()
def str2 = td.select(".J_Price .J_Price-tip").text()
def p = str1.replace(str2, "").replace(",", "")
if (p) {
if (p.isInteger()) {
result.startPrice = p.toInteger()
} else {
result.startPrice = (int) (p.toDouble())
}
}
}
}
}
}
result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc)
def trs = JQueryUtils.select(["#J_desc table tr"], doc)
if (trs) {
def detailMap = [:]
for (def tr : trs) {
def d_tds = tr.select("td")
if (d_tds && d_tds.size() >= 2) {
if (d_tds.size() == 3) {
detailMap.put(d_tds[1].text(), d_tds[2].text())
} else {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
} else {
if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
}
}
}
if (detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if (!result.address) {
result.address = detailMap.get("拍品名称")
if (!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs = JQueryUtils.select([".slide-bigpic img"], doc)
if (imgs) {
def imgLinks = []
for (def img : imgs) {
def href = img.attr("src")
if (href.contains("http")) {
imgLinks << href
} else {
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
}
}
result.imgUrls = imgLinks
}
}
//面积
def article = JQueryUtils.selectText(["#J_ItemNotice"], doc)
if (article) {
def area = DataUtil.getAreaNumber(article, 5000)
if (!area) {
article = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
if (article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if (!area) {
article = JQueryUtils.selectText(["body"], doc)
if (article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if (area) {
result.area = area.toDouble()
if (result.tradePrice) {
result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble())
}
}
}
}
return urls;
}
def detailCrawl(pageUrls){
if(pageUrls){
for(def pageUrl : pageUrls){
def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk");
if(document){
def pages = JQueryUtils.select(".page-total", document)
def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc)
if (s1) {
def dateRangs = DataUtil.getDateRange(s1)
if (dateRangs) {
for (def j = 0; j < dateRangs.size(); j++) {
def date = dateRangs[j]
def pattern = null
if (date.endsWith("日时") || date.endsWith("日")) {
pattern = "yyyy年MM月dd日"
} else {
def ind = date.indexOf("日")
if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) {
date = date + "时"
}
pattern = "yyyy年MM月dd日HH时"
}
if (j == 0) {
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
} else {
if (!result.finishDate) {
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
}
}
}
}
if (!result.startPrice) {
result.startPrice = DataUtil.getStartPrice(s1)
}
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
def estateType = DataUtil.findEstateType(s)
if (estateType) {
result.matchEstateType = estateType
}
result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString()
result.url = url
result.mDate = new Date()
results << result
}
return results;
}
@@ -5,10 +5,15 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
class BlankEvaluationBatch {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
class EvaluationBatch {
static main(args) {
def task = new EvaluationBatch();
def task = new BlankEvaluationBatch();
task.run()
}
def run(){
@@ -0,0 +1,85 @@
package com.fangjia.script.gj.price
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import groovy.sql.Sql
class FzMatchTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new FzMatchTest();
// task.run()
task.caluatePPE()
}
def caluatePPE(){
def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区_结果1).xlsx"
ExcelUtils.multiThreadingModify(filePath, { data ->
if(data['F'] && data['G']){
def dist = CommonDatautils.findDistrict("福州", data['F'], data['G'], "district")
if(dist){
def avg_price = findAvgPrice(dist._id.toString())
if(data['i'] && avg_price){
def weekendPrice = data['i'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
data['J'] = ppe15
}
}
}
},1)
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
def run(){
def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区).xlsx"
def city = "福州"
ExcelUtils.multiThreadingModify(filePath, { data ->
// if(data['G']){
// if(!data['H']){
// def dist = CommonDatautils.findDistrict(city, data['F'], data['G'], "district")
//
// if(dist && dist.finish_building_date){
// data['H'] = dist.finish_building_date.substring(0,4)
// }
// }
// if(!data['I']){
// def price = CommonDatautils.queryDistrictPrice(city, data['F'], data['G'])
// if(price){
// data['I'] = price
// }
// }
//
// }
if(!data['G'] && data['B']){
def name = data['B'].replace("(", "").replace(")","").replace("\\","").replace("#","")
def rMap = SeaweedDataMatchProcess.doMatch(city, null, name, null);
if(rMap) {
data['F'] = rMap.region
data['G'] = rMap.name
def dist = CommonDatautils.findDistrictById(rMap.id)
if(dist && dist.finish_building_date){
data['H'] = dist.finish_building_date.substring(0,4)
}
def price = CommonDatautils.queryDistrictPrice(city, rMap.region, rMap.name)
if(price){
data['I'] = price
}
}
}
}, 1);
}
}
@@ -37,7 +37,10 @@ class CrawlDistrictIdSelect {
}
def run(){
def datas = findCrawlData()
def source = "58同城"
def root = "58.com"
def shortCityMap = findCitySeeds(root)
def datas = findCrawlData(source, shortCityMap, root)
def list = []
datas.each {k,v->
if(v){
@@ -58,15 +61,32 @@ class CrawlDistrictIdSelect {
}
}
def findCrawlData(){
def findCitySeeds(root){
def q = new BasicDBObject()
q.put("root", root)
q.put("city", "福州")
def dbcursor = districtseedsColl.find(q)
def cityMap = [:]
for(def d : dbcursor){
if(!cityMap.keySet().contains(d.city)){
cityMap.put(d.city, d.city_short)
}
}
return cityMap
}
def findCrawlData(def source, shortCityMap, def root){
def q = new BasicDBObject()
q.put("city","¸£ÖÝ")
q.put("source", "³ÇÊз¿²ú")
q.put("s_date",202012)
q.put("source", source)
q.put("s_date",202112)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = ".cityhouse.cn/ha/"
def prefix = "xiaoqu/"
def keyList =new HashSet()
def index = 0
for(def db : dbcursor){
def url = null
if(!db.url){
@@ -76,33 +96,42 @@ class CrawlDistrictIdSelect {
}else{
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
if(url){
def district_id = ""
if(url.contains(prefix)){
district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1)
}else{
district_id = url;
}
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
def ids = retMap.get(db.city)
if(!ids){
ids = []
retMap.put(key, ids)
retMap.put(db.city, ids)
}
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "cityhouse.com"
map.city_short= shortCity
map.city = db.city
map.region = db.region
map.district_id = district_id
map.source_url = url
ids << map
keyList << key
if(!keyList.contains(db.city+"#"+district_id)){
def count = findIsExists(db.city, district_id, root)
if(!count){
def map = new BasicDBObject()
map.root = "58.com"
map.city_short= shortCityMap.get(db.city)
map.city = db.city
map.region = db.region
map.district_id = district_id
map.source_url = url
map.name = db.name
ids << map
}else{
index++
println "[${index}]已存在:"+db.city +" "+db.name
}
keyList << db.city+"#"+district_id
}else{
println "ÖØ¸´£º"+key
println "重复:"+db.city+"#"+db.name
}
}else{
println url +" Á´½Ó²»ºÏÒªÇó"
@@ -110,4 +139,13 @@ class CrawlDistrictIdSelect {
}
return retMap
}
def findIsExists(city, district_id, root){
def q = new BasicDBObject()
q.put("city",city)
q.put("district_id", district_id)
q.put("root",root)
def count = districtseedsColl.count(q)
return count
}
}
@@ -68,7 +68,7 @@ class FZloationUpdate {
Long lng0 = lng*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[ÐÞ¸Ä-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
}
@@ -153,7 +153,7 @@ class FZloationUpdate {
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(dist.city, addr);
if(reps && reps.count.toInteger()>=1){
if(reps && reps.count && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
@@ -42,6 +42,55 @@ class SFTP {
return sftp;
}
/**
* 删除本地文件
*
* @param filePath
* @return
*/
public boolean deleteFile(String filePath) {
File file = new File(filePath);
if (!file.exists()) {
return false;
}
if (!file.isFile()) {
file.deleteDir();
return false;
}
boolean rs = file.delete();
return rs;
}
public static void deleteSFTP(String directory, ChannelSftp sftp) {
try {
if (isDirExist(directory, sftp)) {
Vector<ChannelSftp.LsEntry> vector = sftp.ls(directory);
if (vector.size() == 1) { // 文件,直接删除
sftp.rm(directory);
} else if (vector.size() == 2) { // 空文件夹,直接删除
sftp.rmdir(directory);
} else {
String fileName = "";
// 删除文件夹下所有文件
for (ChannelSftp.LsEntry en : vector) {
fileName = en.getFilename();
if (".".equals(fileName) || "..".equals(fileName)) {
continue;
} else {
deleteSFTP(directory + "/" + fileName);
}
}
// 删除文件夹
sftp.rmdir(directory);
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 上传文件
* @param directory 上传的目录
@@ -132,7 +181,7 @@ class SFTP {
sftp.cd(filePath.toString());
} else {
// 建立目录
sftp.mkdir(filePath.toString());
sftp.mkdir(filePath.toString())
// 进入并设置为当前目录
sftp.cd(filePath.toString());
}
@@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch {
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "G00jia123456");
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
email.addTo(mail.receiverEmail);
email.setFrom("service@fangjia.com");//·¢ËÍ·½
email.setSubject(mail.subject); //±êÌâ
@@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch {
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "G00jia123456");
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
email.addTo(mail.receiverEmail);
email.setFrom("service@fangjia.com");
email.setSubject(mail.subject);
@@ -0,0 +1,205 @@
package com.fangjia.script.loudong
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class CqfdcChromeCrawl {
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
static main(args) {
def task = new CqfdcChromeCrawl();
task.run()
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def list = getListDatas(driver)
getDetails(driver, list);
}
def getDetails(driver, list){
def index = 0
for(def map_old : list){
def datas = []
index++
def url = "http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=${map_old.building_id}&block=${map_old.house_num_desc}"
// driver.get(url)
// def doc = driver.getPageSource()
def cityDoc = getDocment(url, driver)
def trs = JQueryUtils.select(".building_table tr", cityDoc)
def unit1 = null
def unit2 = null
def unit3 = null
def unit1_text = null
def unit2_text = null
def unit3_text = null
for(def i=0;i<trs.size();i++){
if(i==1){
continue
}
def tr = trs.get(i)
def tds = tr.select("td")
for(def j=2;j< tds.size();j++){
def td = tds[j]
if(i==0){
def text = td.text()
if(!unit1){
if(text && text.contains("单元")){
unit1 = td.attr("colspan").toInteger()
unit1_text = text.replace("单元","")
}
}else if(unit1 && !unit2){
if(text && text.contains("单元")) {
unit2 = td.attr("colspan").toInteger()
unit2_text = text.replace("单元", "")
}
}else if(unit1 && !unit3){
if(text && text.contains("单元")) {
unit3 = td.attr("colspan").toInteger()
unit3_text = text.replace("单元", "")
}
}
}else{
def room_num = td.select("a font").text()
if(room_num){
def map = [:]
def a = td.select("a")
map.use_type = td.select("span:contains(用途:)").text().replace("用途:","")
map.room_type = td.select("span:contains(户型:)").text().replace("户型:","")
map.structure_type = td.select("span:contains(结构:)").text().replace("结构:","")
map.area = td.select("span:contains(建面:)").text().replace("建面:","")
map.siji_area = td.select("span:contains(套内:)").text().replace("套内:","")
map.price = td.select("span:contains(建面拟售价格:)").text().replace("建面拟售价格:","")
map.siji_price = td.select("span:contains(套内拟售价格:)").text().replace("套内拟售价格:","")
map.fitment = td.select("span:contains(装修水平:)").text().replace("装修水平:","").replace("null","")
def physical_floor = tds[0]//物理层
def nominal_floor = tds[1]//名义层
map.nominal_floor = nominal_floor.text()
map.physical_floor = physical_floor.text()
if(unit1 && (j+1 <= unit1+2)){
map.unit_num = unit1_text
}else if(unit2 && (j+1 <= unit1+unit2+2)){
map.unit_num = unit2_text
}else if(unit3 && (j+1 <= unit1+unit2+unit3+2)){
map.unit_num = unit3_text
}
map.room_num = room_num
map_old.each{k,v->
map.put(k,v)
}
datas << map
}
}
}
}
saveDatas(datas)
println index+" "+map_old.name+" "+map_old.house_num_desc +" "+datas.size()
}
}
def getDocment(def url, def driver){
def stop = false
def doc = null
def error = 0
while (!stop || error>=10){
try{
driver.get(url)
def docment = driver.getPageSource()
if(docment){
doc = Jsoup.parse(docment)
stop = true;
}else{
error ++
}
}catch(Exception e){
error ++
}
}
return doc;
}
def getListDatas(driver){
driver.get("http://cqfdc.cq315house.com/HtmlPage/PresaleDetail.html")
def datas = []
for(def i=2;i<=208;i++){
def docment = Jsoup.parse(driver.getPageSource());
def trs = JQueryUtils.select("#searchbody tr", docment)
for(def tr : trs){
def tds = tr.select("td")
def hrefs = tds.get(3).select("a")
for(def href : hrefs){
def map = [:]
map.house_num_desc = href.text()
def str = href.attr("href")
def building_id= str.substring(str.lastIndexOf("['")+2, str.lastIndexOf("','"))
map.building_id = building_id
map.city = "重庆"
map.name = tds.get(0).text()
map.company = tds.get(1).text()
map.license_key = tds.get(2).text()
datas << map
}
}
println i +" "+datas.size()
try{
driver.findElement(By.id("nextpage")).click()
}catch(Exception e){
Thread.sleep(8000)
driver.findElement(By.id("nextpage")).click()
}
}
println "列表:"+datas.size()
return datas;
}
def saveDatas(def datas){
def rows = []
def keys = []
for(def d : datas){
def q = new BasicDBObject()
q.put("city", d.city)
q.put("name",d.name)
q.put("house_num_desc", d.house_num_desc)
q.put("room_num", d.room_num)
q.put("unit_num",d.unit_num)
def obj = loudongConn.findOne(q)
if(!obj){
def key = d.house_num_desc +"#"+d.unit_num+"#"+d.room_num
if(!keys.contains(key)){
def udata = new BasicDBObject()
d.each {k,v->
udata.put(k,v)
}
udata.put("c_date", new Date())
rows << udata
keys << key
}
}
}
if(rows){
loudongConn.insert(rows);
println "save: "+rows.size()
}
}
}
@@ -0,0 +1,299 @@
package com.fangjia.script.loudong
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class CqfdcMatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new CqfdcMatch();
// task.run()
// task.stat()
// task.export2()
// task.matchLoudong()
task.addLoudong()
}
def export(){
def q = new BasicDBObject()
q.put("match_city", new BasicDBObject('$exists', false))
def field = new BasicDBObject()
field.put("city",1)
field.put("region",1)
field.put("district_name",1)
field.put("address",1)
field.put("type",1)
def dbcursor = loudongConn.find(q, field).batchSize(5000)
def keys = []
def rets = []
rets << ["城市","区域","名称","地址","类型"]
for(def d : dbcursor){
def key = d.city +"##"+d.district_name
if(!keys.contains(key)){
keys << key
rets << [d.city, d.region, d.district_name, d.address, d.type]
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配1.xlsx",rets, "数据")
}
def run(){
def datas= findAddress2()
def results = match(datas);
update(results)
}
def export2(){
def q = new BasicDBObject()
q.put("match_city", new BasicDBObject('$exists', false))
def field = new BasicDBObject()
field.put("city",1)
field.put("name",1)
field.put("use_type",1)
def dbcursor = loudongFdcConn.find(q, field).batchSize(5000)
def keys = []
def rets = []
rets << ["城市","地址","类型"]
for(def d : dbcursor){
def key = d.city +"##"+d.name
if(!keys.contains(key)){
keys << key
rets << [d.city, d.name,d.use_type]
}
}
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配2.xlsx",rets, "数据")
}
def update(results){
for(def m : results){
def q = new BasicDBObject()
q.put("city", m.old_city)
q.put("district_name", m.old_name)
q.put("region",m.old_region)
def udata = new BasicDBObject()
udata.put("match_city", m.match_city)
udata.put("match_region", m.match_region)
udata.put("match_name", m.match_name)
loudongConn.update(q, new BasicDBObject('$set',udata),false, true)
}
}
def match(def datas){
def rets = []
for(def d : datas){
def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
def map = [:]
map.match_city = jsonObject.get("city")
map.match_region = jsonObject.get("district")
map.match_name = jsonObject.get("name")
map.old_name = d.name
map.old_city = d.city
map.old_region = d.region
rets << map
}
}
return rets;
}
def findAddress(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q)
def rets = []
def keys = []
for(def d : dbcursor){
if(!keys.contains(d.name)){
keys << d.name
def map = [:]
map.city = d.city
map.name = d.name
rets << map
}
}
return rets;
}
def findAddress2(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000)
def rets = []
def keys = []
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.district_name
if(!keys.contains(key)){
keys << key
def map = [:]
map.city = d.city
map.region = d.region
map.name = d.district_name
rets << map
}
}
return rets;
}
def findAllLoundongByHouseNum(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000)
def rets = []
def map = [:]
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num
if(d.unit_num){
key = key +"##"+d.unit_num
}
def detailMap= map.get(key)
if(!detailMap){
detailMap = [:]
map.put(key,detailMap)
}
if(detailMap.height){
}
}
return rets;
}
def stat(){
def q = new BasicDBObject()
q.put("match_city", new BasicDBObject('$exists', true))
def field = new BasicDBObject()
field.put("match_city",1)
field.put("match_region",1)
field.put("match_name",1)
def dbcursor = loudongConn.find(q, field).batchSize(2000)
def keys = []
for(def d : dbcursor){
def key = d.match_city+"##"+d.match_region+"##"+d.match_name
if(!keys.contains(key)){
keys << key
}
}
def dbcursor2 = loudongFdcConn.find(q, field).batchSize(5000)
for(def d : dbcursor2){
def key = d.match_city+"##"+d.match_region+"##"+d.match_name
if(!keys.contains(key)){
keys << key
}
}
println "小区:"+keys.size()
}
def matchLoudong(){
def names = findAllLoudong("重庆")
print names.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(names.size())
def retMap = new ConcurrentHashMap()
for(def i=0;i<names.size();i++){
def key = names[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def arrs = key.split("#")
def jsonObject = OpenApiClient.addressMatch(arrs[0], arrs[2], "property", false, arrs[1], null)
if(jsonObject != null && jsonObject.precise!=-1) {
def m = [:]
m.id = jsonObject.get("id")
m.city = jsonObject.get("city")
m.region = jsonObject.get("district")
m.name = jsonObject.get("name")
def dist = CommonDatautils.findDistrictById(m.id)
if(dist){
m.sid = dist.sid
}
retMap.put(key, m)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
println "匹配后:"+retMap.size()
updateLoudong(retMap)
println "修改完成!"
}
def updateLoudong(retMap){
def index = 0
retMap.each{key,m->
def arrs = key.split("#")
def q = new BasicDBObject("city", arrs[0])
q.put('region', arrs[1]);
q.put('name', arrs[2]);
def udata = new BasicDBObject()
udata.put("district_id", m.id.toString())
udata.put("name_in_seaweed", m.name)
udata.put("district_no", m.sid)
HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true);
index++
println index
}
}
def findAllLoudong(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("district_id",new BasicDBObject('$exists', false))
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("name", 1)
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
def names = new CopyOnWriteArrayList()
for(def db : dbcursor){
def key = db.city +"#"+db.region+"#"+db.name
if(!names.contains(key)){
names << key
}
}
return names;
}
def addLoudong(){
}
}
@@ -0,0 +1,261 @@
package com.fangjia.script.loudong
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.interactions.Actions
import org.openqa.selenium.support.ui.ExpectedCondition
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
class HouseBookChromeCrawl {
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
static def city = "重庆"
static main(args) {
def task = new HouseBookChromeCrawl();
task.run()
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def listPageUrl = getIndexPage(driver)
def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl)
// keywords.remove("城南家园")
// keywords.remove("康居西城")
println city +"共 "+keywords.size()+" 个小区"
getDistrictIndex(driver, keywords)
}
def getListPage(driver, url){
driver.get(url)
def cityDoc = Jsoup.parse(driver.getPageSource())
def total = JQueryUtils.selectText(".result-count-number:eq(0)", cityDoc)
if(total){
def page = getTotalPage(total, 6)
def keywords = getAllDist(driver, url, page);
return keywords
}
return null;
}
def getDistrictIndex(driver, keywords){
for(def word : keywords){
def code = URLEncoder.encode(URLEncoder.encode(word, "utf-8"),"utf-8")
goToDetailIndex(driver, code)
}
}
def getAllDist(driver, url, page){
def keywords = []
for(def i=73;i<=page;i++){
driver.get(url+"?selectPages="+i)
def doc = Jsoup.parse(driver.getPageSource())
def eles = JQueryUtils.select(".title-area .title-text", doc)
for(def e : eles){
def text = e.text()
// if(text=="富州北郡"){
// println i
// }
keywords << text
}
}
return keywords
}
def getTotalPage(def total, def pageSize){
def mode = total.toInteger()/pageSize
if(mode > mode.toInteger()){
mode = mode.toInteger() +1
}
return mode;
}
def goToDetailIndex(driver, code){
def url = "http://www.house-book.com.cn/buildAction!pnameHome?pname="+code+"&price=&totalPrice=&bsqar=&htype=&btype=&sellKind="
driver.get(url)
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("main"))));
def docment = Jsoup.parse(driver.getPageSource());
def dist_name = JQueryUtils.select(".info-pname", docment).text()
def addInfo = JQueryUtils.select(".addr-info", docment).text()
def region = null
if(addInfo.contains(":") && addInfo.contains("-")){
region = addInfo.substring(addInfo.indexOf(":")+1, addInfo.indexOf("-"))
}
def address = JQueryUtils.select(".addr-info span", docment).text()
// wait.until(new ExpectedCondition<WebElement>(){
// @Override
// public WebElement apply(WebDriver d){
// return d.findElement(By.linkText("[更多]"))
// }
// }).click()
driver.findElement(By.linkText("[更多]")).click()
// Thread.sleep(10)
// wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.id("shouUl"))));
def lis = checkElement(driver, "#shouUl .dongli");
def datas = []
if(lis){
for(def li : lis){
def id = li.attr("id")
def houseNum = li.select(".title-span").text()
def map = [:]
map.id = id
map.city = city
map.region = region
map.district_name = dist_name
map.address = address
map.house_num = houseNum
map.num = li.select(".dong_ul2 li:contains(许可证号:)").text().replace("许可证号:","")
map.type = li.select(".dong_ul2 .title-span-hbType").text()
def floor_heigth = li.select(".dong_ul2 label:contains(总楼层:)").text().replace("总楼层:","").replace("层","")
if(floor_heigth){
map.floor_heigth = floor_heigth
}
def room_count = li.select(".dong_ul2 label:contains(总户数:)").text()
if(room_count){
room_count = room_count.substring(room_count.indexOf("总户数:")+4, room_count.lastIndexOf("户"))
if(room_count){
map.household_count = room_count
}
}
datas << map
}
}
println dist_name +" 共 "+datas.size()+" 栋"
for(def m : datas){
try{
driver.findElement(By.id(m.id)).click()
def rows = parseDetail(driver)
println "栋divId: "+m.id +" 室:"+rows.size()
saveDate(m, rows)
}catch(Exception e){
println dist_name+" 栋divId: "+m.id+" 找不到相应的div element"
}
// driver.navigate().back();
}
}
def checkElement(driver, className){
def loop = 0
def stop = false
def lis = null
while (!stop){
loop++
def doc = Jsoup.parse(driver.getPageSource());
lis = JQueryUtils.select(className, doc)
if(lis || loop>=10){
stop = true;
}else{
Thread.sleep(5000)
}
}
return lis
}
def parseDetail(driver){
def trs = checkElement(driver, ".rooms tr");
def rows = []
for(def tr : trs){
def tds = tr.select("td")
if(tds){
def floor = null
for(def j=0;j<tds.size();j++){
if(j == 0){
floor = tds[0].text().replace("第","").replace("层","")
}else {
def td = tds[j]
def detail = [:]
def room_num = td.select("b").text()
if (room_num) {
detail.room_num = room_num
}
def area = td.select("span:contains(面积:)").text().replace("面积:", "").replace("㎡", "")
if (area) {
detail.area = area.toDouble()
}
def avg_price = td.select("span:contains(单价:)").text().replace("单价:", "").replace("元/㎡", "")
if (avg_price) {
detail.avg_price = avg_price.toInteger()
}
def house_type = td.select("span:contains(房屋类型:)").text().replace("房屋类型:", "")
if(house_type){
detail.estate_type = house_type
}
if(detail){
detail.floor = floor
rows << detail
}
}
}
}
}
return rows;
}
def getIndexPage(driver){
driver.get("http://www.house-book.com.cn/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select("a:contains(新 房)", cityDoc)
if(cityEles){
def listPageUrl = cityEles[0].attr("href")
return listPageUrl
}
return null
}
def saveDate(m, rows){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", m.region)
q.put("district_name", m.district_name)
def datas = []
for(def map : rows){
q.put("house_num", m.house_num)
q.put("room_num", map.room_num)
q.put("floor_heigth" , m.floor_heigth)
q.put("floor", map.floor)
q.put("area", map.area)
def obj = loudongConn.findOne(q)
if(!obj){
def data = new BasicDBObject()
m.each{k,v->
data.put(k,v)
}
map.each{k,v->
data.put(k,v)
}
data.put("c_date", new Date())
datas << data
}
}
if(datas){
loudongConn.insert(datas)
}
}
}
@@ -0,0 +1,59 @@
package com.fangjia.script.loudong
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.thirdPriceCrawl.UserAgents
import com.fangjia.script.util.ABuYunUtil
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.jsoup.Jsoup
import org.jsoup.select.Elements
class HouseBookCrawl {
public def CHARSET = "utf-8";
static main(args) {
def task = new HouseBookCrawl()
task.run()
}
// def run(){
// goTocitypage()
// }
// def goTocitypage(){
// def url = "http://www.house-book.com.cn/mainAction/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html"
// def headers = [:]
// headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8");
// headers.put("User-Agent", UserAgents.get())
// headers.put("cookCity", "%2525E9%252587%25258D%2525E5%2525BA%252586%2525E5%2525B8%252582")
// headers.put("HBJSESSIONID","afb32a63-796e-4bbc-bf39-0f8cb188079e")
// def params = [:]
// def responseMap = ABuYunUtil.doPostResponse(url, null, headers, CHARSET, false)
// def cookie = getCookie(responseMap)
// }
//
// def getCookie(responseMap) {
// def cookieAuth = null
// try {
// def response = responseMap.get("response");
// if (StringUtils.isNotEmpty(response)) {
// JSONObject reJson = JSONObject.fromObject(response)
// //登录成功获取COOKIE
// if (reJson.success) {
// responseMap.get("headerFields").each { k, v ->
// if (k && k.equals("Set-Cookie")) {
// if (v) {
// cookieAuth = v.join(';')
// }
// }
// }
// }
// }
// } catch (Exception e) {
// println "goApiLogin(${responseMap})发生异常, 异常信息为:" + e.getMessage()
// }
// println("最终构造cookieAuth为:" + cookieAuth)
// return cookieAuth
// }
}
@@ -0,0 +1,147 @@
package com.fangjia.script.loudong
import net.sourceforge.tess4j.ITesseract
import net.sourceforge.tess4j.Tesseract
import net.sourceforge.tess4j.util.ImageHelper
import org.apache.commons.io.FileUtils
import org.ehcache.xml.model.TimeUnit
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.OutputType
import org.openqa.selenium.Point
import org.openqa.selenium.TakesScreenshot
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import javax.imageio.ImageIO
import java.awt.image.BufferedImage
public class ImageTest {
public static void main(String[] args) throws IOException {
// File imageFile = new File("D:\\gaojuan\\img\\WaterMark.gif");
// Tesseract instance = new Tesseract();
//
// //将验证码图片的内容识别为字符串
// String result = instance.doOCR(imageFile);
// println result
//1.获取网页的浏览器driver
//(1)定义url
String url="http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=2l/xVhsMJPAZKivgt8ciOA&block=3%E5%8F%B7%E6%A5%BC#";
//(2)设置driver驱动chromedriver.exe(版本:2.39)路径(相对路径)
String path = System.getProperty("user.dir");
System.setProperty("webdriver.chrome.driver","C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
//(3)建立selenium 驱动
//WebDriver driver = new ChromeDriver(options);
WebDriver driver = new ChromeDriver();
//(4)设置窗口最大化及网页等待时间
driver.manage().window().maximize(); //窗口最大化
// driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
//(5)启动driver浏览网页
driver.get(url); //输入网址
Thread.sleep(1000);
driver.findElement(By.id("cell1-11-2")).click()
Thread.sleep(1000);
def docment = Jsoup.parse(driver.getPageSource());
println docment
WebElement ele = driver.findElement(By.id("vimg"));
File screenshot = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE);
BufferedImage pic1 = ImageIO.read(screenshot);
Point point = ele.getLocation();
//(2)输入验证码
File srcfile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE);
FileUtils.copyFile(srcfile, new File("picture/source.png"));
//原图片
//String soursePicPath = "D:\\data_dev\\test\\test.png";
String soursePicPath = "picture/source.png";
File sourcePic = new File(soursePicPath);
int eleWid = ele.getSize().getWidth()
int eleHeiht = ele.getSize().getHeight()
BufferedImage pic2 = pic1.getSubimage(point.getX(), point.getY(), eleWid, eleHeiht);
ImageIO.write(pic2, "jpg", sourcePic);
//File desImage = new File("D:\\data_dev\\test\\test02.png");
File desImage = new File("picture/answer.png");
//加载待读取图片
//File imageFile = new File("D:\\data_dev\\test\\test02.png");
File imageFile = new File("picture/answer.png");
BufferedImage image = ImageIO.read(imageFile);
//对图片进行处理
image = convertImage(image);
//创建tess对象
ITesseract instance = new Tesseract();
//设置训练文件目录
//instance.setDatapath("E:\\workspace\\workspace_idea\\com_mth_data\\m-kuaidi100-data\\tessdata");
String path02 = System.getProperty("user.dir");
instance.setDatapath(path02 + "\\tessdata");
//设置训练语言
//instance.setLanguage("chi_sim");
instance.setLanguage("eng");
//执行转换
String s = instance.doOCR(image);
System.out.println(s);
String[] split = s.replace(" ", "").split("=|:");
String a = split[split.length - 1].trim();
char[] chars = a.toCharArray();
if (chars.length == 2) {
if (chars[0] == '7') {
chars[0] = '1';
}
}
a = String.valueOf(chars);
System.out.println(a);
String b = s.substring(0, 1);
System.out.println(b);
String answer = String.valueOf(Integer.parseInt(a.trim()) - Integer.parseInt(b.trim()));
WebElement input2 = driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_TextBox2\"]"));
input2.sendKeys(answer);
Thread.sleep(4000);
//(3)点击"查询"按钮
driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_Button1\"]")).click();
String SecondtHandle = driver.getWindowHandle(); //首先得到最先的窗口 权柄
for (String winHandle1 : driver.getWindowHandles()) { //得到浏览器所有窗口的权柄为Set集合,遍历
if (!winHandle1.equals(SecondtHandle)) { //如果为 最先的窗口 权柄跳出
driver.close();
driver.switchTo().window(winHandle1); //如果不为 最先的窗口 权柄,将 新窗口的操作权柄 给 driver
System.out.println(driver.getCurrentUrl()); //打印是否为新窗口
}
}
Thread.sleep(4000);
//4.关闭浏览器
driver.close();
}
//对图片进行处理 - 提高识别度
public static BufferedImage convertImage(BufferedImage image) throws Exception {
//按指定宽高创建一个图像副本
//image = ImageHelper.getSubImage(image, 0, 0, image.getWidth(), image.getHeight());
//图像转换成灰度的简单方法 - 黑白处理
image = ImageHelper.convertImageToGrayscale(image);
//图像缩放 - 放大n倍图像
image = ImageHelper.getScaledInstance(image, image.getWidth() * 15, image.getHeight() * 15);
return image;
}
}
@@ -19,6 +19,8 @@ class SnapshotMonitor extends BaseBatch {
// 运维组 运维警报
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
// 房金部自动检测机器人
// https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736
static void main(String[] args) {
runBatch(new SnapshotMonitor(), args)
@@ -0,0 +1,74 @@
package com.fangjia.script.temp.shenjian.address
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang3.StringUtils
final DBCollection townConn = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018)
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址(估家科技)V2.xlsx", {
List item ->
def id = item['A']
def province = item['D']
def city = item['E']
def area = item['F']
String township = item['G']
String community = item['H']
if (StringUtils.isBlank(city)) {
return ;
}
if (item['T'] != null) {
return ;
}
// if (community == null) {
// return ;
// }
if (township) {
if (township.endsWith("镇") || township.endsWith("乡")) {
item['T'] = 2
} else if (township.endsWith("街道")) {
item['T'] = 1
}
}
// community = community.replaceAll("居委会", "")
// community = community.replaceAll("居民委员会", "")
// community = community.replaceAll("村民委员会", "")
// community = community.replaceAll("村委会", "")
//
// if (ProvinceData.isGovernmentCity(city)) {
// townConn.find(new BasicDBObject("province", "重庆市")
// .append("region", area + "区")
// .append("street", township)
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
// obj ->
// item['T'] = obj.juweihui_type
// }
// if (item['T'] == null) {
// townConn.find(new BasicDBObject("province", "重庆市")
// .append("region", area + "区")
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
// obj ->
// item['T'] = obj.juweihui_type
// }
// }
// } else {
// townConn.find(new BasicDBObject("city", city + "市")
// .append("region", area + "区")
// .append("street", township)
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
// obj ->
// item['T'] = obj.juweihui_type
// }
// if (item['T'] == null) {
// townConn.find(new BasicDBObject("city", city + "市")
// .append("region", area + "区")
// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each {
// obj ->
// item['T'] = obj.juweihui_type
// }
// }
// }
})
@@ -0,0 +1,23 @@
package com.fangjia.script.temp.shenjian.address
import com.fangjia.script.util.ExcelUtils
Map<String, List<String>> map = new HashMap<>()
List<List<String>> readList = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\(曹合1 )重庆农商行测试地址V8-1(街道、社区、街道代码补充)处理--新-芝.xlsx")
for (List<String> result : readList) {
map.put(result.get(0), result)
}
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", {
List item ->
def id = item['A']
def township = item['G']
def community = item['H']
if (org.apache.commons.lang.StringUtils.isBlank(township)) {
item['G'] = map[id]['F']
item['Y'] = '人工乡镇'
}
if (org.apache.commons.lang.StringUtils.isBlank(community)) {
item['H'] = map[id]['G']
item['Z'] = '人工社区'
}
})
@@ -0,0 +1,57 @@
package com.fangjia.script.temp.shenjian.address
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang.StringUtils
/**
* 补充详细地址中缺失的乡镇街道与居委社区
*/
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", {
List item ->
def area = item['F']
def township = item['G']
def community = item['H']
String detailAddr = item['I']
if (detailAddr == '重庆市云阳县双江街道恒生场2-02') {
print("test")
}
// 不包含街道与居委
if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) &&
!detailAddr.contains(township) && !detailAddr.contains(community) ) {
if (StringUtils.isNotBlank(area)) {
if (detailAddr.indexOf(area + "区") > 0) {
int index = detailAddr.indexOf(area + "区") + area.length() + 1
item['K'] = detailAddr.substring(0, index) + township + community +
detailAddr.substring(index)
} else if (detailAddr.indexOf(area + "县") > 0) {
int index = detailAddr.indexOf(area + "县") + area.length() + 1
item['K'] = detailAddr.substring(0, index) + township + community +
detailAddr.substring(index)
}
}
// 不包含社区
} else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) &&
detailAddr.contains(township) && !detailAddr.contains(community)) {
if (detailAddr.indexOf(township) > 0) {
int index = detailAddr.indexOf(township) + township.length()
item['K'] = detailAddr.substring(0, index) + community +
detailAddr.substring(index)
}
} // 只不包含街道
else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) &&
!detailAddr.contains(township) && detailAddr.contains(community)) {
if (StringUtils.isNotBlank(area)) {
if (detailAddr.indexOf(area + "区") > 0) {
int index = detailAddr.indexOf(area + "区") + area.length() + 1
item['K'] = detailAddr.substring(0, index) + township +
detailAddr.substring(index)
} else if (detailAddr.indexOf(area + "县") > 0) {
int index = detailAddr.indexOf(area + "县") + area.length() + 1
item['K'] = detailAddr.substring(0, index) + township +
detailAddr.substring(index)
}
}
}
})
@@ -0,0 +1,78 @@
package com.fangjia.script.temp.shenjian.address
import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
//ExcelUtils.removeEmptyRow("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V10(优化补充社区冗余信息).xlsx")
ners = ["houseno", "cellno", "floorno", "roomno"]
Pattern GROUP_PATTERN = Pattern.compile("[一二三四五六七八九十甲乙丙丁\\d]+组")
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx",{List item->
if (StringUtils.isNotBlank(item["K"])) {
item["I"] = item["K"]
}
// def addr = item["B"].toString()
// def detailAddr = item["I"]?.toString()
// def pd = item["C"]
// item['J'] = ""
// if (StringUtils.isNotBlank(pd)) {
// int len = addr.lastIndexOf(pd)
// if (len > 3) {
// String aim = addr.substring(len - 3, len)
// int aimIndex = detailAddr.lastIndexOf(aim)
// if (aimIndex > 0) {
// item['J'] = detailAddr.substring(0, aimIndex + 3) + pd
// }
// }
// }
// if(detailAddr{
// String group = DataUtil.match(pd, GROUP_PATTERN)
// if(group == null ){
// return
// }
// def arabic_group = null
// if(!detailAddr.contains(group)){
// arabic_group = CNNMFilter.qCNNumericToArabic(group)
// }
// if(group){
// def flg = addr.substring(addr.indexOf(group) - 3, addr.indexOf(group))
// flg = flg.replace("-", "")
// if(arabic_group != null){
// group = arabic_group
// }
// if(!detailAddr.contains(flg + group)){
// println(item)
// detailAddr = detailAddr.replace(group, '')
// def idx = detailAddr.lastIndexOf(flg)
// try{
// detailAddr = detailAddr.substring(0, idx) + flg + group + detailAddr.substring(idx + flg.length())
// item["j"] = detailAddr
// }catch(Exception e){
// item["W"] ="组替换失败"
// }
//
//
// }
// }
// }
// HttpRequest request = new HttpRequest()
// JSONObject result = request.getJsonContent("http://192.168.0.19:31080/FangJiaNlpService/ner",["address":addr], "utf-8")
// def index = Integer.MAX_VALUE
// for(JSONObject entity : result.entities){
// if(entity.type in ners){
// if(entity.start < index){
// index = entity.start
// }
// }
// }
// if(index != Integer.MAX_VALUE){
// item["C"] = addr.substring(index)
// println(item)
// }
})
@@ -10,6 +10,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
@@ -27,11 +29,13 @@ class SeaweedExport {
static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017)
static def map = new ConcurrentHashMap()
static void main(String[] args) {
def resultMap = new ConcurrentHashMap()
List<List<Object>> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9)
List<List<Object>> data = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\重庆样本匹配结果V4.xlsx")
CountDownLatch latch = new CountDownLatch(data.size())
for (List<Object> districtT : data) {
final List<Object> district = districtT
@@ -39,72 +43,55 @@ class SeaweedExport {
@Override
void run() {
try {
def city = district.get(1)
def district_code = district.get(2)
def name = district.get(3)
def matchedCity = district.get(22)
def matchedRegion = district.get(23)
def matchedName = district.get(24)
def region;
def no = district.get(0)
def id = district.get(2)
def ret = []
if (id != null) {
DBObject query = new BasicDBObject()
query.put("_id", new ObjectId(id))
//query.put("region", matchedRegion)
query.put("cat", "district")
query.put("visible", 0)
//query.put("name", matchedName)
if (map.get("district_code") != null) {
region = map.get("district_code")
} else {
cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj ->
region = obj.fangjia_district
map.put("district_code", region)
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("developer", 1)
filter.put("estate_company", 1)
filter.put("cubage_percent", 1) // 容积率
filter.put("virescence_percent", 1) // 绿化率
filter.put("alias", 1)
filter.put("car_site_count", 1)
filter.put("all_area", 1)
filter.put("loudong_count", 1)
filter.put("household_count", 1)
filter.put("estate_charge", 1)
filter.put("open_date", 1) // 开盘时间
filter.put("live_date", 1) // 交房时间
filter.put("elevator", 1) // 电梯
DBObject obj = seaweed.findOne(new ObjectId(id))
if (obj) {
ret[0] = obj.estate_company
ret[1] = obj.developer
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
ret[4] = SeaweedUtils.formatAlias(obj.mapping_alias)
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
ret[7] = ""
ret[8] = SeaweedUtils.formatAllArea(obj.all_area)
ret[9] = obj.loudong_count
ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count)
ret[11] = ""
ret[12] = obj.estate_charge
ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date)
ret[14] = obj.live_date
ret[15] = obj.is_elevator == true ? "有" : (obj.is_elevator == false ? "无" : "")
ret[16] = no
resultMap.put(no, ret)
}
}
if (matchedRegion == null) {
matchedRegion = region
matchedName = name
}
def ret = []
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("region", matchedRegion)
query.put("cat", "district")
query.put("visible", 0)
query.put("name", matchedName)
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("developer", 1)
filter.put("estate_company", 1)
filter.put("cubage_percent", 1) // 容积率
filter.put("virescence_percent", 1) // 绿化率
filter.put("alias", 1)
filter.put("car_site_count", 1)
filter.put("all_area", 1)
filter.put("loudong_count", 1)
filter.put("household_count", 1)
filter.put("estate_charge", 1)
filter.put("open_date", 1) // 开盘时间
filter.put("live_date", 1) // 交房时间
filter.put("lng", 1)
filter.put("lat", 1)
DbUtils.seaweed.find(query).limit(1).each { obj ->
ret[0] = obj.estate_company
ret[1] = obj.developer
ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent)
ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent)
ret[4] = SeaweedUtils.formatAlias(obj.alias)
ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count)
ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count)
ret[7] = ""
ret[8] = SeaweedUtils.formatAllArea(obj.all_area)
ret[9] = obj.loudong_count
ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count)
ret[11] = ""
ret[12] = obj.estate_charge
ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date)
ret[14] = obj.live_date
ret[15] = obj.lng
ret[16] = obj.lat
}
ret[17] = name
resultMap.put(district_code + name, ret)
} catch (Exception ex) {
ex.printStackTrace()
} finally {
@@ -115,13 +102,12 @@ class SeaweedExport {
};
latch.await();
def ret = [];
ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"];
ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "是否有电梯", "序号"];
for (List<Object> district : data) {
def district_code = district.get(2)
def name = district.get(3)
ret << resultMap.get(district_code + name)
def no = district.get(0)
ret << resultMap.get(no)
};
com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商及物业信息.xlsx", ret, "开发商及物业信息");
com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\开发商及物业信息.xlsx", ret, "开发商及物业信息");
executorService.shutdown()
}
}
@@ -12,7 +12,8 @@ import com.fangjia.script.util.DateUtils
class DingTalkUtils {
static main(String[] args) {
Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ;
// 要导出的月份第一天
Date now = DateUtils.parseFromTightByPattern("2022-03-01", "yyyy-MM-dd") ;
String startTime = DateUtils.addMonth(now, -1) + " 00:00:00"
String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss")
@@ -21,10 +21,10 @@ import java.util.concurrent.Executors
class CommitteeLatLng {
static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017)
static ExecutorService executorService = Executors.newFixedThreadPool(20)
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static void main(String[] args) {
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("name", "席庄二村委会"))
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021))
//dealLocation()
executorService.shutdown()
}

Some files were not shown because too many files have changed in this diff Show More