Merge remote-tracking branch 'origin/master'
This commit is contained in:
+9
@@ -0,0 +1,9 @@
|
||||
package com.fangjia.dal.mongo.open.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
|
||||
public interface CommunityBasePriceDao extends MongoDao<CommunityBasePrice> {
|
||||
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.fangjia.dal.mongo.open.dao.impl;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
|
||||
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
|
||||
public class CommunityBasePriceDaoImpl extends MongoDaoImpl<CommunityBasePrice> implements CommunityBasePriceDao {
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package com.fangjia.dal.mongo.open.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
|
||||
/**
|
||||
* Ð¡Çø½¨ÐÐÖÃÐŶÈÐÅÏ¢
|
||||
*
|
||||
* @author shenjian@fangjia.com
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "business.community_base_price_gj")
|
||||
public class CommunityBasePrice implements MongoEntity {
|
||||
|
||||
private static final long serialVersionUID = -8010010773207622878L;
|
||||
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String name;
|
||||
private Double credit;
|
||||
|
||||
@Id
|
||||
@Column(name = "_id")
|
||||
@Override
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "city")
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
|
||||
@Column(name = "region")
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
|
||||
public void setRegion(String region) {
|
||||
this.region = region;
|
||||
}
|
||||
|
||||
@Column(name = "name")
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
@Column(name = "credit")
|
||||
public Double getCredit() {
|
||||
return credit;
|
||||
}
|
||||
|
||||
public void setCredit(Double credit) {
|
||||
this.credit = credit;
|
||||
}
|
||||
}
|
||||
+144
-33
@@ -31,6 +31,7 @@ import javax.validation.Valid;
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.dal.*;
|
||||
import com.fangjia.dal.mongo.model.*;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
import com.fangjia.dal.seawater.dao.SeawaterDao;
|
||||
import com.fangjia.dal.seawater.model.Seawater;
|
||||
@@ -177,7 +178,9 @@ public class PropertyController {
|
||||
private SeawaterDao seawaterDao;
|
||||
private SeawaterService seawaterService;
|
||||
private SqmetreRentService sqmetreRentService;
|
||||
private CommunityPoiServiceImpl communityPoiService;
|
||||
private CommunityPoiService communityPoiService;
|
||||
private CommunityBasePriceService communityBasePriceService;
|
||||
private EvaluateFactorService evaluateFactorService;
|
||||
|
||||
private FileDao fileDao = FileDao.getInstance();
|
||||
private static final int DEFAULT_PAGE_SIZE = 20;
|
||||
@@ -393,10 +396,31 @@ public class PropertyController {
|
||||
put("5000", "five");// 五线
|
||||
}
|
||||
};
|
||||
/**
|
||||
* 第三方均价映射
|
||||
*/
|
||||
private static final Map<String, String> OTHER_PRICE_MAP = new HashMap<String, String>() {
|
||||
{
|
||||
put("zhuge_price", "诸葛找房");
|
||||
put("anjuke_price", "安居客");
|
||||
put("fang_price", "房天下");
|
||||
put("lianjia_price", "链家");
|
||||
put("cityhouse_price", "禧泰数据");
|
||||
put("maitian_price", "麦田");
|
||||
put("ke_price", "贝壳");
|
||||
put("taofang_price", "淘房365");
|
||||
put("zhongyuan_price", "中原");
|
||||
put("tbs_price", "兔博士");
|
||||
put("qfang_price", "Q房");
|
||||
put("fangtan_price", "房探");
|
||||
put("fangduoduo_price", "房多多");
|
||||
put("wiwj_price", "我爱我家");
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 利用百度API把GPS转换成Google经纬度
|
||||
*
|
||||
*
|
||||
* @param lng
|
||||
* @param lat
|
||||
* @return
|
||||
@@ -418,7 +442,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 利用算法模板把GPS转换成Google经纬度
|
||||
*
|
||||
*
|
||||
* @param lng
|
||||
* @param lat
|
||||
* @return
|
||||
@@ -610,7 +634,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 判断小区是否有经纬度
|
||||
*
|
||||
*
|
||||
* @param district
|
||||
* @return
|
||||
*/
|
||||
@@ -621,7 +645,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 调用匹配接口匹配小区
|
||||
*
|
||||
*
|
||||
* @param city
|
||||
* @param address
|
||||
* @param region
|
||||
@@ -995,6 +1019,32 @@ public class PropertyController {
|
||||
}
|
||||
}
|
||||
|
||||
// 调整面积不同面积价格
|
||||
try {
|
||||
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
|
||||
int orgAvgPrice = 0;
|
||||
if (districtStatBo != null) {
|
||||
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
|
||||
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
|
||||
} else if (districtStatBo.getDistrictStat() != null) {
|
||||
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
|
||||
}
|
||||
}
|
||||
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
|
||||
int factorPrice = evaluation.getAvgPrice();
|
||||
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
|
||||
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
|
||||
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
|
||||
// 偏离度超过5%不进行调整
|
||||
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||
evaluation.setAvgPrice(avgPrice);
|
||||
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
|
||||
}
|
||||
|
||||
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
|
||||
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
|
||||
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
|
||||
@@ -1165,6 +1215,31 @@ public class PropertyController {
|
||||
evaluationService.remove(evaluation.getId());
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
|
||||
return mav;
|
||||
}
|
||||
// 调整面积不同面积价格
|
||||
try {
|
||||
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
|
||||
int orgAvgPrice = 0;
|
||||
if (districtStatBo != null) {
|
||||
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
|
||||
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
|
||||
} else if (districtStatBo.getDistrictStat() != null) {
|
||||
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
|
||||
}
|
||||
}
|
||||
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
|
||||
int factorPrice = evaluation.getAvgPrice();
|
||||
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
|
||||
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
|
||||
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
|
||||
// 偏离度超过5%不进行调整
|
||||
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||
evaluation.setAvgPrice(avgPrice);
|
||||
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
|
||||
}
|
||||
Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>();
|
||||
districtPriceMap.put("city", district.getCity());
|
||||
@@ -1179,6 +1254,13 @@ public class PropertyController {
|
||||
// }
|
||||
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
|
||||
districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
|
||||
// 新增置信度
|
||||
CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId());
|
||||
if (communityBasePrice != null) {
|
||||
districtPriceMap.put("credit", communityBasePrice.getCredit());
|
||||
} else {
|
||||
districtPriceMap.put("credit", evaluation.getCredit());
|
||||
}
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add(district.getName());
|
||||
if(district.getAlias() != null){
|
||||
@@ -1190,6 +1272,31 @@ public class PropertyController {
|
||||
}
|
||||
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
|
||||
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
|
||||
Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9);
|
||||
Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1);
|
||||
// 新增第三方均价
|
||||
try {
|
||||
double dCredit = 0.61;
|
||||
Map<String, Object> rMap = getLastEvaluationPrice(district, evaluation, dCredit);
|
||||
if (null != rMap) {
|
||||
if (rMap.containsKey("other_price")) {
|
||||
Map<String, Integer> otherPriceMap = (Map<String, Integer>) rMap.get("other_price");
|
||||
Map<String, Integer> priceResultMap = new HashMap<String, Integer>();
|
||||
for (String key : otherPriceMap.keySet()) {
|
||||
Integer price = otherPriceMap.get(key);
|
||||
// 剔除偏差大于10的
|
||||
if (min10Price <= price && max10Price >= price) {
|
||||
if (OTHER_PRICE_MAP.containsKey(key)) {
|
||||
priceResultMap.put(OTHER_PRICE_MAP.get(key), price);
|
||||
}
|
||||
}
|
||||
}
|
||||
districtPriceMap.put("otherPrice", priceResultMap);
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
|
||||
}
|
||||
List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
|
||||
if(houses.size() > 3){
|
||||
districtPriceMap.put("calcType", 1);
|
||||
@@ -1491,7 +1598,7 @@ public class PropertyController {
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 四川锦程 定制接口 多建筑年代
|
||||
* @param model
|
||||
@@ -1603,7 +1710,7 @@ public class PropertyController {
|
||||
districtPriceMap.put("buildYear", matcher3.group());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// 根据城市等 级设置D级可信度
|
||||
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
|
||||
@@ -2116,7 +2223,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区租金查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -2595,7 +2702,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 批量获取小区的均价,均价走势数据
|
||||
*
|
||||
*
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
@@ -2737,7 +2844,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 批量获取小区的配套数据,最多12个小区
|
||||
*
|
||||
*
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
@@ -2882,7 +2989,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 获取JSON参数中的num参数值
|
||||
*
|
||||
*
|
||||
* @param jo
|
||||
* @return
|
||||
*/
|
||||
@@ -2907,7 +3014,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 获取JSON参数中的radius值
|
||||
*
|
||||
*
|
||||
* @param jo
|
||||
* @return
|
||||
*/
|
||||
@@ -3715,7 +3822,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 解析出关键字中的小区名
|
||||
*
|
||||
*
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
@@ -3832,7 +3939,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区统计分析接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -3917,7 +4024,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区交通接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -3975,7 +4082,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区周边接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4034,7 +4141,7 @@ public class PropertyController {
|
||||
/**
|
||||
* 相同小区相似户型房源挂牌量接口<br>
|
||||
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
|
||||
*
|
||||
*
|
||||
* @author yinjihuan
|
||||
* @date 2016年9月1日
|
||||
* @param city
|
||||
@@ -4074,7 +4181,7 @@ public class PropertyController {
|
||||
/**
|
||||
* 周边小区相似户型挂牌量接口<br>
|
||||
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
|
||||
*
|
||||
*
|
||||
* @author yinjihuan
|
||||
* @date 2016年9月1日
|
||||
* @param city
|
||||
@@ -4125,7 +4232,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区周边一公里均价
|
||||
*
|
||||
*
|
||||
* @author wenyan
|
||||
* @param model
|
||||
* @return
|
||||
@@ -4148,7 +4255,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 相同小区相似户型租金范围查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4191,7 +4298,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 周边小区相似户型租金范围查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4254,7 +4361,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 室号查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4290,7 +4397,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 楼栋楼层信息查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4315,7 +4422,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 楼栋查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4360,7 +4467,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 楼栋查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4813,7 +4920,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 房产交易记录查询
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4873,7 +4980,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 房产拍卖记录
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4934,7 +5041,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 房产近期评估次数
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -5650,11 +5757,15 @@ public class PropertyController {
|
||||
this.rentDealPriceService = rentDealPriceService;
|
||||
}
|
||||
|
||||
public CommunityPoiServiceImpl getCommunityPoiService() {
|
||||
return communityPoiService;
|
||||
}
|
||||
|
||||
public void setCommunityPoiService(CommunityPoiServiceImpl communityPoiService) {
|
||||
public void setCommunityPoiService(CommunityPoiService communityPoiService) {
|
||||
this.communityPoiService = communityPoiService;
|
||||
}
|
||||
|
||||
public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) {
|
||||
this.evaluateFactorService = evaluateFactorService;
|
||||
}
|
||||
|
||||
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
|
||||
this.communityBasePriceService = communityBasePriceService;
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+2
-2
@@ -38,8 +38,8 @@ class ExportOracleData {
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-03-19")
|
||||
exportBigdataDeal("2022-03-01")
|
||||
exportBigdataForsale("2022-04-19")
|
||||
exportBigdataDeal("2022-04-01")
|
||||
|
||||
|
||||
//计算结果相关
|
||||
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DistrictCodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* oralce中小区统计数据同步至85中
|
||||
*/
|
||||
class OracleDistrict2Mongo85 {
|
||||
|
||||
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017)
|
||||
|
||||
static main(args) {
|
||||
// 只写入重庆市数据
|
||||
def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ"
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def city = DistrictCodeUtils.getCityName(r['cityid'])
|
||||
def region = RegionData.formatRegion(city, r['distname'])
|
||||
DBObject dbObject = new BasicDBObject("_id", r['commid'])
|
||||
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
|
||||
.append("name", r['commname'])
|
||||
.append('subway', Integer.parseInt(r['subway'].toString()))
|
||||
.append("traffic", Integer.parseInt(r['traffic'].toString()))
|
||||
.append("bank", Integer.parseInt(r['bank'].toString()))
|
||||
.append("primary", Integer.parseInt(r['primary'].toString()))
|
||||
.append("junior", Integer.parseInt(r['junior'].toString()))
|
||||
.append("hospital", Integer.parseInt(r['hospital'].toString()))
|
||||
.append("funeral", Integer.parseInt(r['funeral'].toString()))
|
||||
.append("c_date", new Date());
|
||||
communityPoiConn.update(dbObject, updateObject, true, false)
|
||||
count++;
|
||||
if (count % 500 == 0) {
|
||||
println "已写入" + count + "条小区数据"
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202201
|
||||
def endDate = 202203
|
||||
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202202
|
||||
def endDate = 202204
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+8
-8
@@ -40,8 +40,8 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-01-01"
|
||||
def date = "2022-02-01"
|
||||
def endDate = "2022-03-01"
|
||||
def newDate = "2022-04-01"
|
||||
def startDate = "2022-02-01"
|
||||
def date = "2022-03-01"
|
||||
def endDate = "2022-04-01"
|
||||
def newDate = "2022-05-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -131,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import groovy.sql.Sql
|
||||
|
||||
class CcbDistNameMatch {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDistNameMatch();
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
if(d['D'].contains("、")){
|
||||
println d['D']
|
||||
}
|
||||
def arrs = d['D'].split("、")
|
||||
def list = []
|
||||
def flags = []
|
||||
for(def s : arrs){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district")
|
||||
if(dist){
|
||||
list << dist.name
|
||||
flags << "完全一致"
|
||||
}else{
|
||||
def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
if(jsonObject.get("district") == newRegion){
|
||||
list << jsonObject.get("name")
|
||||
flags << "匹配接口"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if(list){
|
||||
list = list.unique()
|
||||
d['E'] = list.join(",")
|
||||
d['F'] = flags.join(",")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d ->
|
||||
def regionCode = null
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if(dist){
|
||||
d['D'] = dist.name
|
||||
def count = findDistrictByOracle(dist._id.toString())
|
||||
if(count && count>0){
|
||||
d['F'] = 1
|
||||
}else{
|
||||
d['H'] = dist.estate_type2
|
||||
d['J'] = dist.name_cat
|
||||
}
|
||||
}else{
|
||||
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (rets) {
|
||||
if(rets.size()>1){
|
||||
d['E'] = "别名-多个"
|
||||
}else{
|
||||
dist = rets[0]
|
||||
d['E'] = d['C'] +"--"+dist.name
|
||||
def count = findAliasByOracle(dist._id.toString())
|
||||
if(count && count>0) {
|
||||
d['G'] = 1
|
||||
}else{
|
||||
d['H'] = dist.estate_type2
|
||||
d['J'] = dist.name_cat
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def findDistrictByOracle(commid){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
def findAliasByOracle(commid){
|
||||
def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
}
|
||||
+14
-13
@@ -23,22 +23,22 @@ class MatchEvaluateRegionTest {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\估值限时测试-重庆农商行1.0.xlsx"
|
||||
def total = 1500
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateRegionTest()
|
||||
task.run()//小区匹配
|
||||
task.run1()//地图匹配
|
||||
// task.run()//小区匹配
|
||||
// task.run1()//地图匹配
|
||||
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
task.addPrice()
|
||||
task.addArroundPrice()
|
||||
task.addAuthor()
|
||||
// task.addArroundPrice()
|
||||
// task.addAuthor()
|
||||
}
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], "property", false,null,null,)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = jsonObject.get("id")
|
||||
row['G'] = jsonObject.get("city")
|
||||
@@ -57,7 +57,7 @@ class MatchEvaluateRegionTest {
|
||||
def run1(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], null, true, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = jsonObject.get("id")
|
||||
row['G'] = jsonObject.get("city")
|
||||
@@ -82,7 +82,7 @@ class MatchEvaluateRegionTest {
|
||||
row['N']= (int)(priceObj.avg_price * row['E'])
|
||||
}
|
||||
|
||||
def avg_price = findAvgPrice(row['M'])
|
||||
def avg_price = findAvgPrice(row['F'])
|
||||
if(row['M'] && avg_price){
|
||||
def weekendPrice = row['M'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
@@ -95,12 +95,12 @@ class MatchEvaluateRegionTest {
|
||||
|
||||
def addArroundPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['A'] && !row['K']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
|
||||
if(!row['M']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['M'] = priceObj.unitPrice
|
||||
row['N'] = (int) (priceObj.unitPrice * row['C'])
|
||||
row['N'] = (int) (priceObj.unitPrice * row['E'])
|
||||
row['Q'] = "周边价格"
|
||||
}
|
||||
}
|
||||
@@ -109,21 +109,22 @@ class MatchEvaluateRegionTest {
|
||||
|
||||
def addAuthor(){
|
||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def total = 747
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['B']){
|
||||
if(index < avg){
|
||||
row['P'] = authors[count]
|
||||
row['N'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+9
-9
@@ -28,14 +28,14 @@ class MatchEvaluateTest {
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateTest()
|
||||
// task.run()//小区匹配
|
||||
// task.run1()//地图匹配
|
||||
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
// task.addPrice()
|
||||
// task.addArroundPrice()
|
||||
// task.addAuthor()
|
||||
task.run()//小区匹配
|
||||
task.run1()//地图匹配
|
||||
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
task.addPrice()
|
||||
task.addArroundPrice()
|
||||
task.addAuthor()
|
||||
|
||||
task.stat()
|
||||
// task.stat()
|
||||
}
|
||||
|
||||
def addArroundPrice(){
|
||||
@@ -57,7 +57,7 @@ class MatchEvaluateTest {
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['D'] && row['B']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||
def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['D'] = jsonObject.get("id")
|
||||
row['E'] = jsonObject.get("city")
|
||||
@@ -77,7 +77,7 @@ class MatchEvaluateTest {
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['D'] && row['B']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['D'] = jsonObject.get("id")
|
||||
row['E'] = jsonObject.get("city")
|
||||
|
||||
@@ -0,0 +1,763 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PocStat {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
|
||||
static main(args) {
|
||||
def task = new PocStat();
|
||||
task.regionDistStat();
|
||||
// task.sampleStat()
|
||||
// task.tradeStat()
|
||||
// task.paimaiStat()
|
||||
// task.evalStat()
|
||||
//
|
||||
// task.fileldStat()
|
||||
}
|
||||
|
||||
def regionDistStat(){
|
||||
def city = "重庆"
|
||||
def regionMap = CommonDatautils.getRegionsByCity(city)
|
||||
def set =regionMap.keySet();
|
||||
def rows = []
|
||||
rows << ["区域","小区量"]
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
|
||||
for(def region : set){
|
||||
def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district")
|
||||
|
||||
def regionCount = 0
|
||||
def regionHouseLDCount = 0
|
||||
for(def d : dists){
|
||||
if(d.region == region){
|
||||
def loudongCount = CommonDatautils.findLoudong(d._id.toString())
|
||||
if(loudongCount && loudongCount>0){
|
||||
regionCount = regionCount+ loudongCount
|
||||
}
|
||||
|
||||
def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString())
|
||||
if(houseLouDongCount && houseLouDongCount>0){
|
||||
regionHouseLDCount = regionHouseLDCount + houseLouDongCount
|
||||
}
|
||||
}
|
||||
}
|
||||
rows << [region, count, regionCount, regionHouseLDCount]
|
||||
println region +" 完成"
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计")
|
||||
}
|
||||
|
||||
def sampleStat(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||
def zCityDistTotal = new AtomicLong()
|
||||
def fCityDistTotal = new AtomicLong()
|
||||
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
|
||||
def zHLevel1 = new AtomicLong()
|
||||
def zHLevel2 = new AtomicLong()
|
||||
def zHLevel3 = new AtomicLong()
|
||||
def zHLevel4 = new AtomicLong()
|
||||
def zHLevel5 = new AtomicLong()
|
||||
|
||||
def fHLevel1 = new AtomicLong()
|
||||
def fHLevel2 = new AtomicLong()
|
||||
def fHLevel3 = new AtomicLong()
|
||||
def fHLevel4 = new AtomicLong()
|
||||
def fHLevel5 = new AtomicLong()
|
||||
|
||||
def rets = []
|
||||
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
|
||||
|
||||
|
||||
def index = new AtomicLong()
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
index.addAndGet(1)
|
||||
// println index +" "+d.name
|
||||
def commid = d._id.toString()
|
||||
//房源
|
||||
def jsonObject1 = OpenApiClient.houseList(commid,"20220201",60)
|
||||
if(jsonObject1 && jsonObject1.result) {
|
||||
def results = jsonObject1.result
|
||||
|
||||
if(zRegions.contains(d.region)){
|
||||
zCityDistTotal.addAndGet(1)
|
||||
if(results){
|
||||
if(results.size()<5){
|
||||
zHLevel2.addAndGet(1)
|
||||
}else if(results.size()<20){
|
||||
zHLevel3.addAndGet(1)
|
||||
}else if(results.size()<50){
|
||||
zHLevel4.addAndGet(1)
|
||||
}else if(results.size()>=50){
|
||||
zHLevel5.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
|
||||
if(results){
|
||||
if(results.size()<5){
|
||||
fHLevel2.addAndGet(1)
|
||||
}else if(results.size()<20){
|
||||
fHLevel3.addAndGet(1)
|
||||
}else if(results.size()<50){
|
||||
fHLevel4.addAndGet(1)
|
||||
}else if(results.size()>=50){
|
||||
fHLevel5.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(zRegions.contains(d.region)) {
|
||||
zCityDistTotal.addAndGet(1)
|
||||
zHLevel1.addAndGet(1)
|
||||
|
||||
println d.city+"#"+d.region+"#"+d.name
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
fHLevel1.addAndGet(1)
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
def ret1 = ["主城区", zCityDistTotal.get(), zHLevel1.get(), zHLevel2.get(), zHLevel3.get(), zHLevel4.get(), zHLevel5.get()]
|
||||
def ret2 = ["非主城区", fCityDistTotal.get(), fHLevel1.get(), fHLevel2.get(), fHLevel3.get(), fHLevel4.get(), fHLevel5.get()]
|
||||
rets.add(ret1)
|
||||
rets.add(ret2)
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-样本")
|
||||
}
|
||||
|
||||
|
||||
def tradeStat(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||
def zCityDistTotal = new AtomicLong()
|
||||
def fCityDistTotal = new AtomicLong()
|
||||
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
|
||||
def zTLevel1 = new AtomicLong()
|
||||
def zTLevel2 = new AtomicLong()
|
||||
def zTLevel3 = new AtomicLong()
|
||||
def zTLevel4 = new AtomicLong()
|
||||
def zTLevel5 = new AtomicLong()
|
||||
|
||||
def fTLevel1 = new AtomicLong()
|
||||
def fTLevel2 = new AtomicLong()
|
||||
def fTLevel3 = new AtomicLong()
|
||||
def fTLevel4 = new AtomicLong()
|
||||
def fTLevel5 = new AtomicLong()
|
||||
|
||||
def rets = []
|
||||
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
|
||||
|
||||
|
||||
def index = new AtomicLong()
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
index.addAndGet(1)
|
||||
println index +" "+d.name
|
||||
def commid = d._id.toString()
|
||||
//房源
|
||||
def jsonObject1 = OpenApiClient.searchDealPrice(commid,"20220201",60)
|
||||
if(jsonObject1 && jsonObject1.result) {
|
||||
def results = jsonObject1.result
|
||||
|
||||
if(zRegions.contains(d.region)){
|
||||
zCityDistTotal.addAndGet(1)
|
||||
if(results){
|
||||
if(results.size()<5){
|
||||
zTLevel2.addAndGet(1)
|
||||
}else if(results.size()<20){
|
||||
zTLevel3.addAndGet(1)
|
||||
}else if(results.size()<50){
|
||||
zTLevel4.addAndGet(1)
|
||||
}else if(results.size()>=50){
|
||||
zTLevel5.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
|
||||
if(results){
|
||||
if(results.size()<5){
|
||||
fTLevel2.addAndGet(1)
|
||||
}else if(results.size()<20){
|
||||
fTLevel3.addAndGet(1)
|
||||
}else if(results.size()<50){
|
||||
fTLevel4.addAndGet(1)
|
||||
}else if(results.size()>=50){
|
||||
fTLevel5.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(zRegions.contains(d.region)) {
|
||||
zCityDistTotal.addAndGet(1)
|
||||
zTLevel1.addAndGet(1)
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
fTLevel1.addAndGet(1)
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
def ret1 = ["主城区", zCityDistTotal.get(), zTLevel1.get(), zTLevel2.get(), zTLevel3.get(), zTLevel4.get(), zTLevel5.get()]
|
||||
def ret2 = ["非主城区", fCityDistTotal.get(), fTLevel1.get(), fTLevel2.get(), fTLevel3.get(), fTLevel4.get(), fTLevel5.get()]
|
||||
rets.add(ret1)
|
||||
rets.add(ret2)
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-成交")
|
||||
println "成交案例统计完成"
|
||||
}
|
||||
|
||||
def paimaiStat(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||
def zCityDistTotal = new AtomicLong()
|
||||
def fCityDistTotal = new AtomicLong()
|
||||
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
|
||||
def zPLevel1 = new AtomicLong()
|
||||
def zPLevel2 = new AtomicLong()
|
||||
def zPLevel3 = new AtomicLong()
|
||||
def zPLevel4 = new AtomicLong()
|
||||
def zPLevel5 = new AtomicLong()
|
||||
|
||||
def fPLevel1 = new AtomicLong()
|
||||
def fPLevel2 = new AtomicLong()
|
||||
def fPLevel3 = new AtomicLong()
|
||||
def fPLevel4 = new AtomicLong()
|
||||
def fPLevel5 = new AtomicLong()
|
||||
|
||||
def rets = []
|
||||
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
|
||||
|
||||
|
||||
def index = new AtomicLong()
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
index.addAndGet(1)
|
||||
println index +" "+d.name
|
||||
def commid = d._id.toString()
|
||||
//房源
|
||||
def jsonObject1 = OpenApiClient.searchPaimais(d.city, d.region, d.name,"20220201",100)
|
||||
if(jsonObject1 && jsonObject1.result) {
|
||||
def results = jsonObject1.result
|
||||
|
||||
if(zRegions.contains(d.region)){
|
||||
zCityDistTotal.addAndGet(1)
|
||||
if(results){
|
||||
if(results.size()<5){
|
||||
zPLevel2.addAndGet(1)
|
||||
}else if(results.size()<20){
|
||||
zPLevel3.addAndGet(1)
|
||||
}else if(results.size()<50){
|
||||
zPLevel4.addAndGet(1)
|
||||
}else if(results.size()>=50){
|
||||
zPLevel5.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
|
||||
if(results){
|
||||
if(results.size()<5){
|
||||
fPLevel2.addAndGet(1)
|
||||
}else if(results.size()<20){
|
||||
fPLevel3.addAndGet(1)
|
||||
}else if(results.size()<50){
|
||||
fPLevel4.addAndGet(1)
|
||||
}else if(results.size()>=50){
|
||||
fPLevel5.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(zRegions.contains(d.region)) {
|
||||
zCityDistTotal.addAndGet(1)
|
||||
zPLevel1.addAndGet(1)
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
fPLevel1.addAndGet(1)
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
def ret1 = ["主城区", zCityDistTotal.get(), zPLevel1.get(), zPLevel2.get(), zPLevel3.get(), zPLevel4.get(), zPLevel5.get()]
|
||||
def ret2 = ["非主城区", fCityDistTotal.get(), fPLevel1.get(), fPLevel2.get(), fPLevel3.get(), fPLevel4.get(), fPLevel5.get()]
|
||||
rets.add(ret1)
|
||||
rets.add(ret2)
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-拍卖")
|
||||
println "拍卖案例统计完成"
|
||||
}
|
||||
def evalStat(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||
def zCityDistTotal = new AtomicLong()
|
||||
def fCityDistTotal = new AtomicLong()
|
||||
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
|
||||
def zELevel1 = new AtomicLong()
|
||||
def zELevel2 = new AtomicLong()
|
||||
def zELevel3 = new AtomicLong()
|
||||
def zELevel4 = new AtomicLong()
|
||||
def zELevel5 = new AtomicLong()
|
||||
|
||||
def fELevel1 = new AtomicLong()
|
||||
def fELevel2 = new AtomicLong()
|
||||
def fELevel3 = new AtomicLong()
|
||||
def fELevel4 = new AtomicLong()
|
||||
def fELevel5 = new AtomicLong()
|
||||
|
||||
def rets = []
|
||||
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
|
||||
|
||||
|
||||
def index = new AtomicLong()
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
index.addAndGet(1)
|
||||
println index +" "+d.name
|
||||
def commid = d._id.toString()
|
||||
//房源
|
||||
def jsonObject1 = OpenApiClient.evaluaionList(commid,"20220201",60)
|
||||
if(jsonObject1 && jsonObject1.result) {
|
||||
def results = jsonObject1.result
|
||||
|
||||
if(zRegions.contains(d.region)){
|
||||
zCityDistTotal.addAndGet(1)
|
||||
if(results){
|
||||
if(results.size()<5){
|
||||
zELevel2.addAndGet(1)
|
||||
}else if(results.size()<20){
|
||||
zELevel3.addAndGet(1)
|
||||
}else if(results.size()<50){
|
||||
zELevel4.addAndGet(1)
|
||||
}else if(results.size()>=50){
|
||||
zELevel5.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
|
||||
if(results){
|
||||
if(results.size()<5){
|
||||
fELevel2.addAndGet(1)
|
||||
}else if(results.size()<20){
|
||||
fELevel3.addAndGet(1)
|
||||
}else if(results.size()<50){
|
||||
fELevel4.addAndGet(1)
|
||||
}else if(results.size()>=50){
|
||||
fELevel5.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(zRegions.contains(d.region)) {
|
||||
zCityDistTotal.addAndGet(1)
|
||||
zELevel1.addAndGet(1)
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
fELevel1.addAndGet(1)
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
def ret1 = ["主城区", zCityDistTotal.get(), zELevel1.get(), zELevel2.get(), zELevel3.get(), zELevel4.get(), zELevel5.get()]
|
||||
def ret2 = ["非主城区", fCityDistTotal.get(), fELevel1.get(), fELevel2.get(), fELevel3.get(), fELevel4.get(), fELevel5.get()]
|
||||
rets.add(ret1)
|
||||
rets.add(ret2)
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-评估")
|
||||
println "评估案例统计完成"
|
||||
}
|
||||
|
||||
def fileldStat(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
|
||||
def alias_count = new AtomicLong()
|
||||
def address_count = new AtomicLong()
|
||||
def block_count = new AtomicLong()
|
||||
def township_count = new AtomicLong()
|
||||
def price_count = new AtomicLong()
|
||||
def lng_count = new AtomicLong()
|
||||
def finishBuildTime_count = new AtomicLong()
|
||||
def developer_count = new AtomicLong()
|
||||
def green_count = new AtomicLong()
|
||||
def ratio_count = new AtomicLong()
|
||||
def carsite_count = new AtomicLong()
|
||||
def propertyManagement_count = new AtomicLong()
|
||||
def managementFee_count = new AtomicLong()
|
||||
def building_count = new AtomicLong()
|
||||
def holed_count = new AtomicLong()
|
||||
def car_holed_count = new AtomicLong()
|
||||
def all_area_count = new AtomicLong()
|
||||
def key_school_count = new AtomicLong()
|
||||
def school_count = new AtomicLong()
|
||||
def traffic_count = new AtomicLong()
|
||||
def market_count = new AtomicLong()
|
||||
def hospital_count = new AtomicLong()
|
||||
def bank_count = new AtomicLong()
|
||||
|
||||
def f_alias_count = new AtomicLong()
|
||||
def f_address_count = new AtomicLong()
|
||||
def f_block_count = new AtomicLong()
|
||||
def f_township_count = new AtomicLong()
|
||||
def f_price_count = new AtomicLong()
|
||||
def f_lng_count = new AtomicLong()
|
||||
def f_finishBuildTime_count = new AtomicLong()
|
||||
def f_developer_count = new AtomicLong()
|
||||
def f_green_count = new AtomicLong()
|
||||
def f_ratio_count = new AtomicLong()
|
||||
def f_carsite_count = new AtomicLong()
|
||||
def f_propertyManagement_count = new AtomicLong()
|
||||
def f_managementFee_count = new AtomicLong()
|
||||
def f_building_count = new AtomicLong()
|
||||
def f_holed_count = new AtomicLong()
|
||||
def f_car_holed_count = new AtomicLong()
|
||||
def f_all_area_count = new AtomicLong()
|
||||
def f_key_school_count = new AtomicLong()
|
||||
def f_school_count = new AtomicLong()
|
||||
def f_traffic_count = new AtomicLong()
|
||||
def f_market_count = new AtomicLong()
|
||||
def f_hospital_count = new AtomicLong()
|
||||
def f_bank_count = new AtomicLong()
|
||||
|
||||
def zCityDistTotal = new AtomicLong()
|
||||
def fCityDistTotal = new AtomicLong()
|
||||
def index = new AtomicLong()
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
index.addAndGet(1)
|
||||
println index +" "+dist.name
|
||||
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
|
||||
if(jsonObject) {
|
||||
|
||||
if(zRegions.contains(jsonObject.get("district"))){
|
||||
zCityDistTotal.addAndGet(1)
|
||||
if(jsonObject.get("alias")){
|
||||
alias_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(jsonObject.get("address")){
|
||||
address_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("commercialArea")){//板块
|
||||
block_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("township")){//街道***********
|
||||
township_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("avgPrice")) {//基准价
|
||||
price_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("lng")){//经纬度
|
||||
lng_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("finishBuildTime")){//竣工时间
|
||||
finishBuildTime_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("developer")){//开发商名称
|
||||
developer_count.addAndGet(1)
|
||||
}
|
||||
def green = SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
|
||||
if(green){
|
||||
green_count.addAndGet(1)
|
||||
}
|
||||
def ratio = SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
|
||||
if(ratio){
|
||||
ratio_count.addAndGet(1)
|
||||
}
|
||||
def carsite = SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
|
||||
if(carsite){
|
||||
carsite_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("propertyManagement")){//物业公司
|
||||
propertyManagement_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("managementFee")){//物业费
|
||||
managementFee_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("buildingCount")){//住宅幢数
|
||||
building_count.addAndGet(1)
|
||||
}
|
||||
def holed = SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||
if(holed){
|
||||
holed_count.addAndGet(1)
|
||||
}
|
||||
if(carsite && holed){
|
||||
car_holed_count.addAndGet(1)
|
||||
}
|
||||
def all_area = SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
|
||||
if(all_area){
|
||||
all_area_count.addAndGet(1)
|
||||
}
|
||||
}else{
|
||||
fCityDistTotal.addAndGet(1)
|
||||
if(jsonObject.get("alias")){
|
||||
f_alias_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(jsonObject.get("address")){
|
||||
f_address_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("commercialArea")){//板块
|
||||
f_block_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("township")){//街道***********
|
||||
f_township_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("avgPrice")) {//基准价
|
||||
f_price_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("lng")){//经纬度
|
||||
f_lng_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("finishBuildTime")){//竣工时间
|
||||
f_finishBuildTime_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("developer")){//开发商名称
|
||||
f_developer_count.addAndGet(1)
|
||||
}
|
||||
def green = SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
|
||||
if(green){
|
||||
f_green_count.addAndGet(1)
|
||||
}
|
||||
def ratio = SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
|
||||
if(ratio){
|
||||
f_ratio_count.addAndGet(1)
|
||||
}
|
||||
def carsite = SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
|
||||
if(carsite){
|
||||
f_carsite_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("propertyManagement")){//物业公司
|
||||
f_propertyManagement_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("managementFee")){//物业费
|
||||
f_managementFee_count.addAndGet(1)
|
||||
}
|
||||
if(jsonObject.get("buildingCount")){//住宅幢数
|
||||
f_building_count.addAndGet(1)
|
||||
}
|
||||
def holed = SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||
if(holed){
|
||||
f_holed_count.addAndGet(1)
|
||||
}
|
||||
if(carsite && holed){
|
||||
f_car_holed_count.addAndGet(1)
|
||||
}
|
||||
def all_area = SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
|
||||
if(all_area){
|
||||
f_all_area_count.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
|
||||
if(facilityObj && facilityObj.result){
|
||||
def result = facilityObj.result
|
||||
if(zRegions.contains(dist.region)){
|
||||
if(result.keySchool){
|
||||
key_school_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(result.school){
|
||||
school_count.addAndGet(1)
|
||||
}
|
||||
|
||||
def bus = []
|
||||
if(result.bus){
|
||||
bus.addAll(result.bus)
|
||||
}
|
||||
if(result.metro){
|
||||
bus.addAll(result.metro)
|
||||
}
|
||||
|
||||
if(bus){
|
||||
traffic_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(result.supermarket){
|
||||
market_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(result.hospital){
|
||||
hospital_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(result.bank){
|
||||
bank_count.addAndGet(1)
|
||||
}
|
||||
}else{
|
||||
if(result.keySchool){
|
||||
f_key_school_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(result.school){
|
||||
f_school_count.addAndGet(1)
|
||||
}
|
||||
|
||||
def bus = []
|
||||
if(result.bus){
|
||||
bus.addAll(result.bus)
|
||||
}
|
||||
if(result.metro){
|
||||
bus.addAll(result.metro)
|
||||
}
|
||||
|
||||
if(bus){
|
||||
f_traffic_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(result.supermarket){
|
||||
f_market_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(result.hospital){
|
||||
f_hospital_count.addAndGet(1)
|
||||
}
|
||||
|
||||
if(result.bank){
|
||||
f_bank_count.addAndGet(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
def rets = []
|
||||
rets << ["城区","小区名称","小区id","小区量","别名","地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构","楼盘每月均价走势","板块每月均价走势","行政区每月均价走势","城市每月均价走势"]
|
||||
|
||||
def alias_pre = DataHandleUtils.getDoubleRound(2, 100-alias_count*100/zCityDistTotal)
|
||||
def addr_pre = DataHandleUtils.getDoubleRound(2, 100-address_count*100/zCityDistTotal)
|
||||
def block_pre = DataHandleUtils.getDoubleRound(2, 100-block_count*100/zCityDistTotal)
|
||||
def town_pre = DataHandleUtils.getDoubleRound(2, 100-township_count*100/zCityDistTotal)
|
||||
def price_pre = DataHandleUtils.getDoubleRound(2, 100-price_count*100/zCityDistTotal)
|
||||
def lat_pre = DataHandleUtils.getDoubleRound(2, 100-lng_count*100/zCityDistTotal)
|
||||
def finish_pre = DataHandleUtils.getDoubleRound(2, 100-finishBuildTime_count*100/zCityDistTotal)
|
||||
def developer_pre = DataHandleUtils.getDoubleRound(2, 100-developer_count*100/zCityDistTotal)
|
||||
def green_pre = DataHandleUtils.getDoubleRound(2, 100-green_count*100/zCityDistTotal)
|
||||
def ratio_pre = DataHandleUtils.getDoubleRound(2, 100-ratio_count*100/zCityDistTotal)
|
||||
def carsite_pre = DataHandleUtils.getDoubleRound(2, 100-carsite_count*100/zCityDistTotal)
|
||||
def pm_pre = DataHandleUtils.getDoubleRound(2, 100-propertyManagement_count*100/zCityDistTotal)
|
||||
def mf_pre = DataHandleUtils.getDoubleRound(2, 100-managementFee_count*100/zCityDistTotal)
|
||||
def building_pre = DataHandleUtils.getDoubleRound(2, 100-building_count*100/zCityDistTotal)
|
||||
def holed_pre = DataHandleUtils.getDoubleRound(2, 100-holed_count*100/zCityDistTotal)
|
||||
def car_holed_pre = DataHandleUtils.getDoubleRound(2, 100-car_holed_count*100/zCityDistTotal)
|
||||
def all_area_pre = DataHandleUtils.getDoubleRound(2, 100-all_area_count*100/zCityDistTotal)
|
||||
def key_school_pre = DataHandleUtils.getDoubleRound(2, 100-key_school_count*100/zCityDistTotal)
|
||||
def school_pre = DataHandleUtils.getDoubleRound(2, 100-school_count*100/zCityDistTotal)
|
||||
def traffic_pre = DataHandleUtils.getDoubleRound(2, 100-traffic_count*100/zCityDistTotal)
|
||||
def market_pre = DataHandleUtils.getDoubleRound(2, 100-market_count*100/zCityDistTotal)
|
||||
def hosp_pre = DataHandleUtils.getDoubleRound(2, 100-hospital_count*100/zCityDistTotal)
|
||||
def bank_pre = DataHandleUtils.getDoubleRound(2, 100-bank_count*100/zCityDistTotal)
|
||||
def ret1 = ["主城区","0" ,"0",zCityDistTotal.get(), alias_pre,addr_pre,block_pre,town_pre,price_pre,lat_pre,
|
||||
finish_pre,developer_pre,green_pre,ratio_pre,carsite_pre,pm_pre,mf_pre,building_pre,
|
||||
holed_pre,car_holed_pre,all_area_pre,key_school_pre,school_pre,traffic_pre,market_pre,
|
||||
hosp_pre,bank_pre,price_pre,0,0,0]
|
||||
|
||||
|
||||
def f_alias_pre = DataHandleUtils.getDoubleRound(2, 100-f_alias_count*100/zCityDistTotal)
|
||||
def f_addr_pre = DataHandleUtils.getDoubleRound(2, 100-f_address_count*100/zCityDistTotal)
|
||||
def f_block_pre = DataHandleUtils.getDoubleRound(2, 100-f_block_count*100/zCityDistTotal)
|
||||
def f_town_pre = DataHandleUtils.getDoubleRound(2, 100-f_township_count*100/zCityDistTotal)
|
||||
def f_price_pre = DataHandleUtils.getDoubleRound(2, 100-f_price_count*100/zCityDistTotal)
|
||||
def f_lat_pre = DataHandleUtils.getDoubleRound(2, 100-f_lng_count*100/zCityDistTotal)
|
||||
def f_finish_pre = DataHandleUtils.getDoubleRound(2, 100-f_finishBuildTime_count*100/zCityDistTotal)
|
||||
def f_developer_pre = DataHandleUtils.getDoubleRound(2, 100-f_developer_count*100/zCityDistTotal)
|
||||
def f_green_pre = DataHandleUtils.getDoubleRound(2, 100-f_green_count*100/zCityDistTotal)
|
||||
def f_ratio_pre = DataHandleUtils.getDoubleRound(2, 100-f_ratio_count*100/zCityDistTotal)
|
||||
def f_carsite_pre = DataHandleUtils.getDoubleRound(2, 100-f_carsite_count*100/zCityDistTotal)
|
||||
def f_pm_pre = DataHandleUtils.getDoubleRound(2, 100-f_propertyManagement_count*100/zCityDistTotal)
|
||||
def f_mf_pre = DataHandleUtils.getDoubleRound(2, 100-f_managementFee_count*100/zCityDistTotal)
|
||||
def f_building_pre = DataHandleUtils.getDoubleRound(2, 100-f_building_count*100/zCityDistTotal)
|
||||
def f_holed_pre = DataHandleUtils.getDoubleRound(2, 100-f_holed_count*100/zCityDistTotal)
|
||||
def f_car_holed_pre = DataHandleUtils.getDoubleRound(2, 100-f_car_holed_count*100/zCityDistTotal)
|
||||
def f_all_area_pre = DataHandleUtils.getDoubleRound(2, 100-f_all_area_count*100/zCityDistTotal)
|
||||
def f_key_school_pre = DataHandleUtils.getDoubleRound(2, 100-f_key_school_count*100/zCityDistTotal)
|
||||
def f_school_pre = DataHandleUtils.getDoubleRound(2, 100-f_school_count*100/zCityDistTotal)
|
||||
def f_traffic_pre = DataHandleUtils.getDoubleRound(2, 100-f_traffic_count*100/zCityDistTotal)
|
||||
def f_market_pre = DataHandleUtils.getDoubleRound(2, 100-f_market_count*100/zCityDistTotal)
|
||||
def f_hosp_pre = DataHandleUtils.getDoubleRound(2, 100-f_hospital_count*100/zCityDistTotal)
|
||||
def f_bank_pre = DataHandleUtils.getDoubleRound(2, 100-f_bank_count*100/zCityDistTotal)
|
||||
|
||||
def ret2 = ["非主城区","0" ,"0",fCityDistTotal.get(), f_alias_pre,f_addr_pre,f_block_pre,f_town_pre,f_price_pre,f_lat_pre,
|
||||
f_finish_pre,f_developer_pre,f_green_pre,f_ratio_pre,f_carsite_pre,f_pm_pre,f_mf_pre,f_building_pre,
|
||||
f_holed_pre,f_car_holed_pre,f_all_area_pre,f_key_school_pre,f_school_pre,f_traffic_pre,f_market_pre,
|
||||
f_hosp_pre,f_bank_pre,f_price_pre,0,0,0]
|
||||
rets.add(ret1)
|
||||
rets.add(ret2)
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "字段-安")
|
||||
println "字段统计完成"
|
||||
|
||||
}
|
||||
}
|
||||
+153
-70
@@ -2,9 +2,11 @@ package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.util.SourceNameUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
@@ -15,28 +17,75 @@ class PocTest {
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
// match("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx")
|
||||
// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
|
||||
|
||||
// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
|
||||
// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
|
||||
// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
|
||||
/**
|
||||
* A 序号
|
||||
* B 城市
|
||||
* C 区域
|
||||
* D 地址
|
||||
* E 面积
|
||||
* @return
|
||||
*/
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\重庆农商行\\poc\\重庆农商行.xlsx"
|
||||
match(inputfile)
|
||||
addPrice(inputfile)
|
||||
addArroundPrice(inputfile)
|
||||
// matchMap(inputfile)
|
||||
|
||||
standard(inputfile)
|
||||
addLoudongNum(inputfile)
|
||||
|
||||
|
||||
|
||||
// addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
|
||||
//
|
||||
// createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
|
||||
// createPaimai(inputfile,"D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
|
||||
// createHouse(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||
// createDealPrice(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||
// createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||
}
|
||||
|
||||
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['P'] && row['C']){
|
||||
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null)
|
||||
if(!row['Q'] && row['D']){
|
||||
def city = row['B']
|
||||
def address = row['D']
|
||||
def region = row['C']
|
||||
println "city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['P'] = jsonObject.get("name")
|
||||
row['Q'] = jsonObject.get("id")
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['R'] = jsonObject.get("id")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchMap(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['Q'] && row['D']){
|
||||
def city = row['B']
|
||||
def address = row['D']
|
||||
def region = row['C']
|
||||
println "city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
def propertyType = jsonObject.get("propertyType")
|
||||
if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['R'] = new ObjectId().toString()
|
||||
row['W'] = propertyType
|
||||
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(city, address)
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['F'] = priceObj.unitPrice
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -44,34 +93,51 @@ class PocTest {
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['Q']){
|
||||
def dist = CommonDatautils.findDistrictById(row['Q'])
|
||||
def commid = row['R']
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid)
|
||||
if(dist){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['E'] = priceObj.avg_price
|
||||
row['F'] = priceObj.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addArroundPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['F'] = priceObj.unitPrice
|
||||
row['W'] = "周边均价"
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def standard(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['C']){
|
||||
def jsonObject = OpenApiClient.standard2Action(row['A'], row['C'])
|
||||
def city = row['B']
|
||||
def address = row['D']
|
||||
if(address){
|
||||
println "[standard] address:"+address
|
||||
def jsonObject = OpenApiClient.standard2Action(city, address)
|
||||
if(jsonObject){
|
||||
println jsonObject
|
||||
row['F'] = jsonObject.get("province")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("township")
|
||||
row['J'] = jsonObject.get("street")
|
||||
row['K'] = jsonObject.get("streetNumber")
|
||||
row['L'] = jsonObject.get("buildingNumber")
|
||||
row['M'] = jsonObject.get("unitNumber")
|
||||
row['N'] = jsonObject.get("floor")
|
||||
row['O'] = jsonObject.get("roomNumber")
|
||||
row['G'] = jsonObject.get("province")
|
||||
row['H'] = jsonObject.get("city")
|
||||
row['I'] = jsonObject.get("district")
|
||||
row['J'] = jsonObject.get("township")
|
||||
row['K'] = jsonObject.get("street")
|
||||
row['L'] = jsonObject.get("streetNumber")
|
||||
row['M'] = jsonObject.get("buildingNumber")
|
||||
row['N'] = jsonObject.get("unitNumber")
|
||||
row['O'] = jsonObject.get("floor")
|
||||
row['P'] = jsonObject.get("roomNumber")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,23 +147,27 @@ class PocTest {
|
||||
def addLoudongNum(path){
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if (row['Q'] && row['L']) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
|
||||
def commid = row['R']
|
||||
def houseNum = row['M']
|
||||
def unitNum = row['N']
|
||||
println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum
|
||||
if (commid && houseNum) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum)
|
||||
if (jsonObject && jsonObject.result) {
|
||||
if(jsonObject.result.size()>=1){
|
||||
def obj = jsonObject.result[0]
|
||||
row['S'] = obj.id
|
||||
row['R'] = obj.houseNumber
|
||||
row['T'] = obj.id
|
||||
row['S'] = obj.houseNumber
|
||||
}
|
||||
}
|
||||
|
||||
if(row['O']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
|
||||
if(row['P']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P'])
|
||||
if (jsonObject1 && jsonObject1.result) {
|
||||
if(jsonObject.result.size()>=1){
|
||||
def obj = jsonObject.result[0]
|
||||
row['U'] = obj.id
|
||||
row['T'] = obj.roomNum
|
||||
if(jsonObject1.result.size()>=1){
|
||||
def obj = jsonObject1.result[0]
|
||||
row['V'] = obj.id
|
||||
row['U'] = obj.roomNum
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -111,12 +181,12 @@ class PocTest {
|
||||
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
|
||||
for(def row : datas){
|
||||
def data = []
|
||||
data << row['A']
|
||||
data << row['B']
|
||||
data << row['C']
|
||||
data << row['D']
|
||||
if (row['Q'] && row['L']) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
|
||||
data << row['E']
|
||||
if (row['R'] && row['M']) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N'])
|
||||
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
|
||||
def obj = jsonObject.result[0]
|
||||
data << obj.houseNumber
|
||||
@@ -144,8 +214,8 @@ class PocTest {
|
||||
data << ""//景观
|
||||
}
|
||||
|
||||
if(row['O']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
|
||||
if(row['P']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P'])
|
||||
if (jsonObject1 && jsonObject1.result) {
|
||||
if(jsonObject1.result.size()>=1){
|
||||
def obj = jsonObject1.result[0]
|
||||
@@ -174,18 +244,20 @@ class PocTest {
|
||||
def rows = []
|
||||
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
|
||||
for(def d : datas){
|
||||
if(d['Q']){
|
||||
def dist = CommonDatautils.findDistrictById(d['Q'])
|
||||
def commid = d['R']
|
||||
println "[addDetail] commid:"+commid
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid)
|
||||
if(dist){
|
||||
def row = []
|
||||
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
|
||||
if(jsonObject) {
|
||||
|
||||
row << d['A']
|
||||
row << d['B']
|
||||
row << d['C']
|
||||
row << d['D']
|
||||
row << jsonObject.get("name")//小区名称
|
||||
row << d['Q']//小区ID
|
||||
row << commid//小区ID
|
||||
if(jsonObject.get("alias")){
|
||||
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
|
||||
}else{
|
||||
@@ -207,7 +279,7 @@ class PocTest {
|
||||
row << jsonObject.get("buildingCount")//住宅幢数
|
||||
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
|
||||
row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积
|
||||
row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
|
||||
|
||||
}
|
||||
|
||||
@@ -271,28 +343,35 @@ class PocTest {
|
||||
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
if(d['Q']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['Q'])
|
||||
def commid = d['R']
|
||||
println "[createPaimai] commid:"+commid
|
||||
if(commid) {
|
||||
def dist = CommonDatautils.findDistrictById(commid)
|
||||
if (dist) {
|
||||
def key = dist.city+"#"+dist.region+"#"+dist.name
|
||||
if(keys.contains(key)){
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10)
|
||||
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100)
|
||||
if(jsonObject && jsonObject.result){
|
||||
def results = jsonObject.result
|
||||
for(def r: results){
|
||||
def row = []
|
||||
row << d['A']
|
||||
row << d['B']
|
||||
row << d['C']
|
||||
row << d['D']
|
||||
row << r.matchName
|
||||
row << r.address
|
||||
row << r.houseNum
|
||||
row << r.roomNum
|
||||
row << r.area
|
||||
row << r.startDate
|
||||
if(r.startDate){
|
||||
row << r.startDate.substring(0, r.startDate.indexOf("T"))
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << r.count
|
||||
row << r.status
|
||||
row << r.tradeAvgPrice
|
||||
@@ -314,13 +393,15 @@ class PocTest {
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def commid = d['R']
|
||||
// println "[createHouse] commid:"+commid
|
||||
if (commid) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20)
|
||||
def jsonObject = OpenApiClient.houseList(commid,"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
@@ -368,7 +449,7 @@ class PocTest {
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
println d['R'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -381,13 +462,14 @@ class PocTest {
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
def commid= d['R']
|
||||
println "[createEvaluation] commid:"+commid
|
||||
if (commid) {
|
||||
if (keys.contains(commid)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20)
|
||||
keys << commid
|
||||
def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
@@ -432,7 +514,7 @@ class PocTest {
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
println d['R'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -446,13 +528,14 @@ class PocTest {
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
if (d['Q']) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
def commid= d['R']
|
||||
println "[createDealPrice] commid:"+commid
|
||||
if (commid) {
|
||||
if (keys.contains(commid)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20)
|
||||
keys << commid
|
||||
def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
@@ -501,7 +584,7 @@ class PocTest {
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['Q'] +" "+results.size()
|
||||
println d['R'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+62
-10
@@ -68,6 +68,10 @@ class CommonDatautils {
|
||||
|
||||
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
|
||||
|
||||
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
|
||||
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
|
||||
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
return house_dataStoreConnection
|
||||
@@ -586,8 +590,8 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrict(String city, String region, String name, String cat){
|
||||
|
||||
public static def findDistrictCount(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
if(region){
|
||||
@@ -598,7 +602,24 @@ class CommonDatautils {
|
||||
q.put("cat", cat)
|
||||
q.put("status", 0)
|
||||
|
||||
|
||||
def count = getDataStoreConnection().count(q);
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrict(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
}
|
||||
|
||||
q.put("name", name);
|
||||
q.put("cat", cat)
|
||||
q.put("status", 0)
|
||||
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
@@ -609,6 +630,7 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def findDistrictList(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -704,7 +726,7 @@ class CommonDatautils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","吴中路9弄小区")
|
||||
// q.put("name","ÉÍû·¼²ÝÂ̰¶")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -827,16 +849,24 @@ class CommonDatautils {
|
||||
keys.put("out_pics", 1);
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("lat2", 1);
|
||||
keys.put("lng2", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
keys.put("sid", 1)
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
try{
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
println "error_id:"+id
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1053,9 +1083,16 @@ class CommonDatautils {
|
||||
|
||||
|
||||
public static def queryDistrictById(def id){
|
||||
def dbo = null
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", new ObjectId(id));
|
||||
def dbo = getDataStoreConnection().findOne(q);
|
||||
try{
|
||||
q.put("_id", new ObjectId(id.trim().toString()));
|
||||
dbo = getDataStoreConnection().findOne(q);
|
||||
}catch(Exception e){
|
||||
println id
|
||||
}
|
||||
|
||||
|
||||
return dbo;
|
||||
}
|
||||
|
||||
@@ -2385,6 +2422,7 @@ class CommonDatautils {
|
||||
q.put("block", block);
|
||||
}
|
||||
q.put("cat", cat);
|
||||
q.put("status",0)
|
||||
def cur = getDataStoreConnection().find(q);
|
||||
return cur.count();
|
||||
}
|
||||
@@ -3184,5 +3222,19 @@ class CommonDatautils {
|
||||
def avg_radios = (int)(radios.sum()/radios.size())
|
||||
return avg_radios;
|
||||
}
|
||||
|
||||
static def findLoudong(commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("id_in_seaweed", commid)
|
||||
def count = LOUDONG_LINE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
static def findHouseLoudong(commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("district_id", commid)
|
||||
def count = HOUSE_LOUDONG_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+12
-8
@@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{
|
||||
def run(){
|
||||
def cityMap = CommonDatautils.getAllCityRegions()
|
||||
cityMap.each{city,regions->
|
||||
findPriceByCity(city, regions);
|
||||
|
||||
findPriceByCity(city, regions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{
|
||||
q.put("region" , r.region)
|
||||
q.put("month", r.month)
|
||||
def mom = null
|
||||
if(lastObj){
|
||||
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
|
||||
mom = DataHandleUtils.getDoubleRound(4, mom)
|
||||
if(lastObj && lastObj.avg_price && r.price){
|
||||
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
|
||||
mom = DataHandleUtils.getDoubleRound(4, mom)
|
||||
}
|
||||
if(obj){
|
||||
if(obj.avg_price != r.price){
|
||||
@@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{
|
||||
|
||||
def findEsAvgPrice(city, region){
|
||||
def detail = [:]
|
||||
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1])
|
||||
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0])
|
||||
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
|
||||
|
||||
def must = []
|
||||
@@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{
|
||||
if(!priceValue){
|
||||
priceValue = row['1-metric']
|
||||
}
|
||||
if(priceValue){
|
||||
if(priceValue && priceValue.value){
|
||||
price = priceValue.value.toInteger()
|
||||
}
|
||||
result.price = price;
|
||||
def date = row.key_as_string.replace("-","").substring(0, 6)
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
|
||||
if(date!=curMonth){
|
||||
result.month = date.toInteger()
|
||||
rets << result
|
||||
if(date.toInteger()>200001 && result.price){
|
||||
result.month = date.toInteger()
|
||||
rets << result
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class AnjukeAppCrawl {
|
||||
|
||||
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeAppCrawl();
|
||||
task.run();
|
||||
// task.run2()
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def url = "https://m.anjuke.com/fz/community/"
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def links = getRegion(url, driver)
|
||||
for(def link : links){
|
||||
def doc = getDocument(url, driver)
|
||||
def str = doc.toString().substring(doc.toString().indexOf("<script> window.__NUXT__"), doc.toString().indexOf("</script>"))
|
||||
if(str){
|
||||
str = str.substring(str.indexOf("} ("), str.indexOf("));"))
|
||||
def arrs = str.split(",")
|
||||
pritln arrs.size()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def getRegion(url, driver){
|
||||
//def doc = getDocument(url, driver)
|
||||
def header = [:]
|
||||
header.put("accept","application/json, text/plain, */*")
|
||||
header.put("accept-encoding","gzip, deflate, br")
|
||||
header.put("accept-language","zh-CN,zh;q=0.9")
|
||||
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
|
||||
header.put("referer","https://m.anjuke.com/fz/")
|
||||
header.put("sec-fetch-dest","empty")
|
||||
header.put("sec-fetch-mode","cors")
|
||||
header.put("sec-fetch-site","same-origin")
|
||||
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
|
||||
header.put("sw8-correlation","c291cmNl:YmVpZG91")
|
||||
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
|
||||
|
||||
def doc = ABuYunUtil.doGet(url, null, header, "utf-8")
|
||||
def links = []
|
||||
if(doc){
|
||||
def eles = JQueryUtils.select(".column2 a", doc)
|
||||
for(def e : eles){
|
||||
def href = e.attr("href")
|
||||
links << href
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
//"仓山":4071,
|
||||
def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
|
||||
map.each{region, area_id->
|
||||
for(def i=1;i<40;i++){
|
||||
def stop = false;
|
||||
def all_stop = false
|
||||
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
|
||||
//城口
|
||||
def flagMap = getInfo(stop, i, area_id)
|
||||
stop = flagMap.stop
|
||||
all_stop = flagMap.all_stop
|
||||
}
|
||||
if(all_stop){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getInfo(def stop, def i, def area_id){
|
||||
|
||||
def map = [:]
|
||||
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=35&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
|
||||
|
||||
def header = [:]
|
||||
header.put("accept","application/json, text/plain, */*")
|
||||
header.put("accept-encoding","gzip, deflate, br")
|
||||
header.put("accept-language","zh-CN,zh;q=0.9")
|
||||
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
|
||||
header.put("referer","https://m.anjuke.com/fz/community/")
|
||||
header.put("sec-fetch-dest","empty")
|
||||
header.put("sec-fetch-mode","cors")
|
||||
header.put("sec-fetch-site","same-origin")
|
||||
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
|
||||
header.put("sw8-correlation","c291cmNl:YmVpZG91")
|
||||
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
|
||||
|
||||
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
|
||||
def all_stop = false;
|
||||
if (ret) {
|
||||
JSONObject jsonObj = JSONObject.fromObject(ret);
|
||||
if(jsonObj && jsonObj.status=="ok" ){
|
||||
def data = jsonObj.data
|
||||
if(data && data.total.toInteger()>0){
|
||||
def communities = data.communities
|
||||
|
||||
println "【${i}】 "+communities.size() +" "+url
|
||||
if(communities.size()>0){
|
||||
save("福州", communities)
|
||||
Thread.sleep(10000)
|
||||
stop = true;
|
||||
}else{
|
||||
stop = true;
|
||||
println url +" 返回0"
|
||||
all_stop = true;
|
||||
}
|
||||
|
||||
}else{
|
||||
stop = false;
|
||||
println url +" 无数据"
|
||||
}
|
||||
|
||||
|
||||
}else{
|
||||
stop = false;
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}
|
||||
|
||||
map.stop = stop
|
||||
map.all_stop = all_stop;
|
||||
return map;
|
||||
}
|
||||
|
||||
def save(city, def ret){
|
||||
def index = 0
|
||||
for(def o : ret){
|
||||
index++
|
||||
def obj = findIsExist(o.jumpAction)
|
||||
if(!obj){
|
||||
def udate = new BasicDBObject()
|
||||
o.each{k,v->
|
||||
udate.put(k,v)
|
||||
}
|
||||
udate.put("city", city)
|
||||
udate.put("c_date", new Date())
|
||||
detail.save(udate)
|
||||
println index+" [保存]"+o.jumpAction
|
||||
}else{
|
||||
println index+" [已存在]"+o.jumpAction
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def findIsExist(def url){
|
||||
def q = new BasicDBObject()
|
||||
q.put("jumpAction",url)
|
||||
return detail.findOne(q);
|
||||
}
|
||||
}
|
||||
+173
-61
@@ -11,9 +11,15 @@ import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.remote.DesiredCapabilities
|
||||
import org.openqa.selenium.remote.RemoteWebDriver
|
||||
|
||||
class AnjukeDetailCrawl {
|
||||
|
||||
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||
|
||||
DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeDetailCrawl();
|
||||
task.run();
|
||||
@@ -21,9 +27,15 @@ class AnjukeDetailCrawl {
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://chongqing.anjuke.com/community/?from=navigation"
|
||||
def url = "https://chongqing.anjuke.com/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
// DesiredCapabilities capabilities = DesiredCapabilities.chrome();
|
||||
// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL
|
||||
// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities);
|
||||
|
||||
driver.manage().deleteAllCookies()
|
||||
def regions = getRegions(driver, url)
|
||||
println "区域链接: "+regions.size()
|
||||
getAllRegions(driver, regions)
|
||||
|
||||
}
|
||||
@@ -31,44 +43,88 @@ class AnjukeDetailCrawl {
|
||||
|
||||
def getAllRegions(driver, regions){
|
||||
for(def url : regions){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = getDocument(url, driver)
|
||||
def total = getTotal(doc)
|
||||
if(total) {
|
||||
if (total > 1250) {
|
||||
|
||||
def links = getMoreLinks(url, driver)
|
||||
println url +" "+links.size()
|
||||
saveDetailLinks(links)
|
||||
// parse(links,driver)
|
||||
} else {
|
||||
def links = getAllLinks(url, driver)
|
||||
parse(links,driver)
|
||||
println url +" "+links.size()
|
||||
saveDetailLinks(links)
|
||||
// parse(links,driver)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def saveDetailLinks(links){
|
||||
for(def url : links){
|
||||
def data = new BasicDBObject()
|
||||
data.put("city","重庆")
|
||||
data.put("url", url)
|
||||
data.put("source", "安居客")
|
||||
data.put("c_date", new Date())
|
||||
urlColl.save(data)
|
||||
}
|
||||
}
|
||||
|
||||
def getMoreLinks(url, driver){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc)
|
||||
def links = []
|
||||
if(pricelinks){
|
||||
for(def link : pricelinks){
|
||||
if(link.text()!="不限"){
|
||||
def urlLink = link.attr("href")
|
||||
getDocument(urlLink, driver)
|
||||
// driver.get(urlLink)
|
||||
links.addAll(getAllLinks(urlLink, driver))
|
||||
println urlLink +" "+links.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
def getAllLinks(url, driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
while(!stop) {
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def total = getTotal(doc)
|
||||
def page = getPage(total, 25)
|
||||
def pages = []
|
||||
for(def i=1;i<=page;i++){
|
||||
pages << url.substring(0, url.size()-1)+"-p"+i+"/"
|
||||
}
|
||||
for(def pageUrl : pages){
|
||||
doc = getDocument(pageUrl, driver)
|
||||
def details = JQueryUtils.select(".list-cell a.li-row", doc)
|
||||
|
||||
for (def d : details) {
|
||||
urls << d.attr("href")
|
||||
}
|
||||
def nextEle = JQueryUtils.select(".next", doc)
|
||||
if (nextEle) {
|
||||
driver.findElement(By.cssSelector(".next")).click()
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
if(urls.size()>20){
|
||||
stop = true
|
||||
def link = d.attr("href")
|
||||
if(!urls.contains(link)){
|
||||
urls << d.attr("href")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return urls
|
||||
}
|
||||
|
||||
def getPage(def total, def pageSize){
|
||||
if(total && total.toInteger()>0){
|
||||
def page = total.toInteger()/pageSize;
|
||||
if(page>(int)page){
|
||||
page = (int)page + 1
|
||||
return page
|
||||
}
|
||||
return (int)page;
|
||||
}
|
||||
return null
|
||||
}
|
||||
def getTotal(doc){
|
||||
|
||||
def totalText = JQueryUtils.selectText(".total-info", doc)
|
||||
@@ -82,9 +138,19 @@ class AnjukeDetailCrawl {
|
||||
}
|
||||
|
||||
def getRegions(driver, url){
|
||||
// def doc = getDocument(url, driver)
|
||||
driver.get(url)
|
||||
Thread.sleep(3000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def els = JQueryUtils.select("a:contains(小区)", doc)
|
||||
if(els){
|
||||
def href = els[0].attr("href")
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
// driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click()
|
||||
// driver.findElement(By.linkText("小区")).click()
|
||||
doc = getDocument(href, driver)
|
||||
}
|
||||
|
||||
|
||||
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
|
||||
// if(loginIfm){
|
||||
// def href = loginIfm[0].attr("src")
|
||||
@@ -107,23 +173,35 @@ class AnjukeDetailCrawl {
|
||||
def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
||||
def urls = []
|
||||
for(def e : regionEles){
|
||||
urls << e.attr("href")
|
||||
if(e.text()!="不限"){
|
||||
urls << e.attr("href")
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def getNontCheckPage(url, driver, doc){
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
if(doc){
|
||||
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
def parse(def links, driver){
|
||||
def rets = []
|
||||
for(def link : links){
|
||||
def district = parsing(link, driver)
|
||||
if(district){
|
||||
rets << district
|
||||
// rets << district
|
||||
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
rets = []
|
||||
}
|
||||
// if (rets.size() % 100 == 0) {
|
||||
saveObj(district)
|
||||
println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
// rets = []
|
||||
// }
|
||||
}
|
||||
}
|
||||
if (rets.size()> 0) {
|
||||
@@ -136,33 +214,61 @@ class AnjukeDetailCrawl {
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = detailColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
detailColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
detailColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
}
|
||||
saveObj(ret)
|
||||
}
|
||||
return add
|
||||
}
|
||||
|
||||
|
||||
def saveObj(def ret){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = detailColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
detailColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
detailColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
}
|
||||
}
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
|
||||
def parsing(url, driver){
|
||||
|
||||
@@ -191,10 +297,9 @@ class AnjukeDetailCrawl {
|
||||
|
||||
def infoDiv = JQueryUtils.select(".info-list", doc)
|
||||
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
|
||||
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
|
||||
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
|
||||
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
|
||||
|
||||
district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text())
|
||||
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
|
||||
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
|
||||
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
|
||||
if (houseCountStr) {
|
||||
houseCountStr = houseCountStr.replace("户", "")
|
||||
@@ -202,9 +307,13 @@ class AnjukeDetailCrawl {
|
||||
district.house_count = houseCountStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
|
||||
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
|
||||
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
|
||||
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text())
|
||||
|
||||
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
|
||||
district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text())
|
||||
|
||||
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
|
||||
if(carSiteStr){
|
||||
if(carSiteStr.contains("(")){
|
||||
@@ -214,16 +323,19 @@ class AnjukeDetailCrawl {
|
||||
district.car_site = carSiteStr
|
||||
}
|
||||
}
|
||||
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
|
||||
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
|
||||
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
|
||||
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
|
||||
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
|
||||
district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text()
|
||||
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
|
||||
|
||||
district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
|
||||
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text())
|
||||
|
||||
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
|
||||
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
|
||||
district.url = url
|
||||
district.c_date = new Date()
|
||||
district.m_date = new Date()
|
||||
district.crawl_type = "chrome"
|
||||
district.root = "anjuke.com"
|
||||
|
||||
stop = true;
|
||||
}else{
|
||||
Thread.sleep(3000)
|
||||
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
class AnjukeDistrictMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeDistrictMatch();
|
||||
task.run();
|
||||
// task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def regionPointsMap = findRegionRange("重庆")
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
}
|
||||
def run(){
|
||||
|
||||
|
||||
|
||||
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath,{row->
|
||||
if(row['A'] && !row['J']){
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = row['D']
|
||||
def address = row['E']
|
||||
// def flag = "别名匹配"
|
||||
|
||||
|
||||
// def dist = matchDist(city, null, name)
|
||||
// if(!row['J'] && name.contains("(")){
|
||||
// name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||
// flag = "去括号匹配"
|
||||
// dist = matchDist(city, region, name)
|
||||
// }
|
||||
// if(!dist){
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
|
||||
// flag = "映射别名匹配"
|
||||
// }
|
||||
// if(!dist) {
|
||||
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }
|
||||
|
||||
//
|
||||
// if(!dist){
|
||||
// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district")
|
||||
// flag = "地址匹配"
|
||||
// if(dists.size()>1){
|
||||
// println address +" 地址相同:"+dists.size()
|
||||
// }else{
|
||||
// if(dists.size()==1){
|
||||
// dist = dists[0]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(dist){
|
||||
// row['J'] = flag
|
||||
// row['K'] = dist._id.toString()
|
||||
// row['L'] = dist.region
|
||||
// row['M'] = dist.name
|
||||
// row['N'] = dist.address
|
||||
// }
|
||||
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// row['J'] = "匹配接口"
|
||||
// row['K'] = jsonObject.get("id")
|
||||
// row['L'] = jsonObject.get("district")
|
||||
// row['M'] = jsonObject.get("name")
|
||||
// row['N'] = jsonObject.get("address")
|
||||
row['M'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchDist(city, region, name){
|
||||
def dist = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
return dist;
|
||||
}
|
||||
}
|
||||
+628
@@ -0,0 +1,628 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class AnjukeToDB {
|
||||
|
||||
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeToDB();
|
||||
// task.run();
|
||||
// task.addNoMatch()
|
||||
task.updateEstateType()
|
||||
// task.gaodeRangeData()
|
||||
}
|
||||
|
||||
def updateEstateType(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1)
|
||||
for(def row : datas){
|
||||
def anjukeId = row['A']
|
||||
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||
if(obj) {
|
||||
def type = getEstatType(obj.extend.type)
|
||||
def commId = row['L']
|
||||
if(!commId){
|
||||
commId = row['G']
|
||||
}
|
||||
if(type){
|
||||
if(type=="别墅"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("estate_type2", type)
|
||||
udata.put("m_date", new Date())
|
||||
println commId
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}else{
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("status", 1)
|
||||
// udata.put("name_cat",11)
|
||||
// udata.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def gaodeRangeData(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1)
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
index++
|
||||
println index
|
||||
def personName = d['AB']
|
||||
if(personName){
|
||||
def commid = d['J']
|
||||
if(commid){
|
||||
def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district")
|
||||
if(obj){
|
||||
updateDist(d, obj._id.toString())
|
||||
}
|
||||
}else{
|
||||
saveGaode(d)
|
||||
}
|
||||
}else{
|
||||
def commid = d['J']
|
||||
updateDist(d, commid)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def saveGaode(d){
|
||||
def oldName = d['E']
|
||||
def distName = nameFileter(d['C'], oldName)
|
||||
def udata = new BasicDBObject()
|
||||
def alias = []
|
||||
|
||||
def aliasStr = d['F']
|
||||
if(aliasStr){
|
||||
def arrs = aliasStr.split("/")
|
||||
for(def s : arrs){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
if(alias.size()>0){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
|
||||
def address = AddressUtil.formatAddress("重庆", d['G'])
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
if(d['N']){
|
||||
udata.put("virescence_percent", d['N'])
|
||||
}
|
||||
|
||||
if(d['O']){
|
||||
udata.put("estate_charge", d['O'])
|
||||
}
|
||||
if(d['Q']){
|
||||
udata.put("cubage_percent", d['Q'])
|
||||
}
|
||||
if(d['R']){
|
||||
udata.put("developer", d['R'])
|
||||
}
|
||||
|
||||
if( d['S']){
|
||||
udata.put("build_area", d['S'])
|
||||
}
|
||||
|
||||
if(d['T']){
|
||||
def t = d['T'].substring(0, 3)
|
||||
udata.put("build_date", t)
|
||||
}
|
||||
|
||||
if(d['V'] && d['V'].isNumber()){
|
||||
udata.put("car_site_count", d['V'].toInteger())
|
||||
}
|
||||
|
||||
if(d['Y']){
|
||||
def build_type = getBuildType2(d['Y'])
|
||||
if(build_type){
|
||||
udata.put("build_type", build_type)
|
||||
}
|
||||
}
|
||||
|
||||
if(d['Z']){
|
||||
udata.put("property_life", d['Z'])
|
||||
}
|
||||
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(d['H']){
|
||||
lat = d['H']
|
||||
lng = d['I']
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
|
||||
Long dLat2 = lat2 * 10000000000L
|
||||
Long dLng2 = lng2 * 10000000000L
|
||||
udata.put("lat2", dLat2)
|
||||
udata.put("lng2", dLng2)
|
||||
Long dlat = lat.toDouble() * 10000000000L
|
||||
Long dlng = lng.toDouble() * 10000000000L
|
||||
udata.put("lat", dlat)
|
||||
udata.put("lng", dlng)
|
||||
}
|
||||
def type = "普通住宅"
|
||||
if(distName.contains("别墅")) {
|
||||
type = "别墅"
|
||||
}else if(distName.contains("商住")){
|
||||
type = "商住"
|
||||
}
|
||||
|
||||
if(udata){
|
||||
udata.put("name_cat",0)
|
||||
udata.put("status",0)
|
||||
udata.put("cat","district")
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("city", "重庆")
|
||||
udata.put("region",d['C'])
|
||||
udata.put("name", distName)
|
||||
udata.put("estate_type2",type)
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
udata.put("mark_map",markMap)
|
||||
DbUtils.seaweed.save(udata);
|
||||
|
||||
def stop = false
|
||||
while (!stop){
|
||||
if(udata._id.toString()){
|
||||
addPrice(udata)
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateDist(d, commid){
|
||||
|
||||
if(commid){
|
||||
def dist = CommonDatautils.queryDistrictById(commid)
|
||||
if(dist){
|
||||
def oldName = d['E']
|
||||
def distName = nameFileter(d['C'], oldName)
|
||||
def udata = new BasicDBObject()
|
||||
if(!distName.contains("号")){
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(distName) && dist.name!=distName){
|
||||
alias << distName
|
||||
}
|
||||
def aliasStr = d['F']
|
||||
if(aliasStr){
|
||||
def arrs = aliasStr.split("/")
|
||||
for(def s : arrs){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
if(alias.size()>0 && alias!=dist.alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
}
|
||||
|
||||
if(dist.address && !dist.address.contains("号")){
|
||||
def address = AddressUtil.formatAddress("重庆", d['G'])
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
|
||||
if(!dist.virescence_percent && d['N']){
|
||||
udata.put("virescence_percent", d['N'])
|
||||
}
|
||||
|
||||
if(!dist.estate_charge && d['O']){
|
||||
udata.put("estate_charge", d['O'])
|
||||
}
|
||||
if(!dist.cubage_percent && d['Q']){
|
||||
udata.put("cubage_percent", d['Q'])
|
||||
}
|
||||
if(!dist.developer && d['R']){
|
||||
udata.put("developer", d['R'])
|
||||
}
|
||||
|
||||
if(!dist.build_area && d['S']){
|
||||
udata.put("build_area", d['S'])
|
||||
}
|
||||
|
||||
if(!dist.build_date && d['T']){
|
||||
if(d['T'].size()>=4){
|
||||
def t = d['T'].substring(0, 4)
|
||||
udata.put("build_date", t)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!dist.car_site_count && d['V'] && d['V'].isNumber()){
|
||||
udata.put("car_site_count", d['V'].toInteger())
|
||||
}
|
||||
|
||||
if(!dist.build_type && d['Y']){
|
||||
def build_type = getBuildType2(d['Y'])
|
||||
if(build_type){
|
||||
udata.put("build_type", build_type)
|
||||
}
|
||||
}
|
||||
|
||||
if(!dist.property_life && d['Z']){
|
||||
udata.put("property_life", d['Z'])
|
||||
}
|
||||
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(d['H']){
|
||||
lat = d['H']
|
||||
lng = d['I']
|
||||
|
||||
def slat = dist.lat/10000000000.0
|
||||
def slng = dist.lng/10000000000.0
|
||||
|
||||
def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat);
|
||||
if(distance>1000){
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
|
||||
Long dLat2 = lat2 * 10000000000L
|
||||
Long dLng2 = lng2 * 10000000000L
|
||||
udata.put("lat2", dLat2)
|
||||
udata.put("lng2", dLng2)
|
||||
Long dlat = lat.toDouble() * 10000000000L
|
||||
Long dlng = lng.toDouble() * 10000000000L
|
||||
udata.put("lat", dlat)
|
||||
udata.put("lng", dlng)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改] ${dist.name}"+ udata
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addNoMatch(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1)
|
||||
for(def d : datas){
|
||||
add(d)
|
||||
}
|
||||
}
|
||||
def run(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-芝芝.xlsx",0,1)
|
||||
for(def data : datas){
|
||||
if(data['K']=="芝芝"){
|
||||
def flag = data['M']
|
||||
if(flag && flag=="新增"){
|
||||
add(data)
|
||||
}else{
|
||||
update(data)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def add(def row){
|
||||
def anjukeId = row['A']
|
||||
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||
if(obj){
|
||||
def type = getEstatType(obj.extend.type)
|
||||
if(type){
|
||||
def city = obj.base.cityName
|
||||
def region = obj.base.areaName
|
||||
def lat2 = null
|
||||
def lng2 = null
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(obj.base.lat){
|
||||
lat2 = obj.base.lat
|
||||
lng2 = obj.base.lng
|
||||
|
||||
JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu")
|
||||
if(convertObj.getInt("status") == 1) {
|
||||
String lo = convertObj.get("locations")
|
||||
lng = Double.parseDouble(lo.split(",")[0])
|
||||
lat = Double.parseDouble(lo.split(",")[1])
|
||||
}
|
||||
}
|
||||
|
||||
def distName = nameFileter(obj.base.areaName, obj.base.name)
|
||||
|
||||
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
|
||||
|
||||
def address = AddressUtil.formatAddress(obj.base.city, obj.base.address)
|
||||
|
||||
def dist = CommonDatautils.findDistrict(city, region, distName, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",region)
|
||||
data.put("name", distName)
|
||||
data.put("estate_type2",type)
|
||||
if(address){
|
||||
data.put("address", obj.base.address)
|
||||
}
|
||||
|
||||
if(lat){
|
||||
Long alat = lat*10000000000L
|
||||
Long alng = lng*10000000000L
|
||||
data.put("lat", alat)
|
||||
data.put("lng", alng)
|
||||
Long alat2 = lat2.toDouble()*10000000000L
|
||||
Long alng2 = lng2.toDouble()*10000000000L
|
||||
data.put("lat2",alat2)
|
||||
data.put("lng2",alng2)
|
||||
}
|
||||
data.put("finish_building_date",obj.extend.completionTime)
|
||||
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){
|
||||
data.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
|
||||
}
|
||||
data.put("build_area", obj.extend.totalArea)
|
||||
|
||||
data.put("virescence_percent", obj.extend.landscapingRatio)
|
||||
data.put("build_type", buildTypeStr)
|
||||
data.put("business_quarter",obj.base.tradingAreaName)
|
||||
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
DbUtils.seaweed.save(data);
|
||||
|
||||
def stop = false
|
||||
while (!stop){
|
||||
if(data._id.toString()){
|
||||
addPrice(data)
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}else{
|
||||
println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type
|
||||
}
|
||||
}else{
|
||||
println anjukeId+" 没找到"
|
||||
}
|
||||
|
||||
}
|
||||
def update(def row){
|
||||
def commId = row['L']
|
||||
if(!commId){
|
||||
commId = row['G']
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.queryDistrictById(commId)
|
||||
def anjukeId = row['A']
|
||||
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||
if(obj && dist){
|
||||
def udata = new BasicDBObject()
|
||||
|
||||
def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address)
|
||||
if(address){
|
||||
if(!dist.address){
|
||||
if(obj.base.address){
|
||||
udata.put("address",obj.base.address)
|
||||
}
|
||||
}else{
|
||||
if(dist.address.contains("号")){
|
||||
if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){
|
||||
def addr_alias = dist.addr_alias
|
||||
if(!addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){
|
||||
addr_alias << obj.base.address
|
||||
}
|
||||
if(addr_alias!=dist.addr_alias && addr_alias.size()>0){
|
||||
udata.put("addr_alias", addr_alias)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
udata.put("address",obj.base.address)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def distName = nameFileter(obj.base.areaName, obj.base.name)
|
||||
if(!distName.contains("号")){
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(alias.contains(distName)){
|
||||
alias << distName
|
||||
}
|
||||
if(alias.size()>0 && alias!=dist.alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
}
|
||||
|
||||
if(!dist.build_type){
|
||||
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
|
||||
if(buildTypeStr){
|
||||
udata.put("build_type",buildTypeStr)
|
||||
}
|
||||
|
||||
}
|
||||
if(!dist.household_count){
|
||||
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){
|
||||
udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
|
||||
}
|
||||
}
|
||||
if(!dist.finish_building_date){
|
||||
if(obj.extend.completionTime){
|
||||
udata.put("finish_building_date",obj.extend.completionTime)
|
||||
}
|
||||
}
|
||||
|
||||
if(!dist.virescence_percent){
|
||||
if(obj.extend.landscapingRatio){
|
||||
udata.put("virescence_percent", obj.extend.landscapingRatio)
|
||||
}
|
||||
}
|
||||
if(!dist.business_quarter){
|
||||
if(obj.base.tradingAreaName){
|
||||
udata.put("business_quarter",obj.base.tradingAreaName)
|
||||
}
|
||||
}
|
||||
def lat2 = null
|
||||
def lng2 = null
|
||||
if(dist.lat2){
|
||||
lat2 = dist.lat2/10000000000.0
|
||||
lng2 = dist.lng2/10000000000.0
|
||||
}
|
||||
if(dist.lat && !lat2){
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3)
|
||||
lng2 = baidu.x
|
||||
lat2 = baidu.y
|
||||
|
||||
Long dLat2 = lat2 * 10000000000L
|
||||
Long dLng2 = lng2 * 10000000000L
|
||||
udata.put("lat2", dLat2)
|
||||
udata.put("lng2", dLng2)
|
||||
}
|
||||
|
||||
if(obj.base.lat) {
|
||||
def alat2 = obj.base.lat.toDouble()
|
||||
def alng2 = obj.base.lng.toDouble()
|
||||
def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2);
|
||||
if(distance>1000){
|
||||
println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................"
|
||||
|
||||
def lng = null
|
||||
def lat = null
|
||||
JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu")
|
||||
if(convertObj.getInt("status") == 1) {
|
||||
String lo = convertObj.get("locations")
|
||||
lng = Double.parseDouble(lo.split(",")[0])
|
||||
lat = Double.parseDouble(lo.split(",")[1])
|
||||
}
|
||||
|
||||
Long blat2 = alat2 *10000000000L
|
||||
Long blng2 = alng2 *10000000000L
|
||||
Long gLat2 = lat *10000000000L
|
||||
Long gLng2 = lng *10000000000L
|
||||
|
||||
udata.put("lat2", blat2)
|
||||
udata.put("lng2", blng2)
|
||||
udata.put("lat", gLat2)
|
||||
udata.put("lng", gLng2)
|
||||
}
|
||||
}
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改] ${dist.name}"+ udata
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def findAnjukeDistrictDetail(anjukeId){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id",new ObjectId(anjukeId))
|
||||
return detail.findOne(q)
|
||||
}
|
||||
|
||||
def nameFileter(def region, def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||
return name;
|
||||
}
|
||||
|
||||
def getEstatType(type){
|
||||
if(type){
|
||||
if(type=="公寓住宅"){
|
||||
type = "普通住宅"
|
||||
}else if(type.contains("商住")){
|
||||
type = "商住"
|
||||
}else if(type.contains("别墅")){
|
||||
type = "别墅"
|
||||
}else{
|
||||
type = null
|
||||
}
|
||||
}
|
||||
return type
|
||||
}
|
||||
|
||||
def getBuildType(def buildType){
|
||||
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
|
||||
def list = []
|
||||
if(buildType){
|
||||
def arrs = buildType.split("\\|")
|
||||
for(def s : arrs){
|
||||
if(types.contains(s)){
|
||||
list << s
|
||||
}
|
||||
}
|
||||
if(list){
|
||||
return list.join(",")
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getBuildType2(def buildType){
|
||||
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
|
||||
def list = []
|
||||
if(buildType){
|
||||
def arrs = buildType.split(",| ")
|
||||
for(def s : arrs){
|
||||
if(types.contains(s)){
|
||||
list << s
|
||||
}
|
||||
}
|
||||
if(list){
|
||||
return list.join(",")
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
@@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -358,13 +358,13 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("ÖØÇì")){
|
||||
// if(e.text().contains("¸£ÖÝ")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return list
|
||||
|
||||
+82
-75
@@ -29,7 +29,7 @@ class JdPaimaiCrawl {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def existsIds = findAllIds()
|
||||
def existsIds = []//findAllIds()
|
||||
println "ÒÑ´æÔÚ:"+existsIds.size()
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
@@ -43,78 +43,80 @@ class JdPaimaiCrawl {
|
||||
|
||||
def getList(driver){
|
||||
def list = []
|
||||
for(def i=1;i<=32;i++){
|
||||
if(i==8){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
|
||||
}else if(i==5){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
|
||||
}else if(i==17){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
|
||||
}else if(i==19){
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
|
||||
}else{
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
|
||||
}
|
||||
}
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268"
|
||||
// for(def i=1;i<=32;i++){
|
||||
// if(i==8){
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
|
||||
// }else if(i==5){
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
|
||||
// }else if(i==17){
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
|
||||
// }else if(i==19){
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
|
||||
// }else{
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
|
||||
// }
|
||||
|
||||
// }
|
||||
def links = []
|
||||
for(def url : list){
|
||||
driver.get(url)
|
||||
@@ -430,10 +432,10 @@ class JdPaimaiCrawl {
|
||||
|
||||
def s1 = JQueryUtils.selectText(["li:contains(¾ºÂò¹«¸æ) .floor-content"], doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("·¨Ôº½«ÓÚ")
|
||||
def start = s1.indexOf("将于")
|
||||
def end = s1.indexOf("Ö¹")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("·¨Ôº½«ÓÚ"), s1.indexOf("Ö¹"))
|
||||
s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = DataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
@@ -476,6 +478,11 @@ class JdPaimaiCrawl {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!result.startDate){
|
||||
def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","")
|
||||
result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日")
|
||||
}
|
||||
if(result.finishDate){
|
||||
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
||||
}
|
||||
|
||||
+27
-7
@@ -34,10 +34,30 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
task.standardRegion()
|
||||
// task.addDealRegion();
|
||||
// task.standardRegion()
|
||||
task.updadteDealPriceField()
|
||||
}
|
||||
|
||||
def updadteDealPriceField(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("hall", "0")
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
println "[${index}] "+city +" "+count
|
||||
if(count && count>0){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("hall", 0)
|
||||
DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
|
||||
println "[ÐÞ¸Ä]"+city +" "+ count
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run1(){
|
||||
@@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
@@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["³¤´º","¼ªÁÖ"]
|
||||
def citys= ["¸£ÖÝ"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+15
-4
@@ -30,8 +30,8 @@ class PaimaiTest {
|
||||
def task = new PaimaiTest();
|
||||
// task.run();
|
||||
// task.stat()
|
||||
// task.updatePaimai()
|
||||
task.updateFieldPaimai()
|
||||
task.updatePaimai()
|
||||
// task.updateFieldPaimai()
|
||||
}
|
||||
|
||||
def updateFieldPaimai(){
|
||||
@@ -42,8 +42,8 @@ class PaimaiTest {
|
||||
for(def obj : dbcursor) {
|
||||
index++
|
||||
println index
|
||||
if(obj.depositPrice){
|
||||
obj.put("depositPrice", obj.depositPrice.toInteger())
|
||||
if(obj.address && obj.address.contains("可贷款")){
|
||||
obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
|
||||
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
|
||||
}
|
||||
}
|
||||
@@ -51,11 +51,22 @@ class PaimaiTest {
|
||||
|
||||
def updatePaimai(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", Pattern.compile(".*jd.com.*"))
|
||||
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def index =0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
println index
|
||||
if(!d.url.contains("jd.com")){
|
||||
continue
|
||||
}
|
||||
|
||||
if(!d.proince.contains("重庆")){
|
||||
continue
|
||||
}
|
||||
|
||||
println d.url
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("url", d.url)
|
||||
def dbo = paimai43_COLL.findOne(query);
|
||||
|
||||
+34
-1
@@ -42,8 +42,41 @@ class CqfdcMatch {
|
||||
// task.export2()
|
||||
// task.matchLoudong()
|
||||
|
||||
task.addLoudong()
|
||||
// task.addLoudong()
|
||||
// task.addFdcLoudong()
|
||||
task.updateHouseLoudong()
|
||||
}
|
||||
|
||||
def updateHouseLoudong(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "ÖØÇì")
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("room_num", 1)
|
||||
keys.put("house_floor", 1)
|
||||
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def room_num = d.room_num
|
||||
def house_floor = d.house_floor
|
||||
try{
|
||||
if(room_num && house_floor && room_num.isNumber()){
|
||||
if(house_floor.toInteger()>0){
|
||||
if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){
|
||||
def newValue = house_floor +"-"+room_num
|
||||
println "[${index}] "+room_num +" "+house_floor+" "+newValue
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("room_num",newValue)
|
||||
HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
println d._id.toString()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def export(){
|
||||
def q = new BasicDBObject()
|
||||
|
||||
+3
-3
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "58ͬ³Ç"//
|
||||
def source = "±´¿ÇÕÒ·¿"//
|
||||
def citys = findCrawlCity(source, 202202);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202202);
|
||||
def count2 = findCount(source, city, 202203);
|
||||
def count1 = findCount(source, city, 202203);
|
||||
def count2 = findCount(source, city, 202204);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+4
-3
@@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{
|
||||
void process(Object args) {
|
||||
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
|
||||
|
||||
// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
|
||||
def citys = ["上海","北京","广州","天津","武汉","南京","青岛","合肥","西安","沈阳","宁波","佛山","潍坊","济南","大连","兰州","石家庄","长春","温州","乌鲁木齐","太原","惠州","嘉兴","南通","南昌","烟台","临沂","呼和浩特","厦门","海口","常州","中山","绍兴","淄博","洛阳","珠海","金华","威海","德阳","唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","衡阳","滨州","张家口","赣州","商丘","安庆","湖州","枣庄","聊城","阜阳","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳","蚌埠","九江","临汾","大同","六安","淮南","宿州","三亚","玉溪","六盘水","揭阳","汕尾","恩施","盘锦","伊犁","玉林","随州","抚州","景德镇","四平","丽水","安顺","毕节","天水","阜新","通化","攀枝花","吕梁","阿克苏","新余","松原","楚雄","鹤壁","忻州","本溪","拉萨","三门峡","黔东南","兴安盟","雅安","仙桃","乌海","贵港","安康","白城","平凉","庆阳","延安","潮州","白银","黔西南","丽江","吴忠","防城港","喀什","哈尔滨","钦州","鸡西","石嘴山","哈密","萍乡","鹰潭","武威","保山","铜仁","普洱","酒泉","宜昌","河池","朔州","百色","鹤岗","张家界","湘西","张掖","定西","昭通","济源","中卫","黑河","克拉玛依","铜川","辽源","云浮","博尔塔拉","固原","白山","广元","贺州","潜江","天门","伊春","石河子","临沧","榆林","临夏","来宾","成都","双鸭山","梧州","嘉峪关","海东","昌吉"]
|
||||
|
||||
// for(def city : citys){
|
||||
// def str = cityList.get(city)
|
||||
// for(def i=0;i<1;i++){
|
||||
@@ -209,7 +210,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
|
||||
cityList.each{city, str->
|
||||
for(def i=0;i<1;i++){
|
||||
// if(citys.contains(city)){
|
||||
if(citys.contains(city)){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
|
||||
if(districts){
|
||||
def arrs = str.split("\\|")
|
||||
@@ -219,7 +220,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
addCrawlKeys(city);
|
||||
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
|
||||
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
|
||||
def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -26,7 +26,7 @@ class SynchDistrictPrice {
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = ["重庆"]//CommonUtils.findAllCity()
|
||||
|
||||
sourceMap.each { k, v ->
|
||||
for(def city : citys){
|
||||
|
||||
+11
-10
@@ -236,10 +236,10 @@ public class OpenApiClient {
|
||||
params.put("houseNumber", houseNum);
|
||||
params.put("unitNumber", unitNum);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
|
||||
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
|
||||
println("${id} ${houseNum} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
@@ -266,10 +266,10 @@ public class OpenApiClient {
|
||||
params.put("unitNumber", unitNum);
|
||||
params.put("roomNumber", roomNum);
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
while (result == null) {
|
||||
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
|
||||
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
|
||||
println("${id} ${houseNum} 程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
@@ -687,7 +687,7 @@ public class OpenApiClient {
|
||||
params.put("region", region)
|
||||
params.put("name", name)
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
|
||||
return result
|
||||
@@ -697,13 +697,14 @@ public class OpenApiClient {
|
||||
return null
|
||||
}
|
||||
|
||||
static JSONObject searchPaimais(String city, String region, String name, limit) {
|
||||
static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("token", TOKEN);
|
||||
params.put("city", city)
|
||||
params.put("district", region)
|
||||
params.put("name", name)
|
||||
params.put("limit", "10")
|
||||
params.put("startDate", startDate)
|
||||
params.put("limit", "100")
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
@@ -722,7 +723,7 @@ public class OpenApiClient {
|
||||
params.put("startDate", startDate)
|
||||
params.put("limit", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
@@ -738,7 +739,7 @@ public class OpenApiClient {
|
||||
params.put("startTime", startDate)
|
||||
params.put("pageSize", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
@@ -755,7 +756,7 @@ public class OpenApiClient {
|
||||
params.put("startDate", startDate)
|
||||
params.put("limit", limit.toString())
|
||||
try {
|
||||
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8")
|
||||
JSONObject result = JSONObject.fromObject(content);
|
||||
return result
|
||||
} catch (Exception e) {
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package com.fangjia.service.open;
|
||||
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
|
||||
public interface CommunityBasePriceService {
|
||||
|
||||
CommunityBasePrice getById(String id);
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
package com.fangjia.service.open.impl;
|
||||
|
||||
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
|
||||
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
import com.fangjia.service.open.CommunityBasePriceService;
|
||||
import com.fangjia.service.open.CommunityPoiService;
|
||||
|
||||
public class CommunityBasePriceServiceImpl implements CommunityBasePriceService {
|
||||
|
||||
private CommunityBasePriceDao communityBasePriceDao;
|
||||
|
||||
@Override
|
||||
public CommunityBasePrice getById(String id) {
|
||||
return communityBasePriceDao.findById(id);
|
||||
}
|
||||
|
||||
public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) {
|
||||
this.communityBasePriceDao = communityBasePriceDao;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user