Merge remote-tracking branch 'origin/master'

This commit is contained in:
2022-05-25 10:05:21 +08:00
31 changed files with 2749 additions and 327 deletions
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.open.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityBasePriceDao extends MongoDao<CommunityBasePrice> {
}
@@ -0,0 +1,10 @@
package com.fangjia.dal.mongo.open.dao.impl;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public class CommunityBasePriceDaoImpl extends MongoDaoImpl<CommunityBasePrice> implements CommunityBasePriceDao {
}
@@ -0,0 +1,74 @@
package com.fangjia.dal.mongo.open.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
/**
* Ð¡Çø½¨ÐÐÖÃÐŶÈÐÅÏ¢
*
* @author shenjian@fangjia.com
*/
@Entity
@Table(name = "business.community_base_price_gj")
public class CommunityBasePrice implements MongoEntity {
private static final long serialVersionUID = -8010010773207622878L;
private String id;
private String city;
private String region;
private String name;
private Double credit;
@Id
@Column(name = "_id")
@Override
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "credit")
public Double getCredit() {
return credit;
}
public void setCredit(Double credit) {
this.credit = credit;
}
}
@@ -31,6 +31,7 @@ import javax.validation.Valid;
import com.fangjia.core.util.*;
import com.fangjia.dal.*;
import com.fangjia.dal.mongo.model.*;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater;
@@ -177,7 +178,9 @@ public class PropertyController {
private SeawaterDao seawaterDao;
private SeawaterService seawaterService;
private SqmetreRentService sqmetreRentService;
private CommunityPoiServiceImpl communityPoiService;
private CommunityPoiService communityPoiService;
private CommunityBasePriceService communityBasePriceService;
private EvaluateFactorService evaluateFactorService;
private FileDao fileDao = FileDao.getInstance();
private static final int DEFAULT_PAGE_SIZE = 20;
@@ -393,10 +396,31 @@ public class PropertyController {
put("5000", "five");// 五线
}
};
/**
* 第三方均价映射
*/
private static final Map<String, String> OTHER_PRICE_MAP = new HashMap<String, String>() {
{
put("zhuge_price", "诸葛找房");
put("anjuke_price", "安居客");
put("fang_price", "房天下");
put("lianjia_price", "链家");
put("cityhouse_price", "禧泰数据");
put("maitian_price", "麦田");
put("ke_price", "贝壳");
put("taofang_price", "淘房365");
put("zhongyuan_price", "中原");
put("tbs_price", "兔博士");
put("qfang_price", "Q房");
put("fangtan_price", "房探");
put("fangduoduo_price", "房多多");
put("wiwj_price", "我爱我家");
}
};
/**
* 利用百度API把GPS转换成Google经纬度
*
*
* @param lng
* @param lat
* @return
@@ -418,7 +442,7 @@ public class PropertyController {
/**
* 利用算法模板把GPS转换成Google经纬度
*
*
* @param lng
* @param lat
* @return
@@ -610,7 +634,7 @@ public class PropertyController {
/**
* 判断小区是否有经纬度
*
*
* @param district
* @return
*/
@@ -621,7 +645,7 @@ public class PropertyController {
/**
* 调用匹配接口匹配小区
*
*
* @param city
* @param address
* @param region
@@ -995,6 +1019,32 @@ public class PropertyController {
}
}
// 调整面积不同面积价格
try {
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
int orgAvgPrice = 0;
if (districtStatBo != null) {
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
} else if (districtStatBo.getDistrictStat() != null) {
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
}
}
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
}
} catch (Exception ex) {
}
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
@@ -1165,6 +1215,31 @@ public class PropertyController {
evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
return mav;
}
// 调整面积不同面积价格
try {
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
int orgAvgPrice = 0;
if (districtStatBo != null) {
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
} else if (districtStatBo.getDistrictStat() != null) {
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
}
}
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
}
} catch (Exception ex) {
}
Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>();
districtPriceMap.put("city", district.getCity());
@@ -1179,6 +1254,13 @@ public class PropertyController {
// }
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
// 新增置信度
CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId());
if (communityBasePrice != null) {
districtPriceMap.put("credit", communityBasePrice.getCredit());
} else {
districtPriceMap.put("credit", evaluation.getCredit());
}
List<String> names = new ArrayList<String>();
names.add(district.getName());
if(district.getAlias() != null){
@@ -1190,6 +1272,31 @@ public class PropertyController {
}
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9);
Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1);
// 新增第三方均价
try {
double dCredit = 0.61;
Map<String, Object> rMap = getLastEvaluationPrice(district, evaluation, dCredit);
if (null != rMap) {
if (rMap.containsKey("other_price")) {
Map<String, Integer> otherPriceMap = (Map<String, Integer>) rMap.get("other_price");
Map<String, Integer> priceResultMap = new HashMap<String, Integer>();
for (String key : otherPriceMap.keySet()) {
Integer price = otherPriceMap.get(key);
// 剔除偏差大于10的
if (min10Price <= price && max10Price >= price) {
if (OTHER_PRICE_MAP.containsKey(key)) {
priceResultMap.put(OTHER_PRICE_MAP.get(key), price);
}
}
}
districtPriceMap.put("otherPrice", priceResultMap);
}
}
} catch (Exception ex) {
}
List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
if(houses.size() > 3){
districtPriceMap.put("calcType", 1);
@@ -1491,7 +1598,7 @@ public class PropertyController {
return mav;
}
}
/**
* 四川锦程 定制接口 多建筑年代
* @param model
@@ -1603,7 +1710,7 @@ public class PropertyController {
districtPriceMap.put("buildYear", matcher3.group());
}
}
// 根据城市等 级设置D级可信度
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
@@ -2116,7 +2223,7 @@ public class PropertyController {
/**
* 小区租金查询接口
*
*
* @param model
* @return
*/
@@ -2595,7 +2702,7 @@ public class PropertyController {
/**
* 批量获取小区的均价,均价走势数据
*
*
* @param request
* @return
*/
@@ -2737,7 +2844,7 @@ public class PropertyController {
/**
* 批量获取小区的配套数据,最多12个小区
*
*
* @param request
* @return
*/
@@ -2882,7 +2989,7 @@ public class PropertyController {
/**
* 获取JSON参数中的num参数值
*
*
* @param jo
* @return
*/
@@ -2907,7 +3014,7 @@ public class PropertyController {
/**
* 获取JSON参数中的radius值
*
*
* @param jo
* @return
*/
@@ -3715,7 +3822,7 @@ public class PropertyController {
/**
* 解析出关键字中的小区名
*
*
* @param keyword
* @return
*/
@@ -3832,7 +3939,7 @@ public class PropertyController {
/**
* 小区统计分析接口
*
*
* @param model
* @return
*/
@@ -3917,7 +4024,7 @@ public class PropertyController {
/**
* 小区交通接口
*
*
* @param model
* @return
*/
@@ -3975,7 +4082,7 @@ public class PropertyController {
/**
* 小区周边接口
*
*
* @param model
* @return
*/
@@ -4034,7 +4141,7 @@ public class PropertyController {
/**
* 相同小区相似户型房源挂牌量接口<br>
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
*
*
* @author yinjihuan
* @date 2016年9月1日
* @param city
@@ -4074,7 +4181,7 @@ public class PropertyController {
/**
* 周边小区相似户型挂牌量接口<br>
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
*
*
* @author yinjihuan
* @date 2016年9月1日
* @param city
@@ -4125,7 +4232,7 @@ public class PropertyController {
/**
* 小区周边一公里均价
*
*
* @author wenyan
* @param model
* @return
@@ -4148,7 +4255,7 @@ public class PropertyController {
/**
* 相同小区相似户型租金范围查询接口
*
*
* @param model
* @return
*/
@@ -4191,7 +4298,7 @@ public class PropertyController {
/**
* 周边小区相似户型租金范围查询接口
*
*
* @param model
* @return
*/
@@ -4254,7 +4361,7 @@ public class PropertyController {
/**
* 室号查询接口
*
*
* @param model
* @return
*/
@@ -4290,7 +4397,7 @@ public class PropertyController {
/**
* 楼栋楼层信息查询接口
*
*
* @param model
* @return
*/
@@ -4315,7 +4422,7 @@ public class PropertyController {
/**
* 楼栋查询接口
*
*
* @param model
* @return
*/
@@ -4360,7 +4467,7 @@ public class PropertyController {
/**
* 楼栋查询接口
*
*
* @param model
* @return
*/
@@ -4813,7 +4920,7 @@ public class PropertyController {
/**
* 房产交易记录查询
*
*
* @param model
* @return
*/
@@ -4873,7 +4980,7 @@ public class PropertyController {
/**
* 房产拍卖记录
*
*
* @param model
* @return
*/
@@ -4934,7 +5041,7 @@ public class PropertyController {
/**
* 房产近期评估次数
*
*
* @param model
* @return
*/
@@ -5650,11 +5757,15 @@ public class PropertyController {
this.rentDealPriceService = rentDealPriceService;
}
public CommunityPoiServiceImpl getCommunityPoiService() {
return communityPoiService;
}
public void setCommunityPoiService(CommunityPoiServiceImpl communityPoiService) {
public void setCommunityPoiService(CommunityPoiService communityPoiService) {
this.communityPoiService = communityPoiService;
}
public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) {
this.evaluateFactorService = evaluateFactorService;
}
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
this.communityBasePriceService = communityBasePriceService;
}
}
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -38,8 +38,8 @@ class ExportOracleData {
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-03-19")
exportBigdataDeal("2022-03-01")
exportBigdataForsale("2022-04-19")
exportBigdataDeal("2022-04-01")
//计算结果相关
@@ -0,0 +1,67 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DistrictCodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* oralce中小区统计数据同步至85中
*/
class OracleDistrict2Mongo85 {
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017)
static main(args) {
// 只写入重庆市数据
def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ"
def count = 0
sqlInstance.eachRow(sql, { r ->
def city = DistrictCodeUtils.getCityName(r['cityid'])
def region = RegionData.formatRegion(city, r['distname'])
DBObject dbObject = new BasicDBObject("_id", r['commid'])
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
.append("name", r['commname'])
.append('subway', Integer.parseInt(r['subway'].toString()))
.append("traffic", Integer.parseInt(r['traffic'].toString()))
.append("bank", Integer.parseInt(r['bank'].toString()))
.append("primary", Integer.parseInt(r['primary'].toString()))
.append("junior", Integer.parseInt(r['junior'].toString()))
.append("hospital", Integer.parseInt(r['hospital'].toString()))
.append("funeral", Integer.parseInt(r['funeral'].toString()))
.append("c_date", new Date());
communityPoiConn.update(dbObject, updateObject, true, false)
count++;
if (count % 500 == 0) {
println "已写入" + count + "条小区数据"
}
});
}
}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202201
def endDate = 202203
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202202
def endDate = 202204
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-01-01"
def date = "2022-02-01"
def endDate = "2022-03-01"
def newDate = "2022-04-01"
def startDate = "2022-02-01"
def date = "2022-03-01"
def endDate = "2022-04-01"
def newDate = "2022-05-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -0,0 +1,117 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
class CcbDistNameMatch {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new CcbDistNameMatch();
// task.run()
task.run1()
}
def run1(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
if(d['D'].contains("、")){
println d['D']
}
def arrs = d['D'].split("、")
def list = []
def flags = []
for(def s : arrs){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district")
if(dist){
list << dist.name
flags << "完全一致"
}else{
def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion)
if(jsonObject != null && jsonObject.precise!=-1) {
if(jsonObject.get("district") == newRegion){
list << jsonObject.get("name")
flags << "匹配接口"
}
}
}
}
if(list){
list = list.unique()
d['E'] = list.join(",")
d['F'] = flags.join(",")
}
}
}
})
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d ->
def regionCode = null
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
d['D'] = dist.name
def count = findDistrictByOracle(dist._id.toString())
if(count && count>0){
d['F'] = 1
}else{
d['H'] = dist.estate_type2
d['J'] = dist.name_cat
}
}else{
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
if (rets) {
if(rets.size()>1){
d['E'] = "别名-多个"
}else{
dist = rets[0]
d['E'] = d['C'] +"--"+dist.name
def count = findAliasByOracle(dist._id.toString())
if(count && count>0) {
d['G'] = 1
}else{
d['H'] = dist.estate_type2
d['J'] = dist.name_cat
}
}
}
}
}
}
});
}
def findDistrictByOracle(commid){
def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findAliasByOracle(commid){
def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
}
@@ -23,22 +23,22 @@ class MatchEvaluateRegionTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
def path = "D:\\数据\\重庆农商行\\2022-05-06\\估值限时测试-重庆农商行1.0.xlsx"
def total = 1500
static main(args) {
def task = new MatchEvaluateRegionTest()
task.run()//小区匹配
task.run1()//地图匹配
// task.run()//小区匹配
// task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
task.addArroundPrice()
task.addAuthor()
// task.addArroundPrice()
// task.addAuthor()
}
def run(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], "property", false,null,null,)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
@@ -57,7 +57,7 @@ class MatchEvaluateRegionTest {
def run1(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
def jsonObject = OpenApiClient.addressMatch("重庆", row['C'], null, true, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
@@ -82,7 +82,7 @@ class MatchEvaluateRegionTest {
row['N']= (int)(priceObj.avg_price * row['E'])
}
def avg_price = findAvgPrice(row['M'])
def avg_price = findAvgPrice(row['F'])
if(row['M'] && avg_price){
def weekendPrice = row['M'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
@@ -95,12 +95,12 @@ class MatchEvaluateRegionTest {
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A'] && !row['K']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
if(!row['M']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['M'] = priceObj.unitPrice
row['N'] = (int) (priceObj.unitPrice * row['C'])
row['N'] = (int) (priceObj.unitPrice * row['E'])
row['Q'] = "周边价格"
}
}
@@ -109,21 +109,22 @@ class MatchEvaluateRegionTest {
def addAuthor(){
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def total = 747
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['P'] = authors[count]
row['N'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
@@ -28,14 +28,14 @@ class MatchEvaluateTest {
static main(args) {
def task = new MatchEvaluateTest()
// task.run()//小区匹配
// task.run1()//地图匹配
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
// task.addPrice()
// task.addArroundPrice()
// task.addAuthor()
task.run()//小区匹配
task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
task.addArroundPrice()
task.addAuthor()
task.stat()
// task.stat()
}
def addArroundPrice(){
@@ -57,7 +57,7 @@ class MatchEvaluateTest {
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
@@ -77,7 +77,7 @@ class MatchEvaluateTest {
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
@@ -0,0 +1,763 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class PocStat {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
static main(args) {
def task = new PocStat();
task.regionDistStat();
// task.sampleStat()
// task.tradeStat()
// task.paimaiStat()
// task.evalStat()
//
// task.fileldStat()
}
def regionDistStat(){
def city = "重庆"
def regionMap = CommonDatautils.getRegionsByCity(city)
def set =regionMap.keySet();
def rows = []
rows << ["区域","小区量"]
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
for(def region : set){
def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district")
def regionCount = 0
def regionHouseLDCount = 0
for(def d : dists){
if(d.region == region){
def loudongCount = CommonDatautils.findLoudong(d._id.toString())
if(loudongCount && loudongCount>0){
regionCount = regionCount+ loudongCount
}
def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString())
if(houseLouDongCount && houseLouDongCount>0){
regionHouseLDCount = regionHouseLDCount + houseLouDongCount
}
}
}
rows << [region, count, regionCount, regionHouseLDCount]
println region +" 完成"
}
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计")
}
def sampleStat(){
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
final CountDownLatch latch = new CountDownLatch(dists.size())
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
def zCityDistTotal = new AtomicLong()
def fCityDistTotal = new AtomicLong()
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
def zHLevel1 = new AtomicLong()
def zHLevel2 = new AtomicLong()
def zHLevel3 = new AtomicLong()
def zHLevel4 = new AtomicLong()
def zHLevel5 = new AtomicLong()
def fHLevel1 = new AtomicLong()
def fHLevel2 = new AtomicLong()
def fHLevel3 = new AtomicLong()
def fHLevel4 = new AtomicLong()
def fHLevel5 = new AtomicLong()
def rets = []
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
def index = new AtomicLong()
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
index.addAndGet(1)
// println index +" "+d.name
def commid = d._id.toString()
//房源
def jsonObject1 = OpenApiClient.houseList(commid,"20220201",60)
if(jsonObject1 && jsonObject1.result) {
def results = jsonObject1.result
if(zRegions.contains(d.region)){
zCityDistTotal.addAndGet(1)
if(results){
if(results.size()<5){
zHLevel2.addAndGet(1)
}else if(results.size()<20){
zHLevel3.addAndGet(1)
}else if(results.size()<50){
zHLevel4.addAndGet(1)
}else if(results.size()>=50){
zHLevel5.addAndGet(1)
}
}
}else{
fCityDistTotal.addAndGet(1)
if(results){
if(results.size()<5){
fHLevel2.addAndGet(1)
}else if(results.size()<20){
fHLevel3.addAndGet(1)
}else if(results.size()<50){
fHLevel4.addAndGet(1)
}else if(results.size()>=50){
fHLevel5.addAndGet(1)
}
}
}
}else{
if(zRegions.contains(d.region)) {
zCityDistTotal.addAndGet(1)
zHLevel1.addAndGet(1)
println d.city+"#"+d.region+"#"+d.name
}else{
fCityDistTotal.addAndGet(1)
fHLevel1.addAndGet(1)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
def ret1 = ["主城区", zCityDistTotal.get(), zHLevel1.get(), zHLevel2.get(), zHLevel3.get(), zHLevel4.get(), zHLevel5.get()]
def ret2 = ["非主城区", fCityDistTotal.get(), fHLevel1.get(), fHLevel2.get(), fHLevel3.get(), fHLevel4.get(), fHLevel5.get()]
rets.add(ret1)
rets.add(ret2)
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-样本")
}
def tradeStat(){
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
final CountDownLatch latch = new CountDownLatch(dists.size())
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
def zCityDistTotal = new AtomicLong()
def fCityDistTotal = new AtomicLong()
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
def zTLevel1 = new AtomicLong()
def zTLevel2 = new AtomicLong()
def zTLevel3 = new AtomicLong()
def zTLevel4 = new AtomicLong()
def zTLevel5 = new AtomicLong()
def fTLevel1 = new AtomicLong()
def fTLevel2 = new AtomicLong()
def fTLevel3 = new AtomicLong()
def fTLevel4 = new AtomicLong()
def fTLevel5 = new AtomicLong()
def rets = []
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
def index = new AtomicLong()
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
index.addAndGet(1)
println index +" "+d.name
def commid = d._id.toString()
//房源
def jsonObject1 = OpenApiClient.searchDealPrice(commid,"20220201",60)
if(jsonObject1 && jsonObject1.result) {
def results = jsonObject1.result
if(zRegions.contains(d.region)){
zCityDistTotal.addAndGet(1)
if(results){
if(results.size()<5){
zTLevel2.addAndGet(1)
}else if(results.size()<20){
zTLevel3.addAndGet(1)
}else if(results.size()<50){
zTLevel4.addAndGet(1)
}else if(results.size()>=50){
zTLevel5.addAndGet(1)
}
}
}else{
fCityDistTotal.addAndGet(1)
if(results){
if(results.size()<5){
fTLevel2.addAndGet(1)
}else if(results.size()<20){
fTLevel3.addAndGet(1)
}else if(results.size()<50){
fTLevel4.addAndGet(1)
}else if(results.size()>=50){
fTLevel5.addAndGet(1)
}
}
}
}else{
if(zRegions.contains(d.region)) {
zCityDistTotal.addAndGet(1)
zTLevel1.addAndGet(1)
}else{
fCityDistTotal.addAndGet(1)
fTLevel1.addAndGet(1)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
def ret1 = ["主城区", zCityDistTotal.get(), zTLevel1.get(), zTLevel2.get(), zTLevel3.get(), zTLevel4.get(), zTLevel5.get()]
def ret2 = ["非主城区", fCityDistTotal.get(), fTLevel1.get(), fTLevel2.get(), fTLevel3.get(), fTLevel4.get(), fTLevel5.get()]
rets.add(ret1)
rets.add(ret2)
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-成交")
println "成交案例统计完成"
}
def paimaiStat(){
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
final CountDownLatch latch = new CountDownLatch(dists.size())
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
def zCityDistTotal = new AtomicLong()
def fCityDistTotal = new AtomicLong()
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
def zPLevel1 = new AtomicLong()
def zPLevel2 = new AtomicLong()
def zPLevel3 = new AtomicLong()
def zPLevel4 = new AtomicLong()
def zPLevel5 = new AtomicLong()
def fPLevel1 = new AtomicLong()
def fPLevel2 = new AtomicLong()
def fPLevel3 = new AtomicLong()
def fPLevel4 = new AtomicLong()
def fPLevel5 = new AtomicLong()
def rets = []
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
def index = new AtomicLong()
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
index.addAndGet(1)
println index +" "+d.name
def commid = d._id.toString()
//房源
def jsonObject1 = OpenApiClient.searchPaimais(d.city, d.region, d.name,"20220201",100)
if(jsonObject1 && jsonObject1.result) {
def results = jsonObject1.result
if(zRegions.contains(d.region)){
zCityDistTotal.addAndGet(1)
if(results){
if(results.size()<5){
zPLevel2.addAndGet(1)
}else if(results.size()<20){
zPLevel3.addAndGet(1)
}else if(results.size()<50){
zPLevel4.addAndGet(1)
}else if(results.size()>=50){
zPLevel5.addAndGet(1)
}
}
}else{
fCityDistTotal.addAndGet(1)
if(results){
if(results.size()<5){
fPLevel2.addAndGet(1)
}else if(results.size()<20){
fPLevel3.addAndGet(1)
}else if(results.size()<50){
fPLevel4.addAndGet(1)
}else if(results.size()>=50){
fPLevel5.addAndGet(1)
}
}
}
}else{
if(zRegions.contains(d.region)) {
zCityDistTotal.addAndGet(1)
zPLevel1.addAndGet(1)
}else{
fCityDistTotal.addAndGet(1)
fPLevel1.addAndGet(1)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
def ret1 = ["主城区", zCityDistTotal.get(), zPLevel1.get(), zPLevel2.get(), zPLevel3.get(), zPLevel4.get(), zPLevel5.get()]
def ret2 = ["非主城区", fCityDistTotal.get(), fPLevel1.get(), fPLevel2.get(), fPLevel3.get(), fPLevel4.get(), fPLevel5.get()]
rets.add(ret1)
rets.add(ret2)
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-拍卖")
println "拍卖案例统计完成"
}
def evalStat(){
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
final CountDownLatch latch = new CountDownLatch(dists.size())
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
def zCityDistTotal = new AtomicLong()
def fCityDistTotal = new AtomicLong()
//[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
def zELevel1 = new AtomicLong()
def zELevel2 = new AtomicLong()
def zELevel3 = new AtomicLong()
def zELevel4 = new AtomicLong()
def zELevel5 = new AtomicLong()
def fELevel1 = new AtomicLong()
def fELevel2 = new AtomicLong()
def fELevel3 = new AtomicLong()
def fELevel4 = new AtomicLong()
def fELevel5 = new AtomicLong()
def rets = []
rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
def index = new AtomicLong()
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
index.addAndGet(1)
println index +" "+d.name
def commid = d._id.toString()
//房源
def jsonObject1 = OpenApiClient.evaluaionList(commid,"20220201",60)
if(jsonObject1 && jsonObject1.result) {
def results = jsonObject1.result
if(zRegions.contains(d.region)){
zCityDistTotal.addAndGet(1)
if(results){
if(results.size()<5){
zELevel2.addAndGet(1)
}else if(results.size()<20){
zELevel3.addAndGet(1)
}else if(results.size()<50){
zELevel4.addAndGet(1)
}else if(results.size()>=50){
zELevel5.addAndGet(1)
}
}
}else{
fCityDistTotal.addAndGet(1)
if(results){
if(results.size()<5){
fELevel2.addAndGet(1)
}else if(results.size()<20){
fELevel3.addAndGet(1)
}else if(results.size()<50){
fELevel4.addAndGet(1)
}else if(results.size()>=50){
fELevel5.addAndGet(1)
}
}
}
}else{
if(zRegions.contains(d.region)) {
zCityDistTotal.addAndGet(1)
zELevel1.addAndGet(1)
}else{
fCityDistTotal.addAndGet(1)
fELevel1.addAndGet(1)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
def ret1 = ["主城区", zCityDistTotal.get(), zELevel1.get(), zELevel2.get(), zELevel3.get(), zELevel4.get(), zELevel5.get()]
def ret2 = ["非主城区", fCityDistTotal.get(), fELevel1.get(), fELevel2.get(), fELevel3.get(), fELevel4.get(), fELevel5.get()]
rets.add(ret1)
rets.add(ret2)
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-评估")
println "评估案例统计完成"
}
def fileldStat(){
def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
final CountDownLatch latch = new CountDownLatch(dists.size())
def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
def alias_count = new AtomicLong()
def address_count = new AtomicLong()
def block_count = new AtomicLong()
def township_count = new AtomicLong()
def price_count = new AtomicLong()
def lng_count = new AtomicLong()
def finishBuildTime_count = new AtomicLong()
def developer_count = new AtomicLong()
def green_count = new AtomicLong()
def ratio_count = new AtomicLong()
def carsite_count = new AtomicLong()
def propertyManagement_count = new AtomicLong()
def managementFee_count = new AtomicLong()
def building_count = new AtomicLong()
def holed_count = new AtomicLong()
def car_holed_count = new AtomicLong()
def all_area_count = new AtomicLong()
def key_school_count = new AtomicLong()
def school_count = new AtomicLong()
def traffic_count = new AtomicLong()
def market_count = new AtomicLong()
def hospital_count = new AtomicLong()
def bank_count = new AtomicLong()
def f_alias_count = new AtomicLong()
def f_address_count = new AtomicLong()
def f_block_count = new AtomicLong()
def f_township_count = new AtomicLong()
def f_price_count = new AtomicLong()
def f_lng_count = new AtomicLong()
def f_finishBuildTime_count = new AtomicLong()
def f_developer_count = new AtomicLong()
def f_green_count = new AtomicLong()
def f_ratio_count = new AtomicLong()
def f_carsite_count = new AtomicLong()
def f_propertyManagement_count = new AtomicLong()
def f_managementFee_count = new AtomicLong()
def f_building_count = new AtomicLong()
def f_holed_count = new AtomicLong()
def f_car_holed_count = new AtomicLong()
def f_all_area_count = new AtomicLong()
def f_key_school_count = new AtomicLong()
def f_school_count = new AtomicLong()
def f_traffic_count = new AtomicLong()
def f_market_count = new AtomicLong()
def f_hospital_count = new AtomicLong()
def f_bank_count = new AtomicLong()
def zCityDistTotal = new AtomicLong()
def fCityDistTotal = new AtomicLong()
def index = new AtomicLong()
for(def i=0;i<dists.size();i++) {
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
index.addAndGet(1)
println index +" "+dist.name
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
if(jsonObject) {
if(zRegions.contains(jsonObject.get("district"))){
zCityDistTotal.addAndGet(1)
if(jsonObject.get("alias")){
alias_count.addAndGet(1)
}
if(jsonObject.get("address")){
address_count.addAndGet(1)
}
if(jsonObject.get("commercialArea")){//板块
block_count.addAndGet(1)
}
if(jsonObject.get("township")){//街道***********
township_count.addAndGet(1)
}
if(jsonObject.get("avgPrice")) {//基准价
price_count.addAndGet(1)
}
if(jsonObject.get("lng")){//经纬度
lng_count.addAndGet(1)
}
if(jsonObject.get("finishBuildTime")){//竣工时间
finishBuildTime_count.addAndGet(1)
}
if(jsonObject.get("developer")){//开发商名称
developer_count.addAndGet(1)
}
def green = SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
if(green){
green_count.addAndGet(1)
}
def ratio = SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
if(ratio){
ratio_count.addAndGet(1)
}
def carsite = SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
if(carsite){
carsite_count.addAndGet(1)
}
if(jsonObject.get("propertyManagement")){//物业公司
propertyManagement_count.addAndGet(1)
}
if(jsonObject.get("managementFee")){//物业费
managementFee_count.addAndGet(1)
}
if(jsonObject.get("buildingCount")){//住宅幢数
building_count.addAndGet(1)
}
def holed = SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
if(holed){
holed_count.addAndGet(1)
}
if(carsite && holed){
car_holed_count.addAndGet(1)
}
def all_area = SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
if(all_area){
all_area_count.addAndGet(1)
}
}else{
fCityDistTotal.addAndGet(1)
if(jsonObject.get("alias")){
f_alias_count.addAndGet(1)
}
if(jsonObject.get("address")){
f_address_count.addAndGet(1)
}
if(jsonObject.get("commercialArea")){//板块
f_block_count.addAndGet(1)
}
if(jsonObject.get("township")){//街道***********
f_township_count.addAndGet(1)
}
if(jsonObject.get("avgPrice")) {//基准价
f_price_count.addAndGet(1)
}
if(jsonObject.get("lng")){//经纬度
f_lng_count.addAndGet(1)
}
if(jsonObject.get("finishBuildTime")){//竣工时间
f_finishBuildTime_count.addAndGet(1)
}
if(jsonObject.get("developer")){//开发商名称
f_developer_count.addAndGet(1)
}
def green = SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
if(green){
f_green_count.addAndGet(1)
}
def ratio = SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
if(ratio){
f_ratio_count.addAndGet(1)
}
def carsite = SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
if(carsite){
f_carsite_count.addAndGet(1)
}
if(jsonObject.get("propertyManagement")){//物业公司
f_propertyManagement_count.addAndGet(1)
}
if(jsonObject.get("managementFee")){//物业费
f_managementFee_count.addAndGet(1)
}
if(jsonObject.get("buildingCount")){//住宅幢数
f_building_count.addAndGet(1)
}
def holed = SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
if(holed){
f_holed_count.addAndGet(1)
}
if(carsite && holed){
f_car_holed_count.addAndGet(1)
}
def all_area = SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
if(all_area){
f_all_area_count.addAndGet(1)
}
}
}
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(zRegions.contains(dist.region)){
if(result.keySchool){
key_school_count.addAndGet(1)
}
if(result.school){
school_count.addAndGet(1)
}
def bus = []
if(result.bus){
bus.addAll(result.bus)
}
if(result.metro){
bus.addAll(result.metro)
}
if(bus){
traffic_count.addAndGet(1)
}
if(result.supermarket){
market_count.addAndGet(1)
}
if(result.hospital){
hospital_count.addAndGet(1)
}
if(result.bank){
bank_count.addAndGet(1)
}
}else{
if(result.keySchool){
f_key_school_count.addAndGet(1)
}
if(result.school){
f_school_count.addAndGet(1)
}
def bus = []
if(result.bus){
bus.addAll(result.bus)
}
if(result.metro){
bus.addAll(result.metro)
}
if(bus){
f_traffic_count.addAndGet(1)
}
if(result.supermarket){
f_market_count.addAndGet(1)
}
if(result.hospital){
f_hospital_count.addAndGet(1)
}
if(result.bank){
f_bank_count.addAndGet(1)
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
def rets = []
rets << ["城区","小区名称","小区id","小区量","别名","地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构","楼盘每月均价走势","板块每月均价走势","行政区每月均价走势","城市每月均价走势"]
def alias_pre = DataHandleUtils.getDoubleRound(2, 100-alias_count*100/zCityDistTotal)
def addr_pre = DataHandleUtils.getDoubleRound(2, 100-address_count*100/zCityDistTotal)
def block_pre = DataHandleUtils.getDoubleRound(2, 100-block_count*100/zCityDistTotal)
def town_pre = DataHandleUtils.getDoubleRound(2, 100-township_count*100/zCityDistTotal)
def price_pre = DataHandleUtils.getDoubleRound(2, 100-price_count*100/zCityDistTotal)
def lat_pre = DataHandleUtils.getDoubleRound(2, 100-lng_count*100/zCityDistTotal)
def finish_pre = DataHandleUtils.getDoubleRound(2, 100-finishBuildTime_count*100/zCityDistTotal)
def developer_pre = DataHandleUtils.getDoubleRound(2, 100-developer_count*100/zCityDistTotal)
def green_pre = DataHandleUtils.getDoubleRound(2, 100-green_count*100/zCityDistTotal)
def ratio_pre = DataHandleUtils.getDoubleRound(2, 100-ratio_count*100/zCityDistTotal)
def carsite_pre = DataHandleUtils.getDoubleRound(2, 100-carsite_count*100/zCityDistTotal)
def pm_pre = DataHandleUtils.getDoubleRound(2, 100-propertyManagement_count*100/zCityDistTotal)
def mf_pre = DataHandleUtils.getDoubleRound(2, 100-managementFee_count*100/zCityDistTotal)
def building_pre = DataHandleUtils.getDoubleRound(2, 100-building_count*100/zCityDistTotal)
def holed_pre = DataHandleUtils.getDoubleRound(2, 100-holed_count*100/zCityDistTotal)
def car_holed_pre = DataHandleUtils.getDoubleRound(2, 100-car_holed_count*100/zCityDistTotal)
def all_area_pre = DataHandleUtils.getDoubleRound(2, 100-all_area_count*100/zCityDistTotal)
def key_school_pre = DataHandleUtils.getDoubleRound(2, 100-key_school_count*100/zCityDistTotal)
def school_pre = DataHandleUtils.getDoubleRound(2, 100-school_count*100/zCityDistTotal)
def traffic_pre = DataHandleUtils.getDoubleRound(2, 100-traffic_count*100/zCityDistTotal)
def market_pre = DataHandleUtils.getDoubleRound(2, 100-market_count*100/zCityDistTotal)
def hosp_pre = DataHandleUtils.getDoubleRound(2, 100-hospital_count*100/zCityDistTotal)
def bank_pre = DataHandleUtils.getDoubleRound(2, 100-bank_count*100/zCityDistTotal)
def ret1 = ["主城区","0" ,"0",zCityDistTotal.get(), alias_pre,addr_pre,block_pre,town_pre,price_pre,lat_pre,
finish_pre,developer_pre,green_pre,ratio_pre,carsite_pre,pm_pre,mf_pre,building_pre,
holed_pre,car_holed_pre,all_area_pre,key_school_pre,school_pre,traffic_pre,market_pre,
hosp_pre,bank_pre,price_pre,0,0,0]
def f_alias_pre = DataHandleUtils.getDoubleRound(2, 100-f_alias_count*100/zCityDistTotal)
def f_addr_pre = DataHandleUtils.getDoubleRound(2, 100-f_address_count*100/zCityDistTotal)
def f_block_pre = DataHandleUtils.getDoubleRound(2, 100-f_block_count*100/zCityDistTotal)
def f_town_pre = DataHandleUtils.getDoubleRound(2, 100-f_township_count*100/zCityDistTotal)
def f_price_pre = DataHandleUtils.getDoubleRound(2, 100-f_price_count*100/zCityDistTotal)
def f_lat_pre = DataHandleUtils.getDoubleRound(2, 100-f_lng_count*100/zCityDistTotal)
def f_finish_pre = DataHandleUtils.getDoubleRound(2, 100-f_finishBuildTime_count*100/zCityDistTotal)
def f_developer_pre = DataHandleUtils.getDoubleRound(2, 100-f_developer_count*100/zCityDistTotal)
def f_green_pre = DataHandleUtils.getDoubleRound(2, 100-f_green_count*100/zCityDistTotal)
def f_ratio_pre = DataHandleUtils.getDoubleRound(2, 100-f_ratio_count*100/zCityDistTotal)
def f_carsite_pre = DataHandleUtils.getDoubleRound(2, 100-f_carsite_count*100/zCityDistTotal)
def f_pm_pre = DataHandleUtils.getDoubleRound(2, 100-f_propertyManagement_count*100/zCityDistTotal)
def f_mf_pre = DataHandleUtils.getDoubleRound(2, 100-f_managementFee_count*100/zCityDistTotal)
def f_building_pre = DataHandleUtils.getDoubleRound(2, 100-f_building_count*100/zCityDistTotal)
def f_holed_pre = DataHandleUtils.getDoubleRound(2, 100-f_holed_count*100/zCityDistTotal)
def f_car_holed_pre = DataHandleUtils.getDoubleRound(2, 100-f_car_holed_count*100/zCityDistTotal)
def f_all_area_pre = DataHandleUtils.getDoubleRound(2, 100-f_all_area_count*100/zCityDistTotal)
def f_key_school_pre = DataHandleUtils.getDoubleRound(2, 100-f_key_school_count*100/zCityDistTotal)
def f_school_pre = DataHandleUtils.getDoubleRound(2, 100-f_school_count*100/zCityDistTotal)
def f_traffic_pre = DataHandleUtils.getDoubleRound(2, 100-f_traffic_count*100/zCityDistTotal)
def f_market_pre = DataHandleUtils.getDoubleRound(2, 100-f_market_count*100/zCityDistTotal)
def f_hosp_pre = DataHandleUtils.getDoubleRound(2, 100-f_hospital_count*100/zCityDistTotal)
def f_bank_pre = DataHandleUtils.getDoubleRound(2, 100-f_bank_count*100/zCityDistTotal)
def ret2 = ["非主城区","0" ,"0",fCityDistTotal.get(), f_alias_pre,f_addr_pre,f_block_pre,f_town_pre,f_price_pre,f_lat_pre,
f_finish_pre,f_developer_pre,f_green_pre,f_ratio_pre,f_carsite_pre,f_pm_pre,f_mf_pre,f_building_pre,
f_holed_pre,f_car_holed_pre,f_all_area_pre,f_key_school_pre,f_school_pre,f_traffic_pre,f_market_pre,
f_hosp_pre,f_bank_pre,f_price_pre,0,0,0]
rets.add(ret1)
rets.add(ret2)
ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "字段-安")
println "字段统计完成"
}
}
@@ -2,9 +2,11 @@ package com.fangjia.script.chongqing
import com.fangjia.core.util.SourceNameUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
@@ -15,28 +17,75 @@ class PocTest {
task.run();
}
def run(){
// match("D:\\数据\\重庆农商行\\poc\\test.xlsx")
// addPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx")
// standard("D:\\数据\\重庆农商行\\poc\\test.xlsx")
// addLoudongNum("D:\\数据\\重庆农商行\\poc\\test.xlsx")
// addDetail("D:\\数据\\重庆农商行\\poc\\test.xlsx", "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
// createLoudong("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
// createPaimai("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
// createHouse("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
// createDealPrice("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
createEvaluation("D:\\数据\\重庆农商行\\poc\\test.xlsx","D:\\数据\\重庆农商行\\poc\\案例.xlsx")
/**
* A 序号
* B 城市
* C 区域
* D 地址
* E 面积
* @return
*/
def run(){
def inputfile = "D:\\数据\\重庆农商行\\poc\\重庆农商行.xlsx"
match(inputfile)
addPrice(inputfile)
addArroundPrice(inputfile)
// matchMap(inputfile)
standard(inputfile)
addLoudongNum(inputfile)
// addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
//
// createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
// createPaimai(inputfile,"D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
// createHouse(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
// createDealPrice(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
// createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['P'] && row['C']){
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false, row['B'], null)
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['P'] = jsonObject.get("name")
row['Q'] = jsonObject.get("id")
row['Q'] = jsonObject.get("name")
row['R'] = jsonObject.get("id")
}
}
})
}
def matchMap(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
def propertyType = jsonObject.get("propertyType")
if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){
row['Q'] = jsonObject.get("name")
row['R'] = new ObjectId().toString()
row['W'] = propertyType
def arroundObj = FangjiaClientUtils.arroundPrice(city, address)
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['F'] = priceObj.unitPrice
}
}
}
}
})
@@ -44,34 +93,51 @@ class PocTest {
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['Q']){
def dist = CommonDatautils.findDistrictById(row['Q'])
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid)
if(dist){
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['E'] = priceObj.avg_price
row['F'] = priceObj.avg_price
}
}
}
})
}
def addArroundPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['F'] = priceObj.unitPrice
row['W'] = "周边均价"
}
}
});
}
def standard(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['C']){
def jsonObject = OpenApiClient.standard2Action(row['A'], row['C'])
def city = row['B']
def address = row['D']
if(address){
println "[standard] address:"+address
def jsonObject = OpenApiClient.standard2Action(city, address)
if(jsonObject){
println jsonObject
row['F'] = jsonObject.get("province")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("township")
row['J'] = jsonObject.get("street")
row['K'] = jsonObject.get("streetNumber")
row['L'] = jsonObject.get("buildingNumber")
row['M'] = jsonObject.get("unitNumber")
row['N'] = jsonObject.get("floor")
row['O'] = jsonObject.get("roomNumber")
row['G'] = jsonObject.get("province")
row['H'] = jsonObject.get("city")
row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("township")
row['K'] = jsonObject.get("street")
row['L'] = jsonObject.get("streetNumber")
row['M'] = jsonObject.get("buildingNumber")
row['N'] = jsonObject.get("unitNumber")
row['O'] = jsonObject.get("floor")
row['P'] = jsonObject.get("roomNumber")
}
}
@@ -81,23 +147,27 @@ class PocTest {
def addLoudongNum(path){
ExcelUtils.multiThreadingModify(path, { row ->
if (row['Q'] && row['L']) {
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
def commid = row['R']
def houseNum = row['M']
def unitNum = row['N']
println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum
if (commid && houseNum) {
def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum)
if (jsonObject && jsonObject.result) {
if(jsonObject.result.size()>=1){
def obj = jsonObject.result[0]
row['S'] = obj.id
row['R'] = obj.houseNumber
row['T'] = obj.id
row['S'] = obj.houseNumber
}
}
if(row['O']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
if(row['P']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject.result.size()>=1){
def obj = jsonObject.result[0]
row['U'] = obj.id
row['T'] = obj.roomNum
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
row['V'] = obj.id
row['U'] = obj.roomNum
}
}
}
@@ -111,12 +181,12 @@ class PocTest {
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
for(def row : datas){
def data = []
data << row['A']
data << row['B']
data << row['C']
data << row['D']
if (row['Q'] && row['L']) {
def jsonObject = OpenApiClient.searchLoudong(row['Q'], row['L'], row['M'])
data << row['E']
if (row['R'] && row['M']) {
def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N'])
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
def obj = jsonObject.result[0]
data << obj.houseNumber
@@ -144,8 +214,8 @@ class PocTest {
data << ""//景观
}
if(row['O']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['Q'], row['L'], row['M'], row['O'])
if(row['P']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
@@ -174,18 +244,20 @@ class PocTest {
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
for(def d : datas){
if(d['Q']){
def dist = CommonDatautils.findDistrictById(d['Q'])
def commid = d['R']
println "[addDetail] commid:"+commid
if(commid){
def dist = CommonDatautils.findDistrictById(commid)
if(dist){
def row = []
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
if(jsonObject) {
row << d['A']
row << d['B']
row << d['C']
row << d['D']
row << jsonObject.get("name")//小区名称
row << d['Q']//小区ID
row << commid//小区ID
if(jsonObject.get("alias")){
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
}else{
@@ -207,7 +279,7 @@ class PocTest {
row << jsonObject.get("buildingCount")//住宅幢数
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
row << SeaweedUtils.formatAllArea(jsonObject.get("totalUnitCount"))//占地面积
row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
}
@@ -271,28 +343,35 @@ class PocTest {
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
def keys = []
for(def d : datas){
if(d['Q']) {
def dist = CommonDatautils.findDistrictById(d['Q'])
def commid = d['R']
println "[createPaimai] commid:"+commid
if(commid) {
def dist = CommonDatautils.findDistrictById(commid)
if (dist) {
def key = dist.city+"#"+dist.region+"#"+dist.name
if(keys.contains(key)){
continue
}
keys << key
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,10)
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100)
if(jsonObject && jsonObject.result){
def results = jsonObject.result
for(def r: results){
def row = []
row << d['A']
row << d['B']
row << d['C']
row << d['D']
row << r.matchName
row << r.address
row << r.houseNum
row << r.roomNum
row << r.area
row << r.startDate
if(r.startDate){
row << r.startDate.substring(0, r.startDate.indexOf("T"))
}else{
row << ""
}
row << r.count
row << r.status
row << r.tradeAvgPrice
@@ -314,13 +393,15 @@ class PocTest {
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
if (d['Q']) {
def commid = d['R']
// println "[createHouse] commid:"+commid
if (commid) {
def key = d['Q']
if (keys.contains(key)) {
continue
}
keys << key
def jsonObject = OpenApiClient.houseList(d['Q'],"20210501",20)
def jsonObject = OpenApiClient.houseList(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
@@ -368,7 +449,7 @@ class PocTest {
rows << row
}
println d['Q'] +" "+results.size()
println d['R'] +" "+results.size()
}
}
}
@@ -381,13 +462,14 @@ class PocTest {
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
if (d['Q']) {
def key = d['Q']
if (keys.contains(key)) {
def commid= d['R']
println "[createEvaluation] commid:"+commid
if (commid) {
if (keys.contains(commid)) {
continue
}
keys << key
def jsonObject = OpenApiClient.evaluaionList(d['Q'],"20210501",20)
keys << commid
def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
@@ -432,7 +514,7 @@ class PocTest {
rows << row
}
println d['Q'] +" "+results.size()
println d['R'] +" "+results.size()
}
}
}
@@ -446,13 +528,14 @@ class PocTest {
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
if (d['Q']) {
def key = d['Q']
if (keys.contains(key)) {
def commid= d['R']
println "[createDealPrice] commid:"+commid
if (commid) {
if (keys.contains(commid)) {
continue
}
keys << key
def jsonObject = OpenApiClient.searchDealPrice(d['Q'],"20210501",20)
keys << commid
def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
@@ -501,7 +584,7 @@ class PocTest {
rows << row
}
println d['Q'] +" "+results.size()
println d['R'] +" "+results.size()
}
}
}
@@ -68,6 +68,10 @@ class CommonDatautils {
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -586,8 +590,8 @@ class CommonDatautils {
}
public static def findDistrict(String city, String region, String name, String cat){
public static def findDistrictCount(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
@@ -598,7 +602,24 @@ class CommonDatautils {
q.put("cat", cat)
q.put("status", 0)
def count = getDataStoreConnection().count(q);
return count;
}
public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
q.put("region", region);
}
q.put("name", name);
q.put("cat", cat)
q.put("status", 0)
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
@@ -609,6 +630,7 @@ class CommonDatautils {
}
public static def findDistrictList(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
@@ -704,7 +726,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","吴中路9弄小区")
// q.put("name","É­Íû·¼²ÝÂ̰¶")
DBObject keys = new BasicDBObject();
if(fields){
@@ -827,16 +849,24 @@ class CommonDatautils {
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("lat2", 1);
keys.put("lng2", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
keys.put("sid", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
try{
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
}
}
}catch(Exception e){
println "error_id:"+id
e.printStackTrace()
}
return null;
}
@@ -1053,9 +1083,16 @@ class CommonDatautils {
public static def queryDistrictById(def id){
def dbo = null
DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id));
def dbo = getDataStoreConnection().findOne(q);
try{
q.put("_id", new ObjectId(id.trim().toString()));
dbo = getDataStoreConnection().findOne(q);
}catch(Exception e){
println id
}
return dbo;
}
@@ -2385,6 +2422,7 @@ class CommonDatautils {
q.put("block", block);
}
q.put("cat", cat);
q.put("status",0)
def cur = getDataStoreConnection().find(q);
return cur.count();
}
@@ -3184,5 +3222,19 @@ class CommonDatautils {
def avg_radios = (int)(radios.sum()/radios.size())
return avg_radios;
}
static def findLoudong(commid){
def q = new BasicDBObject()
q.put("id_in_seaweed", commid)
def count = LOUDONG_LINE_COLL.count(q)
return count;
}
static def findHouseLoudong(commid){
def q = new BasicDBObject()
q.put("district_id", commid)
def count = HOUSE_LOUDONG_COLL.count(q)
return count;
}
}
@@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{
def run(){
def cityMap = CommonDatautils.getAllCityRegions()
cityMap.each{city,regions->
findPriceByCity(city, regions);
findPriceByCity(city, regions);
}
}
@@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{
q.put("region" , r.region)
q.put("month", r.month)
def mom = null
if(lastObj){
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
mom = DataHandleUtils.getDoubleRound(4, mom)
if(lastObj && lastObj.avg_price && r.price){
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
mom = DataHandleUtils.getDoubleRound(4, mom)
}
if(obj){
if(obj.avg_price != r.price){
@@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{
def findEsAvgPrice(city, region){
def detail = [:]
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1])
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0])
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
def must = []
@@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{
if(!priceValue){
priceValue = row['1-metric']
}
if(priceValue){
if(priceValue && priceValue.value){
price = priceValue.value.toInteger()
}
result.price = price;
def date = row.key_as_string.replace("-","").substring(0, 6)
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
if(date!=curMonth){
result.month = date.toInteger()
rets << result
if(date.toInteger()>200001 && result.price){
result.month = date.toInteger()
rets << result
}
}
}
}
@@ -0,0 +1,198 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeAppCrawl {
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeAppCrawl();
task.run();
// task.run2()
}
def run2(){
def url = "https://m.anjuke.com/fz/community/"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def links = getRegion(url, driver)
for(def link : links){
def doc = getDocument(url, driver)
def str = doc.toString().substring(doc.toString().indexOf("<script> window.__NUXT__"), doc.toString().indexOf("</script>"))
if(str){
str = str.substring(str.indexOf("} ("), str.indexOf("));"))
def arrs = str.split(",")
pritln arrs.size()
}
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def getRegion(url, driver){
//def doc = getDocument(url, driver)
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/fz/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
header.put("sw8-correlation","c291cmNl:YmVpZG91")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
def doc = ABuYunUtil.doGet(url, null, header, "utf-8")
def links = []
if(doc){
def eles = JQueryUtils.select(".column2 a", doc)
for(def e : eles){
def href = e.attr("href")
links << href
}
}
return links;
}
def run(){
//"仓山":4071,
def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
map.each{region, area_id->
for(def i=1;i<40;i++){
def stop = false;
def all_stop = false
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
//城口
def flagMap = getInfo(stop, i, area_id)
stop = flagMap.stop
all_stop = flagMap.all_stop
}
if(all_stop){
break;
}
}
}
}
def getInfo(def stop, def i, def area_id){
def map = [:]
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=35&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/fz/community/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
header.put("sw8-correlation","c291cmNl:YmVpZG91")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
def all_stop = false;
if (ret) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if(jsonObj && jsonObj.status=="ok" ){
def data = jsonObj.data
if(data && data.total.toInteger()>0){
def communities = data.communities
println "【${i}】 "+communities.size() +" "+url
if(communities.size()>0){
save("福州", communities)
Thread.sleep(10000)
stop = true;
}else{
stop = true;
println url +" 返回0"
all_stop = true;
}
}else{
stop = false;
println url +" 无数据"
}
}else{
stop = false;
Thread.sleep(5000)
}
}
map.stop = stop
map.all_stop = all_stop;
return map;
}
def save(city, def ret){
def index = 0
for(def o : ret){
index++
def obj = findIsExist(o.jumpAction)
if(!obj){
def udate = new BasicDBObject()
o.each{k,v->
udate.put(k,v)
}
udate.put("city", city)
udate.put("c_date", new Date())
detail.save(udate)
println index+" [保存]"+o.jumpAction
}else{
println index+" [已存在]"+o.jumpAction
}
}
}
def findIsExist(def url){
def q = new BasicDBObject()
q.put("jumpAction",url)
return detail.findOne(q);
}
}
@@ -11,9 +11,15 @@ import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.remote.DesiredCapabilities
import org.openqa.selenium.remote.RemoteWebDriver
class AnjukeDetailCrawl {
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url")
static main(args) {
def task = new AnjukeDetailCrawl();
task.run();
@@ -21,9 +27,15 @@ class AnjukeDetailCrawl {
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://chongqing.anjuke.com/community/?from=navigation"
def url = "https://chongqing.anjuke.com/"
WebDriver driver = new ChromeDriver()
// DesiredCapabilities capabilities = DesiredCapabilities.chrome();
// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL
// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities);
driver.manage().deleteAllCookies()
def regions = getRegions(driver, url)
println "区域链接: "+regions.size()
getAllRegions(driver, regions)
}
@@ -31,44 +43,88 @@ class AnjukeDetailCrawl {
def getAllRegions(driver, regions){
for(def url : regions){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def doc = getDocument(url, driver)
def total = getTotal(doc)
if(total) {
if (total > 1250) {
def links = getMoreLinks(url, driver)
println url +" "+links.size()
saveDetailLinks(links)
// parse(links,driver)
} else {
def links = getAllLinks(url, driver)
parse(links,driver)
println url +" "+links.size()
saveDetailLinks(links)
// parse(links,driver)
}
}
}
}
def saveDetailLinks(links){
for(def url : links){
def data = new BasicDBObject()
data.put("city","重庆")
data.put("url", url)
data.put("source", "安居客")
data.put("c_date", new Date())
urlColl.save(data)
}
}
def getMoreLinks(url, driver){
def doc = Jsoup.parse(driver.getPageSource());
def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc)
def links = []
if(pricelinks){
for(def link : pricelinks){
if(link.text()!="不限"){
def urlLink = link.attr("href")
getDocument(urlLink, driver)
// driver.get(urlLink)
links.addAll(getAllLinks(urlLink, driver))
println urlLink +" "+links.size()
}
}
}
return links;
}
def getAllLinks(url, driver){
def stop = false
def urls = []
while(!stop) {
def doc = Jsoup.parse(driver.getPageSource());
def doc = Jsoup.parse(driver.getPageSource());
def total = getTotal(doc)
def page = getPage(total, 25)
def pages = []
for(def i=1;i<=page;i++){
pages << url.substring(0, url.size()-1)+"-p"+i+"/"
}
for(def pageUrl : pages){
doc = getDocument(pageUrl, driver)
def details = JQueryUtils.select(".list-cell a.li-row", doc)
for (def d : details) {
urls << d.attr("href")
}
def nextEle = JQueryUtils.select(".next", doc)
if (nextEle) {
driver.findElement(By.cssSelector(".next")).click()
}else{
stop = true
}
if(urls.size()>20){
stop = true
def link = d.attr("href")
if(!urls.contains(link)){
urls << d.attr("href")
}
}
}
return urls
}
def getPage(def total, def pageSize){
if(total && total.toInteger()>0){
def page = total.toInteger()/pageSize;
if(page>(int)page){
page = (int)page + 1
return page
}
return (int)page;
}
return null
}
def getTotal(doc){
def totalText = JQueryUtils.selectText(".total-info", doc)
@@ -82,9 +138,19 @@ class AnjukeDetailCrawl {
}
def getRegions(driver, url){
// def doc = getDocument(url, driver)
driver.get(url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
def els = JQueryUtils.select("a:contains(小区)", doc)
if(els){
def href = els[0].attr("href")
doc = Jsoup.parse(driver.getPageSource());
// driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click()
// driver.findElement(By.linkText("小区")).click()
doc = getDocument(href, driver)
}
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
// if(loginIfm){
// def href = loginIfm[0].attr("src")
@@ -107,23 +173,35 @@ class AnjukeDetailCrawl {
def regionEles = JQueryUtils.select(".region-parents li a", doc)
def urls = []
for(def e : regionEles){
urls << e.attr("href")
if(e.text()!="不限"){
urls << e.attr("href")
}
}
return urls;
}
def getNontCheckPage(url, driver, doc){
def stop = false;
while(!stop){
if(doc){
}
}
return doc;
}
def parse(def links, driver){
def rets = []
for(def link : links){
def district = parsing(link, driver)
if(district){
rets << district
// rets << district
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
rets = []
}
// if (rets.size() % 100 == 0) {
saveObj(district)
println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
// rets = []
// }
}
}
if (rets.size()> 0) {
@@ -136,33 +214,61 @@ class AnjukeDetailCrawl {
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
saveObj(ret)
}
return add
}
def saveObj(def ret){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def parsing(url, driver){
@@ -191,10 +297,9 @@ class AnjukeDetailCrawl {
def infoDiv = JQueryUtils.select(".info-list", doc)
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text())
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
if (houseCountStr) {
houseCountStr = houseCountStr.replace("户", "")
@@ -202,9 +307,13 @@ class AnjukeDetailCrawl {
district.house_count = houseCountStr.toInteger()
}
}
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text())
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
if(carSiteStr){
if(carSiteStr.contains("(")){
@@ -214,16 +323,19 @@ class AnjukeDetailCrawl {
district.car_site = carSiteStr
}
}
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text()
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
district.url = url
district.c_date = new Date()
district.m_date = new Date()
district.crawl_type = "chrome"
district.root = "anjuke.com"
stop = true;
}else{
Thread.sleep(3000)
@@ -0,0 +1,89 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class AnjukeDistrictMatch {
static main(args) {
def task = new AnjukeDistrictMatch();
task.run();
// task.run1()
}
def run1(){
def regionPointsMap = findRegionRange("重庆")
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
}
def run(){
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
ExcelUtils.multiThreadingModify(filePath,{row->
if(row['A'] && !row['J']){
def city = row['B']
def region = row['C']
def name = row['D']
def address = row['E']
// def flag = "别名匹配"
// def dist = matchDist(city, null, name)
// if(!row['J'] && name.contains("(")){
// name = name.replace("(${region})","").replace("(","").replace(")","")
// flag = "去括号匹配"
// dist = matchDist(city, region, name)
// }
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
// flag = "映射别名匹配"
// }
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
// if(rets && rets.size()==1){
// dist = rets[0]
// }
// }
//
// if(!dist){
// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district")
// flag = "地址匹配"
// if(dists.size()>1){
// println address +" 地址相同:"+dists.size()
// }else{
// if(dists.size()==1){
// dist = dists[0]
// }
// }
// }
// if(dist){
// row['J'] = flag
// row['K'] = dist._id.toString()
// row['L'] = dist.region
// row['M'] = dist.name
// row['N'] = dist.address
// }
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
if(jsonObject != null && jsonObject.precise!=-1) {
// row['J'] = "匹配接口"
// row['K'] = jsonObject.get("id")
// row['L'] = jsonObject.get("district")
// row['M'] = jsonObject.get("name")
// row['N'] = jsonObject.get("address")
row['M'] = jsonObject.get("name")
}
}
})
}
def matchDist(city, region, name){
def dist = CommonDatautils.findDistrict(city, region, name, "district")
return dist;
}
}
@@ -0,0 +1,628 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
class AnjukeToDB {
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeToDB();
// task.run();
// task.addNoMatch()
task.updateEstateType()
// task.gaodeRangeData()
}
def updateEstateType(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1)
for(def row : datas){
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj) {
def type = getEstatType(obj.extend.type)
def commId = row['L']
if(!commId){
commId = row['G']
}
if(type){
if(type=="别墅"){
def udata = new BasicDBObject()
udata.put("estate_type2", type)
udata.put("m_date", new Date())
println commId
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata))
}
}else{
// def udata = new BasicDBObject()
// udata.put("status", 1)
// udata.put("name_cat",11)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata))
}
}
}
}
def gaodeRangeData(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1)
def index = 0
for(def d : datas){
index++
println index
def personName = d['AB']
if(personName){
def commid = d['J']
if(commid){
def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district")
if(obj){
updateDist(d, obj._id.toString())
}
}else{
saveGaode(d)
}
}else{
def commid = d['J']
updateDist(d, commid)
}
}
}
def saveGaode(d){
def oldName = d['E']
def distName = nameFileter(d['C'], oldName)
def udata = new BasicDBObject()
def alias = []
def aliasStr = d['F']
if(aliasStr){
def arrs = aliasStr.split("/")
for(def s : arrs){
alias << s
}
}
if(alias.size()>0){
udata.put("alias", alias)
}
def address = AddressUtil.formatAddress("重庆", d['G'])
if(address){
udata.put("address", address)
}
if(d['N']){
udata.put("virescence_percent", d['N'])
}
if(d['O']){
udata.put("estate_charge", d['O'])
}
if(d['Q']){
udata.put("cubage_percent", d['Q'])
}
if(d['R']){
udata.put("developer", d['R'])
}
if( d['S']){
udata.put("build_area", d['S'])
}
if(d['T']){
def t = d['T'].substring(0, 3)
udata.put("build_date", t)
}
if(d['V'] && d['V'].isNumber()){
udata.put("car_site_count", d['V'].toInteger())
}
if(d['Y']){
def build_type = getBuildType2(d['Y'])
if(build_type){
udata.put("build_type", build_type)
}
}
if(d['Z']){
udata.put("property_life", d['Z'])
}
def lat = null
def lng = null
if(d['H']){
lat = d['H']
lng = d['I']
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
Long dlat = lat.toDouble() * 10000000000L
Long dlng = lng.toDouble() * 10000000000L
udata.put("lat", dlat)
udata.put("lng", dlng)
}
def type = "普通住宅"
if(distName.contains("别墅")) {
type = "别墅"
}else if(distName.contains("商住")){
type = "商住"
}
if(udata){
udata.put("name_cat",0)
udata.put("status",0)
udata.put("cat","district")
udata.put("c_date", new Date())
udata.put("m_date", new Date())
udata.put("city", "重庆")
udata.put("region",d['C'])
udata.put("name", distName)
udata.put("estate_type2",type)
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map",markMap)
DbUtils.seaweed.save(udata);
def stop = false
while (!stop){
if(udata._id.toString()){
addPrice(udata)
stop = true;
}
}
}
}
def updateDist(d, commid){
if(commid){
def dist = CommonDatautils.queryDistrictById(commid)
if(dist){
def oldName = d['E']
def distName = nameFileter(d['C'], oldName)
def udata = new BasicDBObject()
if(!distName.contains("号")){
def alias = dist.alias
if(!alias){
alias = []
}
if(!alias.contains(distName) && dist.name!=distName){
alias << distName
}
def aliasStr = d['F']
if(aliasStr){
def arrs = aliasStr.split("/")
for(def s : arrs){
alias << s
}
}
if(alias.size()>0 && alias!=dist.alias){
udata.put("alias", alias)
}
}
if(dist.address && !dist.address.contains("号")){
def address = AddressUtil.formatAddress("重庆", d['G'])
if(address){
udata.put("address", address)
}
}
if(!dist.virescence_percent && d['N']){
udata.put("virescence_percent", d['N'])
}
if(!dist.estate_charge && d['O']){
udata.put("estate_charge", d['O'])
}
if(!dist.cubage_percent && d['Q']){
udata.put("cubage_percent", d['Q'])
}
if(!dist.developer && d['R']){
udata.put("developer", d['R'])
}
if(!dist.build_area && d['S']){
udata.put("build_area", d['S'])
}
if(!dist.build_date && d['T']){
if(d['T'].size()>=4){
def t = d['T'].substring(0, 4)
udata.put("build_date", t)
}
}
if(!dist.car_site_count && d['V'] && d['V'].isNumber()){
udata.put("car_site_count", d['V'].toInteger())
}
if(!dist.build_type && d['Y']){
def build_type = getBuildType2(d['Y'])
if(build_type){
udata.put("build_type", build_type)
}
}
if(!dist.property_life && d['Z']){
udata.put("property_life", d['Z'])
}
def lat = null
def lng = null
if(d['H']){
lat = d['H']
lng = d['I']
def slat = dist.lat/10000000000.0
def slng = dist.lng/10000000000.0
def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat);
if(distance>1000){
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
Long dlat = lat.toDouble() * 10000000000L
Long dlng = lng.toDouble() * 10000000000L
udata.put("lat", dlat)
udata.put("lng", dlng)
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改] ${dist.name}"+ udata
}
}
}
}
def addNoMatch(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1)
for(def d : datas){
add(d)
}
}
def run(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-芝芝.xlsx",0,1)
for(def data : datas){
if(data['K']=="芝芝"){
def flag = data['M']
if(flag && flag=="新增"){
add(data)
}else{
update(data)
}
}
}
}
def add(def row){
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj){
def type = getEstatType(obj.extend.type)
if(type){
def city = obj.base.cityName
def region = obj.base.areaName
def lat2 = null
def lng2 = null
def lat = null
def lng = null
if(obj.base.lat){
lat2 = obj.base.lat
lng2 = obj.base.lng
JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu")
if(convertObj.getInt("status") == 1) {
String lo = convertObj.get("locations")
lng = Double.parseDouble(lo.split(",")[0])
lat = Double.parseDouble(lo.split(",")[1])
}
}
def distName = nameFileter(obj.base.areaName, obj.base.name)
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
def address = AddressUtil.formatAddress(obj.base.city, obj.base.address)
def dist = CommonDatautils.findDistrict(city, region, distName, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", distName)
data.put("estate_type2",type)
if(address){
data.put("address", obj.base.address)
}
if(lat){
Long alat = lat*10000000000L
Long alng = lng*10000000000L
data.put("lat", alat)
data.put("lng", alng)
Long alat2 = lat2.toDouble()*10000000000L
Long alng2 = lng2.toDouble()*10000000000L
data.put("lat2",alat2)
data.put("lng2",alng2)
}
data.put("finish_building_date",obj.extend.completionTime)
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){
data.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
}
data.put("build_area", obj.extend.totalArea)
data.put("virescence_percent", obj.extend.landscapingRatio)
data.put("build_type", buildTypeStr)
data.put("business_quarter",obj.base.tradingAreaName)
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
DbUtils.seaweed.save(data);
def stop = false
while (!stop){
if(data._id.toString()){
addPrice(data)
stop = true;
}
}
}
}else{
println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type
}
}else{
println anjukeId+" 没找到"
}
}
def update(def row){
def commId = row['L']
if(!commId){
commId = row['G']
}
def dist = CommonDatautils.queryDistrictById(commId)
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj && dist){
def udata = new BasicDBObject()
def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address)
if(address){
if(!dist.address){
if(obj.base.address){
udata.put("address",obj.base.address)
}
}else{
if(dist.address.contains("号")){
if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){
def addr_alias = dist.addr_alias
if(!addr_alias){
addr_alias = []
}
if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){
addr_alias << obj.base.address
}
if(addr_alias!=dist.addr_alias && addr_alias.size()>0){
udata.put("addr_alias", addr_alias)
}
}
}else{
udata.put("address",obj.base.address)
}
}
}
def distName = nameFileter(obj.base.areaName, obj.base.name)
if(!distName.contains("号")){
def alias = dist.alias
if(!alias){
alias = []
}
if(alias.contains(distName)){
alias << distName
}
if(alias.size()>0 && alias!=dist.alias){
udata.put("alias", alias)
}
}
if(!dist.build_type){
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
if(buildTypeStr){
udata.put("build_type",buildTypeStr)
}
}
if(!dist.household_count){
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){
udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
}
}
if(!dist.finish_building_date){
if(obj.extend.completionTime){
udata.put("finish_building_date",obj.extend.completionTime)
}
}
if(!dist.virescence_percent){
if(obj.extend.landscapingRatio){
udata.put("virescence_percent", obj.extend.landscapingRatio)
}
}
if(!dist.business_quarter){
if(obj.base.tradingAreaName){
udata.put("business_quarter",obj.base.tradingAreaName)
}
}
def lat2 = null
def lng2 = null
if(dist.lat2){
lat2 = dist.lat2/10000000000.0
lng2 = dist.lng2/10000000000.0
}
if(dist.lat && !lat2){
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3)
lng2 = baidu.x
lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
}
if(obj.base.lat) {
def alat2 = obj.base.lat.toDouble()
def alng2 = obj.base.lng.toDouble()
def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2);
if(distance>1000){
println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................"
def lng = null
def lat = null
JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu")
if(convertObj.getInt("status") == 1) {
String lo = convertObj.get("locations")
lng = Double.parseDouble(lo.split(",")[0])
lat = Double.parseDouble(lo.split(",")[1])
}
Long blat2 = alat2 *10000000000L
Long blng2 = alng2 *10000000000L
Long gLat2 = lat *10000000000L
Long gLng2 = lng *10000000000L
udata.put("lat2", blat2)
udata.put("lng2", blng2)
udata.put("lat", gLat2)
udata.put("lng", gLng2)
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改] ${dist.name}"+ udata
}
}
}
def findAnjukeDistrictDetail(anjukeId){
def q = new BasicDBObject()
q.put("_id",new ObjectId(anjukeId))
return detail.findOne(q)
}
def nameFileter(def region, def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
name = name.replace("(${region})","").replace("(","").replace(")","")
return name;
}
def getEstatType(type){
if(type){
if(type=="公寓住宅"){
type = "普通住宅"
}else if(type.contains("商住")){
type = "商住"
}else if(type.contains("别墅")){
type = "别墅"
}else{
type = null
}
}
return type
}
def getBuildType(def buildType){
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
def list = []
if(buildType){
def arrs = buildType.split("\\|")
for(def s : arrs){
if(types.contains(s)){
list << s
}
}
if(list){
return list.join(",")
}
}
return null
}
def getBuildType2(def buildType){
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
def list = []
if(buildType){
def arrs = buildType.split(",| ")
for(def s : arrs){
if(types.contains(s)){
list << s
}
}
if(list){
return list.join(",")
}
}
return null
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
}
@@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -358,13 +358,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(e.text().contains("ÖØÇì")){
// if(e.text().contains("¸£ÖÝ")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
}
// }
}
}
return list
@@ -29,7 +29,7 @@ class JdPaimaiCrawl {
task.run();
}
def run(){
def existsIds = findAllIds()
def existsIds = []//findAllIds()
println "ÒÑ´æÔÚ:"+existsIds.size()
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
@@ -43,78 +43,80 @@ class JdPaimaiCrawl {
def getList(driver){
def list = []
for(def i=1;i<=32;i++){
if(i==8){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
}else if(i==5){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
}else if(i==17){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
}else if(i==19){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
}else{
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
}
}
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268"
// for(def i=1;i<=32;i++){
// if(i==8){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
// }else if(i==5){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
// }else if(i==17){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
// }else if(i==19){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
// }else{
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
// }
// }
def links = []
for(def url : list){
driver.get(url)
@@ -430,10 +432,10 @@ class JdPaimaiCrawl {
def s1 = JQueryUtils.selectText(["li:contains(¾ºÂò¹«¸æ) .floor-content"], doc)
if(s1) {
def start = s1.indexOf("·¨Ôº½«ÓÚ")
def start = s1.indexOf("将于")
def end = s1.indexOf("Ö¹")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("·¨Ôº½«ÓÚ"), s1.indexOf("Ö¹"))
s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止"))
}
def dateRangs = DataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
@@ -476,6 +478,11 @@ class JdPaimaiCrawl {
}
}
if(!result.startDate){
def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","")
result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日")
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
@@ -34,10 +34,30 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
task.run()
// task.run()
// task.run1()
task.addDealRegion();
task.standardRegion()
// task.addDealRegion();
// task.standardRegion()
task.updadteDealPriceField()
}
def updadteDealPriceField(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def city : citys){
index ++
def q = new BasicDBObject()
q.put("city", city)
q.put("hall", "0")
def count = DEAL_PRICE_COLL.count(q)
println "[${index}] "+city +" "+count
if(count && count>0){
def udata = new BasicDBObject()
udata.put("hall", 0)
DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
println "[ÐÞ¸Ä]"+city +" "+ count
}
}
}
def run1(){
@@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["³¤´º","¼ªÁÖ"]
def citys= ["¸£ÖÝ"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
@@ -30,8 +30,8 @@ class PaimaiTest {
def task = new PaimaiTest();
// task.run();
// task.stat()
// task.updatePaimai()
task.updateFieldPaimai()
task.updatePaimai()
// task.updateFieldPaimai()
}
def updateFieldPaimai(){
@@ -42,8 +42,8 @@ class PaimaiTest {
for(def obj : dbcursor) {
index++
println index
if(obj.depositPrice){
obj.put("depositPrice", obj.depositPrice.toInteger())
if(obj.address && obj.address.contains("可贷款")){
obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
}
}
@@ -51,11 +51,22 @@ class PaimaiTest {
def updatePaimai(){
def q = new BasicDBObject()
q.put("url", Pattern.compile(".*jd.com.*"))
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def index =0
for(def d : dbcursor){
index++
println index
if(!d.url.contains("jd.com")){
continue
}
if(!d.proince.contains("重庆")){
continue
}
println d.url
def query = new BasicDBObject()
query.put("url", d.url)
def dbo = paimai43_COLL.findOne(query);
@@ -42,8 +42,41 @@ class CqfdcMatch {
// task.export2()
// task.matchLoudong()
task.addLoudong()
// task.addLoudong()
// task.addFdcLoudong()
task.updateHouseLoudong()
}
def updateHouseLoudong(){
def q = new BasicDBObject()
q.put("city", "ÖØÇì")
def keys = new BasicDBObject()
keys.put("room_num", 1)
keys.put("house_floor", 1)
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
def index = 0
for(def d : dbcursor){
index++
def room_num = d.room_num
def house_floor = d.house_floor
try{
if(room_num && house_floor && room_num.isNumber()){
if(house_floor.toInteger()>0){
if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){
def newValue = house_floor +"-"+room_num
println "[${index}] "+room_num +" "+house_floor+" "+newValue
def udata = new BasicDBObject()
udata.put("room_num",newValue)
HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false);
}
}
}
}catch(Exception e){
println d._id.toString()
}
}
}
def export(){
def q = new BasicDBObject()
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "58ͬ³Ç"//
def source = "±´¿ÇÕÒ·¿"//
def citys = findCrawlCity(source, 202202);
for(def city : citys){
def count1 = findCount(source, city, 202202);
def count2 = findCount(source, city, 202203);
def count1 = findCount(source, city, 202203);
def count2 = findCount(source, city, 202204);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("¸£ÖÝ","https://fz.58.com/xiaoqu/", 70);
// def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
def citys = ["上海","北京","广州","天津","武汉","南京","青岛","合肥","西安","沈阳","宁波","佛山","潍坊","济南","大连","兰州","石家庄","长春","温州","乌鲁木齐","太原","惠州","嘉兴","南通","南昌","烟台","临沂","呼和浩特","厦门","海口","常州","中山","绍兴","淄博","洛阳","珠海","金华","威海","德阳","唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","衡阳","滨州","张家口","赣州","商丘","安庆","湖州","枣庄","聊城","阜阳","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳","蚌埠","九江","临汾","大同","六安","淮南","宿州","三亚","玉溪","六盘水","揭阳","汕尾","恩施","盘锦","伊犁","玉林","随州","抚州","景德镇","四平","丽水","安顺","毕节","天水","阜新","通化","攀枝花","吕梁","阿克苏","新余","松原","楚雄","鹤壁","忻州","本溪","拉萨","三门峡","黔东南","兴安盟","雅安","仙桃","乌海","贵港","安康","白城","平凉","庆阳","延安","潮州","白银","黔西南","丽江","吴忠","防城港","喀什","哈尔滨","钦州","鸡西","石嘴山","哈密","萍乡","鹰潭","武威","保山","铜仁","普洱","酒泉","宜昌","河池","朔州","百色","鹤岗","张家界","湘西","张掖","定西","昭通","济源","中卫","黑河","克拉玛依","铜川","辽源","云浮","博尔塔拉","固原","白山","广元","贺州","潜江","天门","伊春","石河子","临沧","榆林","临夏","来宾","成都","双鸭山","梧州","嘉峪关","海东","昌吉"]
// for(def city : citys){
// def str = cityList.get(city)
// for(def i=0;i<1;i++){
@@ -209,7 +210,7 @@ class Esf58Crawl extends BaseBatch{
cityList.each{city, str->
for(def i=0;i<1;i++){
// if(citys.contains(city)){
if(citys.contains(city)){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]);
if(districts){
def arrs = str.split("\\|")
@@ -219,7 +220,7 @@ class Esf58Crawl extends BaseBatch{
addCrawlKeys(city);
addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]);
}
// }
}
}
}
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -26,7 +26,7 @@ class SynchDistrictPrice {
def run(){
def citys = CommonUtils.findAllCity()
def citys = ["重庆"]//CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
@@ -236,10 +236,10 @@ public class OpenApiClient {
params.put("houseNumber", houseNum);
params.put("unitNumber", unitNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchLoudong", params, "utf-8")
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
println("${id} ${houseNum} 程序调用异常")
Thread.sleep(6000)
}
@@ -266,10 +266,10 @@ public class OpenApiClient {
params.put("unitNumber", unitNum);
params.put("roomNumber", roomNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
println("${id} ${houseNum} 程序调用异常")
Thread.sleep(6000)
}
@@ -687,7 +687,7 @@ public class OpenApiClient {
params.put("region", region)
params.put("name", name)
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/district/facilitys", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
@@ -697,13 +697,14 @@ public class OpenApiClient {
return null
}
static JSONObject searchPaimais(String city, String region, String name, limit) {
static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city)
params.put("district", region)
params.put("name", name)
params.put("limit", "10")
params.put("startDate", startDate)
params.put("limit", "100")
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/auction/searchPaimais", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
@@ -722,7 +723,7 @@ public class OpenApiClient {
params.put("startDate", startDate)
params.put("limit", limit.toString())
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/dealPrice/searchList", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
@@ -738,7 +739,7 @@ public class OpenApiClient {
params.put("startTime", startDate)
params.put("pageSize", limit.toString())
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/house/list", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
@@ -755,7 +756,7 @@ public class OpenApiClient {
params.put("startDate", startDate)
params.put("limit", limit.toString())
try {
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/evaluation/bsList", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
@@ -0,0 +1,9 @@
package com.fangjia.service.open;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityBasePriceService {
CommunityBasePrice getById(String id);
}
@@ -0,0 +1,22 @@
package com.fangjia.service.open.impl;
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.service.open.CommunityBasePriceService;
import com.fangjia.service.open.CommunityPoiService;
public class CommunityBasePriceServiceImpl implements CommunityBasePriceService {
private CommunityBasePriceDao communityBasePriceDao;
@Override
public CommunityBasePrice getById(String id) {
return communityBasePriceDao.findById(id);
}
public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) {
this.communityBasePriceDao = communityBasePriceDao;
}
}