Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+17
@@ -2013,6 +2013,23 @@ public class DistrictController {
|
||||
if (newName != null || newRegion != null) {
|
||||
districtUpdateTaskService.createTask(orgDis, 2, newName, newRegion);
|
||||
districtUpdateTaskService.createTask(orgDis, 6, newName, newRegion);
|
||||
|
||||
District distTask = new District();
|
||||
distTask.setCity(orgDis.getCity());
|
||||
if(StringUtils.isNotBlank(newName)){
|
||||
distTask.setRegion(newRegion);
|
||||
}else{
|
||||
distTask.setRegion(orgDis.getRegion());
|
||||
}
|
||||
|
||||
distTask.setEstateType2(orgDis.getEstateType2());
|
||||
if(StringUtils.isNotBlank(newName)){
|
||||
distTask.setName(newName);
|
||||
}else{
|
||||
distTask.setName(orgDis.getName());
|
||||
}
|
||||
|
||||
districtUpdateTaskService.createTask(distTask, 5);
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(district.getEstateType2())
|
||||
|
||||
@@ -365,7 +365,7 @@
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="bg"> 小区性质</td>
|
||||
<td class="bg">权属类别</td>
|
||||
<td>
|
||||
<select id="landProperty" name="district.landProperty">
|
||||
<option value=""></option>
|
||||
|
||||
@@ -590,7 +590,7 @@
|
||||
<#else>
|
||||
<i class="unlock" name="land_property"></i>
|
||||
</#if>
|
||||
小区性质
|
||||
权属类别
|
||||
</td>
|
||||
<td>
|
||||
<select id="landProperty" name="district.landProperty">
|
||||
|
||||
+5
@@ -94,6 +94,11 @@ public class DistrictUpdateTaskDaoImpl extends MongoDaoImpl<DistrictUpdateTask>
|
||||
task.setCreateDate(new Date());
|
||||
task.setStatus(5);
|
||||
task.setName(district.getName());
|
||||
if(district.getEstateType2().contains("ąđĘű")){
|
||||
task.setEstateType("villa");
|
||||
}else{
|
||||
task.setEstateType("district");
|
||||
}
|
||||
this.save(task);
|
||||
}
|
||||
|
||||
|
||||
+66
-69
@@ -746,78 +746,75 @@ public class PropertyController {
|
||||
}
|
||||
}
|
||||
|
||||
if (model.getDetailLevel() != null && model.getDetailLevel() >= 1) {
|
||||
districtInfoDto.setBlock(district.getBlock());
|
||||
districtInfoDto.setBlock(district.getBlock());
|
||||
districtInfoDto.setLoopLine(district.getLoopLineInfo());
|
||||
if (HouseValidator.validateBuildingType(district.getBuildType())) {
|
||||
districtInfoDto.setBuildingType(district.getBuildType());
|
||||
} else {
|
||||
districtInfoDto.setBuildingType("");
|
||||
}
|
||||
districtInfoDto.setAlias(district.getMappingAlias());
|
||||
districtInfoDto.setAddressAlias(district.getAddrAlias());
|
||||
districtInfoDto.setCommunity(district.getStreetCommunity());
|
||||
districtInfoDto.setCommercialArea(district.getBusinessQuarter());
|
||||
if (district.getHouseholdCount() != null) {
|
||||
districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString()));
|
||||
} else {
|
||||
districtInfoDto.setTotalUnitCount("");
|
||||
}
|
||||
districtInfoDto.setManagementFee(district.getEstateCharge());
|
||||
if (district.getCarSiteCount() != null) {
|
||||
districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString()));
|
||||
} else {
|
||||
districtInfoDto.setParking("");
|
||||
}
|
||||
districtInfoDto.setParkingDescription(district.getCarSite());
|
||||
districtInfoDto.setOwnershipPeriod(district.getPropertyLife());
|
||||
districtInfoDto.setOwnershipType(district.getHouseProperty());
|
||||
districtInfoDto.setStartBuildTime(district.getStartBuildingDate());
|
||||
if (district.getOpenDate() != null) {
|
||||
Date minDate = DateUtil.parseDate("1900-01-01", "yyyy-MM-dd");
|
||||
Date maxDate = DateUtil.parseDate("2025-01-01", "yyyy-MM-dd");
|
||||
if (district.getOpenDate().after(minDate) && district.getOpenDate().before(maxDate)) {
|
||||
districtInfoDto
|
||||
.setStartSellingTime(DateUtil.formatByPattern(district.getOpenDate(), "yyyy-MM-dd"));
|
||||
}
|
||||
} else {
|
||||
districtInfoDto.setStartSellingTime("");
|
||||
}
|
||||
|
||||
if (model.getDetailLevel() != null && model.getDetailLevel() >= 2) {
|
||||
districtInfoDto.setLoopLine(district.getLoopLineInfo());
|
||||
if (HouseValidator.validateBuildingType(district.getBuildType())) {
|
||||
districtInfoDto.setBuildingType(district.getBuildType());
|
||||
} else {
|
||||
districtInfoDto.setBuildingType("");
|
||||
}
|
||||
districtInfoDto.setAlias(district.getMappingAlias());
|
||||
districtInfoDto.setAddressAlias(district.getAddrAlias());
|
||||
districtInfoDto.setCommunity(district.getStreetCommunity());
|
||||
districtInfoDto.setCommercialArea(district.getBusinessQuarter());
|
||||
if (district.getHouseholdCount() != null) {
|
||||
districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString()));
|
||||
} else {
|
||||
districtInfoDto.setTotalUnitCount("");
|
||||
}
|
||||
districtInfoDto.setManagementFee(district.getEstateCharge());
|
||||
if (district.getCarSiteCount() != null) {
|
||||
districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString()));
|
||||
} else {
|
||||
districtInfoDto.setParking("");
|
||||
}
|
||||
districtInfoDto.setParkingDescription(district.getCarSite());
|
||||
districtInfoDto.setOwnershipPeriod(district.getPropertyLife());
|
||||
districtInfoDto.setOwnershipType(district.getHouseProperty());
|
||||
districtInfoDto.setStartBuildTime(district.getStartBuildingDate());
|
||||
if (district.getOpenDate() != null) {
|
||||
Date minDate = DateUtil.parseDate("1900-01-01", "yyyy-MM-dd");
|
||||
Date maxDate = DateUtil.parseDate("2025-01-01", "yyyy-MM-dd");
|
||||
if (district.getOpenDate().after(minDate) && district.getOpenDate().before(maxDate)) {
|
||||
districtInfoDto
|
||||
.setStartSellingTime(DateUtil.formatByPattern(district.getOpenDate(), "yyyy-MM-dd"));
|
||||
}
|
||||
} else {
|
||||
districtInfoDto.setStartSellingTime("");
|
||||
}
|
||||
|
||||
districtInfoDto.setFinishBuildTime(district.getFinishBuildingDate());
|
||||
districtInfoDto.setMoveinTime(district.getLiveDate());
|
||||
districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea()));
|
||||
districtInfoDto.setConstructionSpace(district.getBuildArea());
|
||||
districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent()));
|
||||
districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent()));
|
||||
districtInfoDto.setPreSalePermit(district.getLicence());
|
||||
districtInfoDto.setDeveloper(district.getDeveloper());
|
||||
districtInfoDto.setDiscount(district.getDiscount());
|
||||
districtInfoDto.setPropertyManagement(district.getEstateCompany());
|
||||
if (HouseValidator.fetchFitmentInfo(district.getFitment()) == null) {
|
||||
districtInfoDto.setDecorationStandard("");
|
||||
} else {
|
||||
districtInfoDto.setDecorationStandard(HouseValidator.fetchFitmentInfo(district.getFitment()));
|
||||
}
|
||||
districtInfoDto.setBuildingStructure(district.getBuildStructure());
|
||||
districtInfoDto.setSupportingFacilities(district.getFacilities());
|
||||
districtInfoDto.setTransportation(district.getTraffic());
|
||||
districtInfoDto.setRelatedInfo(district.getRelateInfo());
|
||||
districtInfoDto.setDescription(district.getDescription());
|
||||
districtInfoDto.setSalesOfficeAdd(district.getSellAddress());
|
||||
districtInfoDto.setSalesPhoneNumber(district.getMobile());
|
||||
districtInfoDto.setTownship(district.getTownship());
|
||||
if (district.getLoudongCount() != null && district.getLoudongCount() > 0) {
|
||||
districtInfoDto.setBuildingCount(district.getLoudongCount());
|
||||
}
|
||||
Integer finishYear = districtService.getBuildYear(district);
|
||||
if (finishYear != null) {
|
||||
districtInfoDto.setFinishYear(finishYear);
|
||||
}
|
||||
districtInfoDto.setFinishBuildTime(district.getFinishBuildingDate());
|
||||
districtInfoDto.setMoveinTime(district.getLiveDate());
|
||||
districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea()));
|
||||
districtInfoDto.setConstructionSpace(district.getBuildArea());
|
||||
districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent()));
|
||||
districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent()));
|
||||
districtInfoDto.setPreSalePermit(district.getLicence());
|
||||
districtInfoDto.setDeveloper(district.getDeveloper());
|
||||
districtInfoDto.setDiscount(district.getDiscount());
|
||||
districtInfoDto.setPropertyManagement(district.getEstateCompany());
|
||||
if (HouseValidator.fetchFitmentInfo(district.getFitment()) == null) {
|
||||
districtInfoDto.setDecorationStandard("");
|
||||
} else {
|
||||
districtInfoDto.setDecorationStandard(HouseValidator.fetchFitmentInfo(district.getFitment()));
|
||||
}
|
||||
districtInfoDto.setBuildingStructure(district.getBuildStructure());
|
||||
districtInfoDto.setSupportingFacilities(district.getFacilities());
|
||||
districtInfoDto.setTransportation(district.getTraffic());
|
||||
districtInfoDto.setRelatedInfo(district.getRelateInfo());
|
||||
districtInfoDto.setDescription(district.getDescription());
|
||||
districtInfoDto.setSalesOfficeAdd(district.getSellAddress());
|
||||
districtInfoDto.setSalesPhoneNumber(district.getMobile());
|
||||
districtInfoDto.setTownship(district.getTownship());
|
||||
if (district.getLoudongCount() != null && district.getLoudongCount() > 0) {
|
||||
districtInfoDto.setBuildingCount(district.getLoudongCount());
|
||||
}
|
||||
// Integer finishYear = districtService.getBuildYear(district);
|
||||
Integer finishYear = districtService.getCreateYear(district);
|
||||
if (finishYear != null) {
|
||||
districtInfoDto.setFinishYear(finishYear);
|
||||
}
|
||||
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(districtInfoDto));
|
||||
return mav;
|
||||
}
|
||||
|
||||
@@ -433,11 +433,17 @@
|
||||
<artifactId>javadbf</artifactId>
|
||||
<version>1.9.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<!-- <dependency>
|
||||
<groupId>ru.yandex.clickhouse</groupId>
|
||||
<artifactId>clickhouse-jdbc</artifactId>
|
||||
<version>0.1.41</version>
|
||||
</dependency>
|
||||
</dependency>-->
|
||||
<dependency>
|
||||
<groupId>ru.yandex.clickhouse</groupId>
|
||||
<artifactId>clickhouse-jdbc</artifactId>
|
||||
<version>0.2.6</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>fastjson</artifactId>
|
||||
|
||||
+6
-5
@@ -30,12 +30,12 @@ public class ExportAvgPrice {
|
||||
DSCollection dataStoreConnection = null;
|
||||
static main(args) {
|
||||
def task = new ExportAvgPrice();
|
||||
def weekends = ["2019-1-1","2019-2-1","2019-3-1","2019-4-1","2019-5-1","2019-6-1","2019-7-1","2019-8-1", "2019-9-1","2019-10-1","2019-11-1","2019-12-1","2020-1-1","2020-2-1","2020-3-1","2020-4-1","2020-5-1","2020-6-1","2020-7-1","2020-8-1", "2020-9-1","2020-10-1","2020-11-1","2020-12-1","2021-1-1","2021-2-1","2021-3-1","2021-4-1","2021-5-1","2021-6-1","2021-7-1","2021-8-1", "2021-9-1","2021-10-1"];
|
||||
def citys = ["温州"];//findAllCity();
|
||||
// task.exportAllCity(citys, weekends, "e:\\全国城市季度均价.csv");
|
||||
task.exportRegion(citys, weekends, "d:\\温州区域季度均价.csv");
|
||||
def weekends = ["2023-04-01","2023-03-01","2023-02-01","2023-01-01","2022-12-01","2022-11-01","2022-10-01","2022-09-01","2022-08-01","2022-07-01","2022-06-01","2022-05-01","2022-04-01","2022-03-01","2022-02-01","2022-01-01","2021-12-01","2021-11-01","2021-10-01","2021-09-01","2021-08-01","2021-07-01","2021-06-01","2021-05-01","2021-04-01"]
|
||||
def citys = ["上海"];//findAllCity();
|
||||
// task.exportAllCity(citys, weekends, "D:\\数据\\小区\\全国城市季度均价.csv");
|
||||
// task.exportRegion(citys, weekends, "D:\\数据\\小区\\区域季度均价.csv");
|
||||
// task.exportBlock(citys, weekends, "e:\\区域板块季度均价.csv");
|
||||
// task.exportDistrict(citys, weekends, "e:\\小区季度均价.csv");
|
||||
task.exportDistrict(citys, weekends, "D:\\数据\\小区\\小区季度均价.csv");
|
||||
|
||||
// task.exportRegionPrice("江苏");
|
||||
}
|
||||
@@ -182,6 +182,7 @@ public class ExportAvgPrice {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", "district");
|
||||
q.put("city", city);
|
||||
// q.put("name", "龙柏二村")
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q).sort(new BasicDBObject('bscore', -1));
|
||||
Iterator<DBObject> iters = cursor.iterator()
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
for(def d : datas){
|
||||
def map = new BasicDBObject()
|
||||
map.city = "深圳"
|
||||
map.district_name = d['D']
|
||||
map.region = d['B']
|
||||
def roomStr = d['E']
|
||||
if(roomStr){
|
||||
def arrs = roomStr.split("-")
|
||||
map.room = arrs[0].toString().toInteger()
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
}
|
||||
|
||||
map.area = d['F'].toString().toDouble()
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble()*10000)
|
||||
def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd")
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
map.m_date = new Date()
|
||||
DEAL_PRICE_COLL.save(map)
|
||||
}
|
||||
|
||||
+4
-4
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
@@ -78,8 +78,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
|
||||
+42
-42
@@ -67,50 +67,50 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
println "curDate:"+curDate
|
||||
// for(def city : citys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
// }
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
saveToPgSql(rows)
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
// saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
|
||||
+1
@@ -430,6 +430,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉
|
||||
// q.put("name","锦南浦三小区")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
+2
-2
@@ -375,12 +375,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202302
|
||||
def month = 202303
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202303)
|
||||
def rets = findPeGteThirdParty2(city, 202304)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
|
||||
+36
-11
@@ -33,8 +33,8 @@ class FjwPricePkCcbPrice {
|
||||
def task = new FjwPricePkCcbPrice()
|
||||
// task.run()
|
||||
// task.updatePrice()
|
||||
task.addFjwPrice()
|
||||
// task.addCcbPrice()
|
||||
// task.addFjwPrice()
|
||||
// task.addCcbPrice()//oracle
|
||||
|
||||
// task.addCcbPriceFromGuided()
|
||||
// task.addMarketPrice()
|
||||
@@ -42,21 +42,36 @@ class FjwPricePkCcbPrice {
|
||||
// task.exportFjwPriceAndCcbPrice()
|
||||
// task.addNewDistPrice()
|
||||
|
||||
// task.addCcbPrice2()
|
||||
// task.addCcbPrice3()
|
||||
task.addCcbPrice2()//按id查
|
||||
// task.addCcbPrice3()//按city,region,name查
|
||||
// task.standRegion()
|
||||
// task.addGuidPrice()
|
||||
}
|
||||
|
||||
|
||||
def addCcbPrice2(){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d ->
|
||||
if(d['R']){
|
||||
def map = findAvgPrice(d['R'], cityCodeMap.get(d['S']))
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d ->
|
||||
if(d['C']){
|
||||
def map = findAvgPrice(d['C'], cityCodeMap.get(d['E']))
|
||||
if(map){
|
||||
d['AG'] = map.avg_price
|
||||
d['AH'] = map.calc_type
|
||||
d['AI'] = map.credit_value
|
||||
d['AJ'] = map.month
|
||||
d['K'] = map.avg_price
|
||||
d['L'] = map.calc_type
|
||||
// d['AI'] = map.credit_value
|
||||
d['M'] = map.month
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addGuidPrice(){
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_230421.xlsx", { d ->
|
||||
if(d['M']){
|
||||
def map = findGuidePrice(d['C'], d['E'])
|
||||
if(map){
|
||||
d['M'] = map.avg_price
|
||||
d['N'] = map.date.toString()
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -474,4 +489,14 @@ class FjwPricePkCcbPrice {
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
def findGuidePrice(def commid, def city){
|
||||
def sql = "SELECT gujia_bds_drc_prc,gujia_drc_dt from base.t0831_guided_price_gj_h where urbn_nm='${city}' and gujia_bds_id='${commid}' ORDER BY gujia_drc_dt desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_drc_prc
|
||||
map.date = r.gujia_drc_dt
|
||||
})
|
||||
return map
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-21")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-26")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
saveToPgSql(rows)
|
||||
|
||||
//更新静态数据表
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
}
|
||||
|
||||
|
||||
+45
-27
@@ -71,33 +71,33 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
//
|
||||
//})
|
||||
|
||||
def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
|
||||
def allCitys = CcbCommonUtils.citys_all
|
||||
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def c : allCitys){
|
||||
def cityCode = cityCodeMap.get(c)
|
||||
def count = null;
|
||||
index++
|
||||
for(def r : countCitys){
|
||||
if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
|
||||
count = r['B']
|
||||
break
|
||||
}
|
||||
}
|
||||
println index +" "+ c+" "+count
|
||||
}
|
||||
|
||||
def findCcbPriceCredit(sqlInstance, cityid, commid){
|
||||
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
map.month = r.year_month
|
||||
map.credit = r.CONFIDENCE
|
||||
map.price = r.avg_price
|
||||
});
|
||||
return map
|
||||
}
|
||||
//def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
|
||||
//def allCitys = CcbCommonUtils.citys_all
|
||||
//def cityCodeMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def index = 0
|
||||
//for(def c : allCitys){
|
||||
// def cityCode = cityCodeMap.get(c)
|
||||
// def count = null;
|
||||
// index++
|
||||
// for(def r : countCitys){
|
||||
// if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
|
||||
// count = r['B']
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// println index +" "+ c+" "+count
|
||||
//}
|
||||
//
|
||||
//def findCcbPriceCredit(sqlInstance, cityid, commid){
|
||||
// def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
|
||||
// def map = [:]
|
||||
// sqlInstance.eachRow(sql,{r->
|
||||
// map.month = r.year_month
|
||||
// map.credit = r.CONFIDENCE
|
||||
// map.price = r.avg_price
|
||||
// });
|
||||
// return map
|
||||
//}
|
||||
|
||||
def findThirdPriceBySource(sqlInstance, cityid, commid){
|
||||
def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('±´¿Ç','Á´¼Ò') \n" +
|
||||
@@ -1314,6 +1314,24 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "C:\\Users\\gaojuan\\Desktop\\·¿¼ÛÍøÐÂÔöÐ¡ÇøÊý¾Ý_0427.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
if(count && count==1){
|
||||
def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
|
||||
d['E'] = dist.city
|
||||
}
|
||||
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['J'] = dist.status
|
||||
// }
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// d['F'] = newRegion
|
||||
},0)
|
||||
|
||||
//ExcelUtils.modify(fileName, {d->
|
||||
// if(d['A']){
|
||||
// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'"
|
||||
|
||||
+5
-5
@@ -92,13 +92,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","12月总量","23年1月总量","23年2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-12-01"
|
||||
def date = "2023-01-01"
|
||||
def endDate = "2023-02-01"
|
||||
def newDate = "2023-03-01"
|
||||
def startDate = "2023-01-01"
|
||||
def date = "2023-02-01"
|
||||
def endDate = "2023-03-01"
|
||||
def newDate = "2023-04-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+2
-2
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
|
||||
|
||||
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
|
||||
// cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
// regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
}
|
||||
|
||||
|
||||
+8
-7
@@ -34,6 +34,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
|
||||
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
@@ -76,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
// exportBigdataModelingBase(lastMonth)
|
||||
|
||||
//2月8号 临时导出苏州一年的数据
|
||||
exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
|
||||
exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
|
||||
}
|
||||
|
||||
if(curDate.toString().endsWith("28")){
|
||||
@@ -1012,7 +1013,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
|
||||
def exportPgThirdPartyPrice2(price_date, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
@@ -1184,7 +1185,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
});
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
@@ -1273,7 +1274,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
@@ -1381,7 +1382,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgGuideStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
@@ -1417,7 +1418,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgBigdataSupplierStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
@@ -1532,7 +1533,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgBigdataModelingBase(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
|
||||
+8
-2
@@ -622,7 +622,10 @@ class CommonDatautils {
|
||||
public static def findDistrictCount(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
if(city){
|
||||
q.put("city", city);
|
||||
}
|
||||
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
}
|
||||
@@ -639,7 +642,10 @@ class CommonDatautils {
|
||||
public static def findDistrict(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
if(city){
|
||||
q.put("city", city);
|
||||
}
|
||||
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
}
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class FzDataResultStat {
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
// task.regionPriceTrendRun()
|
||||
task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+7
-1
@@ -24,7 +24,13 @@ class AddAvgpriceBatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new AddAvgpriceBatch();
|
||||
task.run1();
|
||||
// task.run1();
|
||||
task.test()
|
||||
}
|
||||
|
||||
def test(){
|
||||
//发重算均价命令
|
||||
addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd'))
|
||||
}
|
||||
def run(){
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -263,8 +264,7 @@ class FiledStrand {
|
||||
if(all_area){
|
||||
def v = getValue(all_area)
|
||||
if(v>=1000 && v<=10000000){
|
||||
|
||||
return v.toString()
|
||||
return v.toString().toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
return null
|
||||
@@ -273,7 +273,7 @@ class FiledStrand {
|
||||
static def processCarSite(car_site_count){
|
||||
def v = getValue(car_site_count.toString());
|
||||
if(v){
|
||||
if(v>=1 && v<=100000){
|
||||
if(v>5 && v<=100000){
|
||||
try {
|
||||
return v.toString().toInteger();
|
||||
}catch(Exception e){
|
||||
@@ -284,6 +284,70 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processCarSiteDesc(car_site){
|
||||
if(car_site && car_site.size()<=20){
|
||||
return car_site.toString()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def processPropertyLife(car_site_count){
|
||||
def v = getValue(car_site_count.toString());
|
||||
if(v){
|
||||
if([40,50,70].contains(v)){
|
||||
try {
|
||||
return v.toString()+"Äê";
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processAlias(esVal, name, alias){
|
||||
if(esVal){
|
||||
if(esVal!=name){
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
def arr = alias.split(",")
|
||||
for(def s : arr){
|
||||
if(!alias.contains(s)){
|
||||
alias << esVal
|
||||
}
|
||||
}
|
||||
|
||||
if(alias){
|
||||
return alias
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processAddress(def esAddress, def city){
|
||||
if(esAddress){
|
||||
def formatAddress = AddressUtil.formatAddress(city, esAddress)
|
||||
if(formatAddress && isRoad(formatAddress)){
|
||||
return formatAddress
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def isRoad(def s){
|
||||
def pattern = Pattern.compile("[·|½Ö|´óµÀ|Ïï]");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def addBuildAreaToData(def dist, def updateFile){
|
||||
def build_area = null;
|
||||
if(dist.build_area){
|
||||
@@ -324,18 +388,12 @@ class FiledStrand {
|
||||
return updateFile;
|
||||
}
|
||||
|
||||
public static def processCubagePercent(cubage_percent, estate_type2){
|
||||
public static def processCubagePercent(cubage_percent){
|
||||
if(cubage_percent){
|
||||
def v = getValue(cubage_percent)
|
||||
if(v){
|
||||
if(estate_type2 == "±ðÊû"){
|
||||
if(v>=0.3 && v<=1){
|
||||
return v.toString()
|
||||
}
|
||||
}else{
|
||||
if(v>1 && v<=10){
|
||||
return v.toString()
|
||||
}
|
||||
if(v>=0.3 && v<=10){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -418,26 +476,26 @@ class FiledStrand {
|
||||
estateCharege = estateCharege.toDouble()
|
||||
if(estateType2=="普通住宅"){
|
||||
if(city=="上海" || city=="北京" || city=="广州" || city=="杭州"){
|
||||
if(estateCharege>=0.6 && estateCharege<=3.5){
|
||||
if(estateCharege>=0.3 && estateCharege<=40){
|
||||
return estateCharege
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}else{
|
||||
if(estateCharege>=0.2 && estateCharege<=3.5){
|
||||
if(estateCharege>=0.2 && estateCharege<=20){
|
||||
return estateCharege
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}else if(estateType2=="别墅"){
|
||||
if(estateCharege>=3){
|
||||
if(estateCharege>=1){
|
||||
return estateCharege;
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}else if(estateType2=="商住"){
|
||||
if(estateCharege>=4){
|
||||
if(estateCharege>=2){
|
||||
return estateCharege;
|
||||
}else{
|
||||
return null;
|
||||
@@ -454,13 +512,13 @@ class FiledStrand {
|
||||
if(household_count){
|
||||
household_count = household_count.toInteger()
|
||||
if(estateType2=="普通住宅" || estateType2=="商住"){
|
||||
if(household_count>=50 && household_count<=100000){
|
||||
if(household_count>=15 && household_count<=100000){
|
||||
return household_count;
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}else if(estateType2=="别墅"){
|
||||
if(household_count>=20 && household_count<=100000){
|
||||
if(household_count>=10 && household_count<=100000){
|
||||
return household_count;
|
||||
}else{
|
||||
return null;
|
||||
@@ -470,6 +528,20 @@ class FiledStrand {
|
||||
return null;
|
||||
}
|
||||
|
||||
static def processLoudongCount(loudongCount){
|
||||
def v = getValue(loudongCount.toString());
|
||||
if(v){
|
||||
if(v>=1 && v<=10000){
|
||||
try {
|
||||
return v.toString().toInteger();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processFinishBuildingDate(def finishBuildingDate){
|
||||
if(finishBuildingDate){
|
||||
Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}");
|
||||
@@ -493,6 +565,39 @@ class FiledStrand {
|
||||
}
|
||||
|
||||
|
||||
static def processCompany(val){
|
||||
if(val){
|
||||
if(val.length()>20){
|
||||
return null
|
||||
}else{
|
||||
if(val.contains("ÎÞ")){
|
||||
return null
|
||||
}
|
||||
def fiterChars = [",",".","?","/","Êý¾Ý",'-','?',' ',' ']
|
||||
for(def s : fiterChars){
|
||||
val = val.replace(s, '')
|
||||
}
|
||||
|
||||
if(val.length()<1){
|
||||
return null
|
||||
}
|
||||
|
||||
if(val.toString().isNumber()){
|
||||
return null
|
||||
}
|
||||
|
||||
Pattern pattern = Pattern.compile("\\d{4}Äê");
|
||||
if (pattern.matcher(val).matches()) {
|
||||
return null
|
||||
}
|
||||
return val
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
def converList(def dbCursor){
|
||||
def list = []
|
||||
for(def dbo : dbCursor){
|
||||
|
||||
@@ -143,17 +143,8 @@ class OpenApiUtils {
|
||||
def result = queryMaxSimilarAddress(city, name)
|
||||
if(result){
|
||||
for(def ret : result){
|
||||
if(ret && ret.lat){
|
||||
try{
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(ret.lng.toDouble(), ret.lat.toDouble())
|
||||
ret.lng2 = gmap.get("lng")
|
||||
ret.lat2 = gmap.get("lat")
|
||||
ret.estateType = ret.type
|
||||
ret.source = "¸ßµÂ"
|
||||
}catch (Exception e){
|
||||
|
||||
}
|
||||
}
|
||||
ret.estateType = ret.type
|
||||
ret.source = "¸ßµÂ"
|
||||
}
|
||||
}
|
||||
return result
|
||||
@@ -161,11 +152,17 @@ class OpenApiUtils {
|
||||
|
||||
|
||||
public static def queryMaxSimilarAddress(String city, String address, boolean isFullMatch = false){
|
||||
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
|
||||
// if(StringUtils.isEmpty(city)){
|
||||
// return null
|
||||
// }
|
||||
if(StringUtils.isEmpty(address)){
|
||||
return null;
|
||||
}
|
||||
address = address.trim().replace(" ","")
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
|
||||
// String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
|
||||
//key:8796b9dfb3889b57c38e2fff1b7e51bd »¨Ç®ÂòµÄ
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&city=" + city + "&output=json&offset=20&page=1&key=8796b9dfb3889b57c38e2fff1b7e51bd&extensions=all&citylimit=true"//citylimitÏÞÖÆ³ÇÊÐ
|
||||
|
||||
|
||||
List<com.alibaba.fastjson.JSONObject> poiList = null;
|
||||
com.alibaba.fastjson.JSONObject result = null;
|
||||
@@ -553,6 +550,35 @@ class OpenApiUtils {
|
||||
return null;
|
||||
}
|
||||
|
||||
static def similarRank(proCode, cityCode, regionCode, name, area, floor, height){
|
||||
def url = "http://localhost:8080/fangjia-open/house/similar?aboveFloorAmount=${height}&buildingName=${name}&city=${cityCode}&district=${regionCode}&" +
|
||||
"propertyTypeCode=01&province=${proCode}&floorNumber=${floor}&strctureArea=${area}"
|
||||
println url
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def dto = []
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def financialRating(city, address){
|
||||
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==0 && doc.result){
|
||||
def dtos = doc.result
|
||||
return dtos
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-01-01"
|
||||
static def statDate = "2023-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年2月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
@@ -3,6 +3,7 @@ package com.fangjia.script.gj
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.client.MongoCollection
|
||||
@@ -12,29 +13,106 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
|
||||
//def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
//def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
|
||||
//Calendar calendar = Calendar.getInstance()
|
||||
//calendar.setTime(startDate)
|
||||
//def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
|
||||
//def rMap = [:]
|
||||
//def index = 0
|
||||
//OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
|
||||
// index++
|
||||
// def day = doc.c_date.format("yyyy-MM-dd")
|
||||
// def count = rMap.get(day)
|
||||
// if (!count) {
|
||||
// rMap.put(day, 1)
|
||||
// } else {
|
||||
// rMap.put(day, count + 1)
|
||||
// }
|
||||
// if(index%10000==0){
|
||||
// println index+" "+new Date().toString()
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//rMap.each {k,val->
|
||||
// println k+" "+val
|
||||
//}
|
||||
|
||||
//def startDate = DateUtil.parseDate("20230416", "yyyyMMdd")
|
||||
//def endDate = DateUtil.parseDate("20230418", "yyyyMMdd")
|
||||
//Calendar calendar = Calendar.getInstance()
|
||||
//calendar.setTime(startDate)
|
||||
//def count = 0
|
||||
//def hashSet = []
|
||||
//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
//while (startDate.before(endDate)) {
|
||||
// calendar.add(Calendar.DATE, 1)
|
||||
// def nextDate = calendar.getTime()
|
||||
// count++
|
||||
// def total = 0
|
||||
// OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
// total++
|
||||
// def result = JSONObject.fromObject(doc.res_map.result)
|
||||
//// if (!hashSet.contains(result)) {
|
||||
// def item = [doc._id.toString(), "neiping2017@czbank", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
// data << item
|
||||
// hashSet << result
|
||||
// println total+" "+ nextDate
|
||||
//// } else {
|
||||
//// println("已存在")
|
||||
//// }
|
||||
// }
|
||||
// if (count % 7 == 0) {
|
||||
// hashSet.clear()
|
||||
// }
|
||||
//
|
||||
// startDate = nextDate
|
||||
//
|
||||
//}
|
||||
//
|
||||
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
//ExcelUtils.write("D:\\浙商调用详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
|
||||
|
||||
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
|
||||
def rMap = [:]
|
||||
def index = 0
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
|
||||
index++
|
||||
def day = doc.c_date.format("yyyy-MM-dd")
|
||||
def count = rMap.get(day)
|
||||
if (!count) {
|
||||
rMap.put(day, 1)
|
||||
} else {
|
||||
rMap.put(day, count + 1)
|
||||
}
|
||||
if(index%10000==0){
|
||||
println index+" "+new Date().toString()
|
||||
}
|
||||
}
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
|
||||
rMap.each {k,val->
|
||||
println k+" "+val
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
hashSet << result
|
||||
}
|
||||
|
||||
|
||||
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd")
|
||||
//def endDate = DateUtil.parseDate("20230506", "yyyyMMdd")
|
||||
//Calendar calendar = Calendar.getInstance()
|
||||
//calendar.setTime(startDate)
|
||||
//def count = 0
|
||||
//def hashSet = []
|
||||
//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
//
|
||||
//OPEN_URI_LOG.find(and(eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
|
||||
// def result = JSONObject.fromObject(doc.res_map.result)
|
||||
// def item = [doc._id.toString(), doc.user_id, doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
// data << item
|
||||
// hashSet << result
|
||||
// count++
|
||||
// println count
|
||||
//}
|
||||
//
|
||||
//
|
||||
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
|
||||
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
+25
-21
@@ -33,11 +33,14 @@ class AnjukeDistrictMatch {
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
// def name = "陆家嘴滨江中心N2座"
|
||||
// name = name.replaceAll("-?\\w+(号写字楼|号楼|楼|座|栋)","")
|
||||
// println name
|
||||
def task = new AnjukeDistrictMatch();
|
||||
// task.run();
|
||||
// task.runCsv();
|
||||
// task.updateCredit()
|
||||
// task.matchCommunity()//查询es小区数据
|
||||
task.matchCommunity()//查询es小区数据
|
||||
// task.run1()
|
||||
// task.updateGreenRateField()//根据第三方数据填补库中字段
|
||||
// task.updteGreenRateFieldByCsv()
|
||||
@@ -50,7 +53,7 @@ class AnjukeDistrictMatch {
|
||||
// task.updteBuildAreaFieldByCsv()
|
||||
// task.updadteAllAreaFieldByCsv()
|
||||
// task.updatePlotRateFieldByCsv()
|
||||
task.updateFinishDateFieldByCsv()
|
||||
// task.updateFinishDateFieldByCsv()
|
||||
}
|
||||
|
||||
def updateCredit(){
|
||||
@@ -728,16 +731,17 @@ class AnjukeDistrictMatch {
|
||||
|
||||
def run(){
|
||||
|
||||
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
|
||||
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\全国绿化率.xlsx"
|
||||
// def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
|
||||
def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)0406.xlsx"
|
||||
def index = 0
|
||||
|
||||
ExcelUtils.multiThreadingModify(filePath,{row->
|
||||
index++
|
||||
if(row['A'] && row['B']){
|
||||
def city = row['A']
|
||||
def name = row['C']
|
||||
def region = row['B']
|
||||
def name = row['B']
|
||||
def region = null
|
||||
// def region = row['B']
|
||||
|
||||
if(name && name.contains("·")){
|
||||
name = name.replace("·", "")
|
||||
@@ -785,27 +789,27 @@ class AnjukeDistrictMatch {
|
||||
row['AC'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
println index +" "+row['Y']
|
||||
// if(row['G']){
|
||||
// def dbo = CommonDatautils.findDistrictById(row['G'])
|
||||
// if(dbo){
|
||||
// row['H'] = dbo.estate_type2
|
||||
// row['I'] = dbo.name_cat
|
||||
//
|
||||
// def count = findBigdataCommunity(row['G'])
|
||||
// if(count){
|
||||
// row['E'] = "是"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// println index +" "+row['Y']
|
||||
if(row['G']){
|
||||
def dbo = CommonDatautils.findDistrictById(row['G'])
|
||||
if(dbo){
|
||||
row['H'] = dbo.estate_type2
|
||||
row['I'] = dbo.name_cat
|
||||
|
||||
def count = findBigdataCommunity(row['G'])
|
||||
if(count){
|
||||
row['E'] = "是"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchCommunity(){
|
||||
def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)3.24_2.xlsx"
|
||||
def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)0406-1.xlsx"
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if (row['A'] && row['B'] && row['I']) {
|
||||
if (row['A'] && row['B'] && !row['O']) {
|
||||
def rets = findAllEsData(row['A'], row['B'])
|
||||
if(rets){
|
||||
def dbo = rets[0]
|
||||
|
||||
+1
-1
@@ -480,7 +480,7 @@ class OfficeProcess {
|
||||
}
|
||||
|
||||
def checkRegion(){
|
||||
def citys = ["北京","广州","重庆","天津","成都","南京"]//"上海","深圳",
|
||||
def citys = ["杭州","苏州"]//"上海","深圳","北京","广州","重庆","成都",
|
||||
for(def city : citys){
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
def addDatas = findAddDatasByClickhouse(city)
|
||||
|
||||
+154
-1
@@ -16,7 +16,140 @@ class FieldStat {
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
// task.run3()
|
||||
// task.offircRun()
|
||||
task.distRun()
|
||||
task.shopRun()
|
||||
}
|
||||
|
||||
def distRun(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def cityMap = [:]
|
||||
def fileds = ["region","name","alias","address","addr_alias","estate_type2","build_type","block","loop_line_info","street_community","household_count","curr_household_count","virescence_percent","cubage_percent","all_area","build_area","estate_charge","car_site","car_site_count","developer","estate_company","finish_building_date","live_date","start_building_date","open_date","property_life","floor_info","fitment","fitment_materials","build_structure","traffic","facilities","characteristic","sell_address","relate_info","description","total_price","mobile","building_status","licence","build_designer","garden_designer","business_quarter","bay_area","payment_pattern","land_plan_license","prj_plan_license","land_use_license","cons_license","elevators_count","build_date","divisible","status","metro_info","metro_names","cons_company","sells_company","contact_people","parking_fee","delivery_level","fitment_level","land_car_site","underground_car_site","avg_price","sell_price","standard_area","pics","loudong_count","committee","house_structure","usage_card","mainTypeArea"]
|
||||
for(def c : citys){
|
||||
def dbo = findCoverPrecent(c)
|
||||
if(dbo && dbo.containsField("district_detail")){
|
||||
def officeDetail = dbo.district_detail
|
||||
cityMap.put(c, officeDetail)
|
||||
}else{
|
||||
cityMap.put(c, [:])
|
||||
}
|
||||
}
|
||||
|
||||
def rets = []
|
||||
rets << ["城市","数量","区域","名称","别名","地址","地址别名","物业类型","建筑类型","板块","环线位置","街道社区","户数","当前户数","绿化率","容积率","占地面积","建筑面积","物业费","停车位","停车位数","开发商","物业公司","竣工时间","入住时间","开工时间","开盘时间","产权年限","楼层情况","装修情况","装修材料","建筑结构","交通情况","周边配","小区特征","售楼地址","相关信息","描述","总价","电话","建筑状态","预售许可证","建筑单位","园艺设计单位","商圈","户型面积","支付方式","用地规划许可证","工程规划许可证","土地使用证","施工许可证","电梯数量","建造年代","是否可分割","状态","地铁沿线","地铁站名","施工单位","销售企业","联系人","停车费","交付标准","装修标准","地上车位数","地下车位数","均价","出售价格","标准层面积","图片","楼栋总数","居委会","房屋结构","土地使用权证号","主力户型"]
|
||||
def ret = []
|
||||
ret << "city"
|
||||
ret << "count"
|
||||
for(def f : fileds){
|
||||
ret << f
|
||||
}
|
||||
rets << ret
|
||||
cityMap.each {k,v->
|
||||
def row = []
|
||||
row << k
|
||||
row << CommonDatautils.findAllDistrictCount("district", k)
|
||||
for(def filed : fileds){
|
||||
if(v){
|
||||
def val = v.get(underlineToCamel(filed))
|
||||
if(val){
|
||||
row << val.toDouble()*100
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
}
|
||||
rets << row
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "小区")
|
||||
}
|
||||
|
||||
def shopRun(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def cityMap = [:]
|
||||
def fileds = ["region","name","alias","address","business_quarter","property_status","rent_type","lease_term","payment_pattern","sell_price","household_count","mobile","floor","floor_info","bay_area","finish_building_date","live_date","divisible","unified_manage","car_site_count","car_site","all_area","build_area","estate_charge","developer","estate_company","metro_info","metro_names","facilities","property_info","traffic","surrounding","business_status","url","description","open_date","sell_address","block","height","loop_line_info","cubage_percent","virescence_percent","characteristic","sells_company","licence","build_designer","cons_company","property_life","insidepos","delivery_level","start_building_date","land_plan_license","prj_plan_license","land_use_license","cons_license","fitment","build_structure","enter_company","build_type","elevators_count","estate_type2","status","standard_area","relate_info","garden_designer","fitment_level","build_date","fitment_materials","rent_price","out_pics","pics","land_car_site","underground_car_site","addr_alias"]
|
||||
for(def c : citys){
|
||||
def dbo = findCoverPrecent(c)
|
||||
if(dbo && dbo.containsField("shop_detail")){
|
||||
def officeDetail = dbo.shop_detail
|
||||
cityMap.put(c, officeDetail)
|
||||
}else{
|
||||
cityMap.put(c, [:])
|
||||
}
|
||||
}
|
||||
|
||||
def rets = []
|
||||
rets << ["城市","数量","区域","名称","别名","地址","商圈","建筑状态","租售类型","租赁年限","付款方式","出售价格","总户数","电话","建筑楼层","楼层状况","开间面积","竣工时间","开业时间","可分割","统一管理","车位数量","车位描述","占地面积","建筑面积","物业费","开发商","物业公司","地铁沿线","地铁站名","配套设施","物业概况","交通情况","周边情况","经营业态","来源url","描述","开盘时间","售楼地址","板块","楼高","环线位置","容积率","绿化率","商业特色","销售企业","销售许可证","设计单位","施工单位","产权年限","交通位置","交付标准","开工时间","用地规划许可证","工程规划许可证","土地使用证","施工许可证","装修","建筑结构","入驻企业","建筑类型","电梯数量","物业类型","状态","标准层面积","相关信息","景观设计单位","装修标准","建造年代","建材装修","出租价格","外景图","其它图","地上车位数","地下车位数","地址别名"]
|
||||
def ret = []
|
||||
ret << "city"
|
||||
ret << "count"
|
||||
for(def f : fileds){
|
||||
ret << f
|
||||
}
|
||||
rets << ret
|
||||
cityMap.each {k,v->
|
||||
def row = []
|
||||
row << k
|
||||
row << CommonDatautils.findAllDistrictCount("shop", k)
|
||||
for(def filed : fileds){
|
||||
if(v){
|
||||
def val = v.get(underlineToCamel(filed))
|
||||
if(val){
|
||||
row << val.toDouble()*100
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
}
|
||||
rets << row
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "商铺")
|
||||
}
|
||||
|
||||
def offircRun(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def cityMap = [:]
|
||||
def fileds = ["region","name","alias","latLng","latLng2","address","addrAlias","bedKind","estateType2","buildType","block","businessQuarter","loopLineInfo","licence","sideset","realUsePercent","buildDesigner","consCompany","estateCharge","carSiteCount","carSite","undergroundCarSite","landCarSite","parkingFee","sight","propertyLife","lobbyHeight","standardHeight","clearHeight","standardArea","foreign","register","lightingVentilation","conditionerTime","","elevatorsCount","personElevatorsCount","goodsElevatorCount","openDate","finishBuildingDate","liveDate","allArea","buildArea","cubagePercent","virescencePercent","developer","estateCompany","networkWay","fitmentLevel","buildStructure"]
|
||||
for(def c : citys){
|
||||
def dbo = findCoverPrecent(c)
|
||||
if(dbo && dbo.containsField("office_detail")){
|
||||
def officeDetail = dbo.office_detail
|
||||
cityMap.put(c, officeDetail)
|
||||
}else{
|
||||
cityMap.put(c, [:])
|
||||
}
|
||||
}
|
||||
|
||||
def rets = []
|
||||
rets << ["城市","数量","区域","名称","名称别名","经纬度(百度)","经纬度(高德)","地址","地址别名","写字楼等级","写字楼类型","建筑类别","板块","商圈","环线位置","预售许可证","是否可分割","得房率","设计单位","施工单位","物业费","停车位数","停车描述","地上车位数","地下车位数","车位租金","开间面积","产权年限","大堂层高","标准层高","净高","标准层面积","是否涉外","是否可注册","空调","空调开放时间","空调费用","电梯数量","客梯数","货梯数","开盘时间","竣工时间","入住时间","占地面积","建筑面积","容积率","绿化率","开发商","物业公司","网络","装修状况","结构"]
|
||||
def ret = []
|
||||
ret << "城市"
|
||||
ret << "数量"
|
||||
for(def f : fileds){
|
||||
ret << f
|
||||
}
|
||||
rets << ret
|
||||
cityMap.each {k,v->
|
||||
def row = []
|
||||
row << k
|
||||
row << CommonDatautils.findAllDistrictCount("office", k)
|
||||
for(def filed : fileds){
|
||||
if(v){
|
||||
def val = v.get(filed)
|
||||
if(val){
|
||||
row << val.toDouble()*100
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
}
|
||||
rets << row
|
||||
}
|
||||
|
||||
ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "写字楼")
|
||||
|
||||
}
|
||||
|
||||
def run3(){
|
||||
@@ -136,4 +269,24 @@ class FieldStat {
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
public static String underlineToCamel(String param){
|
||||
if (param==null||"".equals(param.trim())){
|
||||
return "";
|
||||
}
|
||||
int len=param.length();
|
||||
StringBuilder sb=new StringBuilder(len);
|
||||
for (int i = 0; i < len; i++) {
|
||||
char c=param.charAt(i);
|
||||
if (c=="_"){
|
||||
if (++i<len){
|
||||
sb.append(Character.toUpperCase(param.charAt(i)));
|
||||
}
|
||||
}else{
|
||||
sb.append(c);
|
||||
}
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
|
||||
+40
-4
@@ -14,19 +14,55 @@ import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class priceRank {
|
||||
class ApiTest {
|
||||
|
||||
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new priceRank()
|
||||
task.run()
|
||||
def task = new ApiTest()
|
||||
// task.rankRunTest()
|
||||
// task.similarRunTest()
|
||||
task.loupanLevelTest()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def loupanLevelTest(){
|
||||
ExcelUtils.modify("D:\\数据\\卓信\\上海 - 副本.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.financialRating(d['A'], d['B']+d['C'])
|
||||
if(results){
|
||||
d['D'] = results.consumptionRating
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def similarRunTest(){
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\江南农商行\\住宅信息V2-差异案例(反馈).xlsx", { d ->
|
||||
if(d['A']){
|
||||
def cityCode = d['U'].toString().toDouble().toInteger()
|
||||
def proCode = d['V'].toString().toDouble().toInteger()
|
||||
def regionCode = d['T'].toString().toDouble().toInteger()
|
||||
def name = d['D']
|
||||
def area = d['F']
|
||||
def floor = d['H'].toString().toDouble().toInteger()
|
||||
def height = d['G'].toString().toDouble().toInteger()
|
||||
def results = com.fangjia.script.gj.OpenApiUtils.similarRank(proCode, cityCode,regionCode, name, area, floor, height)
|
||||
if(results){
|
||||
def str = ""
|
||||
for(def r : results){
|
||||
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"\n"
|
||||
}
|
||||
d['Y'] = str
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def rankRunTest(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("ÉϺ£","district", ["city","region","name"])
|
||||
//def dists = CommonDatautils.findDistrictByCityAndRegions("ÉϺ£","district", ["ÆÖ¶«"], ["city","region","name"])
|
||||
def rets = createCityDatas(dists)
|
||||
+417
@@ -0,0 +1,417 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 小区字段入库(自动入库)
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018)
|
||||
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
|
||||
|
||||
//,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"
|
||||
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
|
||||
//"alias":"alias",
|
||||
def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictFieldAddBatch(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京"
|
||||
|
||||
|
||||
def analysRets = []
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
||||
// addFieldFromRonghe(c, totalCount)
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)
|
||||
}
|
||||
println analysRets.toString()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
|
||||
}
|
||||
|
||||
def addFieldFromMatch(c, totalCount, analysRets){
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findEsDatas(c, esWebSource)
|
||||
def matchRets1 = matchApi(esRets)
|
||||
def matchRets2 = selectRestByCredit(matchRets1)
|
||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2)
|
||||
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
if(esRets){
|
||||
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
|
||||
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
|
||||
}
|
||||
analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2]
|
||||
}
|
||||
return analysRets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def addFieldFromRonghe(c, totalCount){
|
||||
sourceMap.each{seaweedmapSource,esWebSource->
|
||||
def rhRets = findRongheDists(c, seaweedmapSource)
|
||||
def esRets = findEsDistrict(c, esWebSource)
|
||||
def updateRets = processFieldsFromRh(c, rhRets, esRets)
|
||||
|
||||
println "【通过融合关系填充字段】网站:"+seaweedmapSource +" 第三方融合表:"+rhRets.size() +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, rhRets.size()*100/totalCount)+"%"
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "----------------------------------------------------------------------------------"
|
||||
}
|
||||
}
|
||||
|
||||
def processFieldsFromRh(c, rhRets, esRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
def fieldValueMap = [:]
|
||||
for(def rhDist : rhRets){
|
||||
def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields)
|
||||
if(!dist){
|
||||
println c+" "+rhDist.commid+" 不存在"
|
||||
continue
|
||||
}
|
||||
def esDbo = esRets.get(c + "#" + rhDist.id)
|
||||
if(!esDbo){
|
||||
println c+" "+rhDist.id+" 不存在[ES]"
|
||||
continue
|
||||
}
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = dist.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = esDbo.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield == "build_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(newValues){
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
}
|
||||
|
||||
println fieldValueMap.size()
|
||||
return fieldValueMap;
|
||||
|
||||
}
|
||||
|
||||
def processFieldsFromMatch(c,matchRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
fields << "city"
|
||||
def fieldValueMap = [:]
|
||||
for(def matchDist : matchRets){
|
||||
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
println c+" "+matchDist.match_id+" 不存在"
|
||||
continue
|
||||
}
|
||||
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = dist.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = matchDist.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield == "build_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
if(seaweedfield =="virescence_percent"){
|
||||
newVal = FiledStrand.processVirescencePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="cubage_percent"){
|
||||
newVal = FiledStrand.processCubagePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
|
||||
newVal = FiledStrand.processArea(esVal)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield =="household_count"){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
if(seaweedfield =="loudong_count"){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="car_site_count"){
|
||||
newVal = FiledStrand.processCarSite(esVal)
|
||||
}
|
||||
if(seaweedfield =="car_site"){
|
||||
newVal = FiledStrand.processCarSiteDesc(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="property_life"){
|
||||
newVal = FiledStrand.processPropertyLife(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="alias"){
|
||||
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
|
||||
}
|
||||
|
||||
if(seaweedfield =="address"){
|
||||
newVal = FiledStrand.processAddress(esVal, dist.city)
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(newValues){
|
||||
println dist._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
}
|
||||
|
||||
return fieldValueMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findRongheDists(def city, def source){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("commid", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("id", 1)
|
||||
keys.put("source", 1)
|
||||
def dbcursor = SEAWEEDMAP_COLL.find(new BasicDBObject().append("city", city).append("source", source), keys)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << ["commid":d.commid, "city":d.city, "source":d.source, "id":d.id]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findEsDistrict(c, esWebSource){
|
||||
def map = [:]
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
|
||||
if(r.id && !map.containsKey(c+"#"+r.id)){
|
||||
map.put(c+"#"+r.id, r)
|
||||
}
|
||||
})
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def findEsDatas(c, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
|
||||
list << r
|
||||
})
|
||||
return list
|
||||
}
|
||||
|
||||
def fieldUpdate(total,updateRets){
|
||||
|
||||
def countMap = [:]
|
||||
|
||||
for(def rm : updateRets){
|
||||
def valMap = rm.value
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
if(valMap.containsKey(seaweedfield)) {
|
||||
if(countMap.containsKey(seaweedfield)) {
|
||||
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
|
||||
}else{
|
||||
countMap.put(seaweedfield, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println countMap.toString()
|
||||
def precentMap = [:]
|
||||
fieldsMap.each() {key,skey->
|
||||
if(countMap.containsKey(key)) {
|
||||
precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
|
||||
}else {
|
||||
precentMap.put(skey, 0)
|
||||
}
|
||||
}
|
||||
return precentMap;
|
||||
}
|
||||
|
||||
|
||||
def matchApi(esRets){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(esRets.size())
|
||||
for(def i=0;i<esRets.size();i++){
|
||||
final def data = esRets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = matching(data)
|
||||
if(ret){
|
||||
// println totalCount.get() +" "+data.city+" "+data.name+" [Match] "+data.match_region+" "+data.match_name +" "+data.flag+" "+data.credit
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def selectRestByCredit(matchRets){
|
||||
def matchRets1 = []
|
||||
for(def r : matchRets){
|
||||
if(r.credit && r.credit>=0.9){
|
||||
matchRets1 << r
|
||||
}
|
||||
}
|
||||
return matchRets1
|
||||
}
|
||||
|
||||
def matching(data){
|
||||
def name = data.name
|
||||
if(data.name && data.name.contains("·")){
|
||||
name = data.name.replace("·", "")
|
||||
}
|
||||
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(data.city, data.region, name)
|
||||
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district")
|
||||
flag = "映射别名匹配"
|
||||
}
|
||||
if(!dist) {
|
||||
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district")
|
||||
if(rets && rets.size()==1){
|
||||
flag = "别名匹配"
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
|
||||
if(dist){
|
||||
data.flag = flag
|
||||
data.match_id = dist._id.toString()
|
||||
data.match_region = dist.region
|
||||
data.match_name = dist.name
|
||||
data.credit = 0.95
|
||||
}else{
|
||||
def jsonObject = OpenApiClient.addressMatch(data.city, name, "property", false,data.region, null)
|
||||
if(jsonObject!=null && jsonObject.precise==-1){
|
||||
if(data.address && isStandardRoad(data.address)){
|
||||
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false,data.region, name)
|
||||
}else{
|
||||
data.flag = "匹配接口-地址"
|
||||
}
|
||||
}else{
|
||||
data.flag = "匹配接口-名称"
|
||||
}
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
data.match_id = jsonObject.get("id")
|
||||
data.match_region = jsonObject.get("district")
|
||||
data.match_name = jsonObject.get("name")
|
||||
data.credit = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
if(data.match_id){
|
||||
return data
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def updateDB(c, updateRets){
|
||||
|
||||
for(def rm : updateRets) {
|
||||
def id = rm.key
|
||||
def valMap = rm.value
|
||||
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
valMap.each {k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
def matchDist(city, region, name){
|
||||
def dist = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
return dist;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class SameDistName {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new SameDistName()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("常州", "district", ["city","region", "name", "address","lat", "lng"])
|
||||
def map = [:]
|
||||
for(def d : dists){
|
||||
def list = map.get(d.name)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(d.name, list)
|
||||
}
|
||||
list << d
|
||||
}
|
||||
|
||||
def datas = []
|
||||
map.each {k,v->
|
||||
if(v && v.size()>=2){
|
||||
datas.addAll(v)
|
||||
}
|
||||
}
|
||||
|
||||
def retDats = []
|
||||
retDats << ["城市","区域","名称","地址","区域状况"]
|
||||
if(datas){
|
||||
def regionPointsMap = findRegionRange("常州")
|
||||
for(def d : datas){
|
||||
def lng = d.lng/10000000000.0
|
||||
def lat = d.lat/10000000000.0
|
||||
def flag = isInArea(lng, lat, regionPointsMap.get(d.region))
|
||||
retDats << [d.city, d.region, d.name, d.address, flag]
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\小区\\相同小区不同区域.xlsx", retDats, "数据")
|
||||
}
|
||||
|
||||
def findRegionRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
def dbcursor = POLYLINECOLL.find(q)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def region = RegionData.formatRegion(city,d.region)
|
||||
regionMap.put(region, d.polyline)
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
|
||||
def isInArea(lng, lat, points){
|
||||
List<Point> pointList = InsidePolygon.createPoint2(points);
|
||||
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
|
||||
return flag;
|
||||
}
|
||||
}
|
||||
+71
-2
@@ -1,6 +1,8 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
@@ -8,16 +10,39 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
|
||||
findLogs()
|
||||
//findLogs()
|
||||
//findFangjiaLogs()
|
||||
stat()
|
||||
|
||||
def stat(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1)
|
||||
def map=[:]
|
||||
for(def d : datas){
|
||||
def count = map.get(d['C'])
|
||||
if(!count){
|
||||
map.put(d['C'], 1)
|
||||
}else{
|
||||
map.put(d['C'], count+1)
|
||||
}
|
||||
}
|
||||
|
||||
map.each {k,v->
|
||||
println k +" "+v
|
||||
}
|
||||
}
|
||||
def findLogs(){
|
||||
def total = 0
|
||||
try{
|
||||
|
||||
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
|
||||
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
|
||||
|
||||
// def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
|
||||
// def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
|
||||
// QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
|
||||
def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com")
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
|
||||
println houses.size()
|
||||
def set = new HashSet()
|
||||
@@ -39,4 +64,48 @@ def findLogs(){
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
def findFangjiaLogs(){
|
||||
|
||||
def startDate = DateUtil.parseDate("20230329", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230429", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def list = []
|
||||
def data = [["mid", "ip", "访问时间"]]
|
||||
while (startDate.before(endDate)) {
|
||||
calendar.add(Calendar.DATE, 1)
|
||||
def nextDate = calendar.getTime()
|
||||
|
||||
// def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
|
||||
// def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false);
|
||||
|
||||
|
||||
def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
|
||||
println houses.size()
|
||||
|
||||
if (houses) {
|
||||
for (def d : houses) {
|
||||
def reqStr = d.request.toString()
|
||||
def mid = null
|
||||
if (reqStr.contains("mid=")) {
|
||||
|
||||
mid = reqStr.substring(reqStr.indexOf("mid=") + 4, reqStr.indexOf("mid=") + 8)
|
||||
}
|
||||
def time = d.get("@timestamp").toString()
|
||||
time = time.substring(0, time.indexOf("T"))
|
||||
def ip = d.remote_addr
|
||||
data << [mid, ip, time]
|
||||
}
|
||||
}
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志")
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
@@ -9,14 +10,18 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
|
||||
findLogs()
|
||||
|
||||
//findLogs()
|
||||
findFangjiaLogs()
|
||||
def findLogs(){
|
||||
def total = 0
|
||||
try{
|
||||
|
||||
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
|
||||
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
|
||||
def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
def q = QueryBuilders.fuzzyQuery("request", "/jnBank")
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder);
|
||||
@@ -53,3 +58,35 @@ def findLogs(){
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findFangjiaLogs(){
|
||||
def total = 0
|
||||
try{
|
||||
|
||||
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
|
||||
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
|
||||
def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
def q = QueryBuilders.fuzzyQuery("request", "/enquire")
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder);
|
||||
println houses.size()
|
||||
def set = new HashSet()
|
||||
if(houses) {
|
||||
for (def d : houses) {
|
||||
|
||||
}
|
||||
}
|
||||
println set
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return total
|
||||
}
|
||||
|
||||
+556
File diff suppressed because one or more lines are too long
+16
-16
@@ -32,7 +32,7 @@ public class BeiKeCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
@@ -44,7 +44,7 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
@@ -112,18 +112,20 @@ public class BeiKeCrawlByChrome {
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
for(def m : citys){
|
||||
driver.get(m.url+"/chengjiao/")
|
||||
def city = m[0]
|
||||
def url = m[1]
|
||||
driver.get(url+"/chengjiao/")
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = JQueryUtils.select(".position dl:contains(λÖÃ)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
def regionText = r.text()
|
||||
def href = m.url+ r.attr("href")
|
||||
def href = url+ r.attr("href")
|
||||
|
||||
println "开始抓取【${m.city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, m.city, href, regionText)
|
||||
println m.url +" 详情链接:"+detailUrls.size()
|
||||
println "开始抓取【${city}】【${regionText}】【${href}】"
|
||||
def detailUrls = getPageDeail(driver, city, href, regionText)
|
||||
println url +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,7 +185,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e, region)
|
||||
@@ -398,16 +400,13 @@ public class BeiKeCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
|
||||
def list = []
|
||||
def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
// cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
def href = "https:"+e.attr("href")
|
||||
if (citys.contains(e.text())){
|
||||
list << [e.text(), href]
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
@@ -432,6 +431,7 @@ public class BeiKeCrawlByChrome {
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // µã»÷ÐÐΪ£»
|
||||
Thread.sleep(10000)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+14
-9
@@ -31,7 +31,7 @@ public class LianjiaCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -117,10 +117,11 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
// cityMap = cityMap.reverse()
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
@@ -184,12 +185,15 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def crawlData(driver, cityList){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
for(def ret : cityList){
|
||||
def city = ret[0]
|
||||
def cityUrl = ret[1]
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
driver.get(cityUrl+"chengjiao/")
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def regions = JQueryUtils.select(".position dl:contains(λÖÃ)+dl a", doc)
|
||||
if(regions){
|
||||
@@ -251,7 +255,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e, regionText)
|
||||
@@ -473,17 +477,17 @@ public class LianjiaCrawlByChrome {
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["ËÕÖÝ"]
|
||||
def cityList = []
|
||||
def citys= ["À¥É½","¾Å½","Ö£ÖÝ","ÀÈ·»","ÕÄÖÝ","ÏÌÑô","³¤´º","ÖØÇì"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityMap.put(e.text(), href)
|
||||
cityList << [e.text(), href]
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
return cityList
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
@@ -505,6 +509,7 @@ public class LianjiaCrawlByChrome {
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // µã»÷ÐÐΪ£»
|
||||
Thread.sleep(20000)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+10
-9
@@ -21,10 +21,10 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\招商\\招商.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addThirdPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
|
||||
}
|
||||
|
||||
@@ -104,12 +104,13 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['M'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
d['AO'] = gxdPrice
|
||||
}
|
||||
if (d['B']) {
|
||||
// def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
// if(dist){
|
||||
// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['G'] = gxdPrice
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+7
-2
@@ -23,11 +23,16 @@ class ChongQingBlankTest {
|
||||
static main(args) {
|
||||
def task = new ChongQingBlankTest()
|
||||
task.run()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row->
|
||||
allRegionMap.each{k,v->
|
||||
println k+" "+v.size()
|
||||
}
|
||||
|
||||
/* ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row->
|
||||
if(row['B'].toString()=="重庆市九龙坡区建工三村7号"){
|
||||
def city = CommonUtils.parseCityInAddress(row["B"])
|
||||
if(!city){
|
||||
@@ -197,7 +202,7 @@ class ChongQingBlankTest {
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}, 0)*/
|
||||
}
|
||||
|
||||
|
||||
|
||||
+1
-2
@@ -27,8 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
|
||||
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
|
||||
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
|
||||
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDistrictPrice(), args);
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202301);
|
||||
def count2 = findTotalCount(source, 202302);
|
||||
def count1 = findTotalCount(source, 202302);
|
||||
def count2 = findTotalCount(source, 202303);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+2
@@ -563,6 +563,8 @@ public interface DistrictService extends ModeService<DistrictKey, DistrictBo> {
|
||||
*/
|
||||
Integer getBuildYear(District district);
|
||||
|
||||
Integer getCreateYear(District district);
|
||||
|
||||
/**
|
||||
* in查询小区信息
|
||||
*
|
||||
|
||||
+9
@@ -2667,6 +2667,15 @@ public class DistrictServiceImpl implements DistrictService {
|
||||
}
|
||||
}
|
||||
|
||||
public Integer getCreateYear(District district) {
|
||||
Integer buildYear = getYearFormDate(district.getCreateYear());
|
||||
int minYear = Integer.MAX_VALUE;
|
||||
if (buildYear != null && buildYear < minYear) {
|
||||
minYear = buildYear;
|
||||
}
|
||||
return minYear;
|
||||
}
|
||||
|
||||
public Integer getBuildYear(District district) {
|
||||
|
||||
Integer finishYear = getYearFormDate(district.getFinishBuildingDate());
|
||||
|
||||
Reference in New Issue
Block a user