This commit is contained in:
2023-05-11 10:09:19 +08:00
43 changed files with 2001 additions and 309 deletions
@@ -2013,6 +2013,23 @@ public class DistrictController {
if (newName != null || newRegion != null) {
districtUpdateTaskService.createTask(orgDis, 2, newName, newRegion);
districtUpdateTaskService.createTask(orgDis, 6, newName, newRegion);
District distTask = new District();
distTask.setCity(orgDis.getCity());
if(StringUtils.isNotBlank(newName)){
distTask.setRegion(newRegion);
}else{
distTask.setRegion(orgDis.getRegion());
}
distTask.setEstateType2(orgDis.getEstateType2());
if(StringUtils.isNotBlank(newName)){
distTask.setName(newName);
}else{
distTask.setName(orgDis.getName());
}
districtUpdateTaskService.createTask(distTask, 5);
}
if (StringUtils.isNotBlank(district.getEstateType2())
@@ -365,7 +365,7 @@
</td>
</tr>
<tr>
<td class="bg"> 小区性质</td>
<td class="bg">权属类别</td>
<td>
<select id="landProperty" name="district.landProperty">
<option value=""></option>
@@ -590,7 +590,7 @@
<#else>
<i class="unlock" name="land_property"></i>
</#if>
小区性质
权属类别
</td>
<td>
<select id="landProperty" name="district.landProperty">
@@ -94,6 +94,11 @@ public class DistrictUpdateTaskDaoImpl extends MongoDaoImpl<DistrictUpdateTask>
task.setCreateDate(new Date());
task.setStatus(5);
task.setName(district.getName());
if(district.getEstateType2().contains("ąđĘű")){
task.setEstateType("villa");
}else{
task.setEstateType("district");
}
this.save(task);
}
@@ -746,78 +746,75 @@ public class PropertyController {
}
}
if (model.getDetailLevel() != null && model.getDetailLevel() >= 1) {
districtInfoDto.setBlock(district.getBlock());
districtInfoDto.setBlock(district.getBlock());
districtInfoDto.setLoopLine(district.getLoopLineInfo());
if (HouseValidator.validateBuildingType(district.getBuildType())) {
districtInfoDto.setBuildingType(district.getBuildType());
} else {
districtInfoDto.setBuildingType("");
}
districtInfoDto.setAlias(district.getMappingAlias());
districtInfoDto.setAddressAlias(district.getAddrAlias());
districtInfoDto.setCommunity(district.getStreetCommunity());
districtInfoDto.setCommercialArea(district.getBusinessQuarter());
if (district.getHouseholdCount() != null) {
districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString()));
} else {
districtInfoDto.setTotalUnitCount("");
}
districtInfoDto.setManagementFee(district.getEstateCharge());
if (district.getCarSiteCount() != null) {
districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString()));
} else {
districtInfoDto.setParking("");
}
districtInfoDto.setParkingDescription(district.getCarSite());
districtInfoDto.setOwnershipPeriod(district.getPropertyLife());
districtInfoDto.setOwnershipType(district.getHouseProperty());
districtInfoDto.setStartBuildTime(district.getStartBuildingDate());
if (district.getOpenDate() != null) {
Date minDate = DateUtil.parseDate("1900-01-01", "yyyy-MM-dd");
Date maxDate = DateUtil.parseDate("2025-01-01", "yyyy-MM-dd");
if (district.getOpenDate().after(minDate) && district.getOpenDate().before(maxDate)) {
districtInfoDto
.setStartSellingTime(DateUtil.formatByPattern(district.getOpenDate(), "yyyy-MM-dd"));
}
} else {
districtInfoDto.setStartSellingTime("");
}
if (model.getDetailLevel() != null && model.getDetailLevel() >= 2) {
districtInfoDto.setLoopLine(district.getLoopLineInfo());
if (HouseValidator.validateBuildingType(district.getBuildType())) {
districtInfoDto.setBuildingType(district.getBuildType());
} else {
districtInfoDto.setBuildingType("");
}
districtInfoDto.setAlias(district.getMappingAlias());
districtInfoDto.setAddressAlias(district.getAddrAlias());
districtInfoDto.setCommunity(district.getStreetCommunity());
districtInfoDto.setCommercialArea(district.getBusinessQuarter());
if (district.getHouseholdCount() != null) {
districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString()));
} else {
districtInfoDto.setTotalUnitCount("");
}
districtInfoDto.setManagementFee(district.getEstateCharge());
if (district.getCarSiteCount() != null) {
districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString()));
} else {
districtInfoDto.setParking("");
}
districtInfoDto.setParkingDescription(district.getCarSite());
districtInfoDto.setOwnershipPeriod(district.getPropertyLife());
districtInfoDto.setOwnershipType(district.getHouseProperty());
districtInfoDto.setStartBuildTime(district.getStartBuildingDate());
if (district.getOpenDate() != null) {
Date minDate = DateUtil.parseDate("1900-01-01", "yyyy-MM-dd");
Date maxDate = DateUtil.parseDate("2025-01-01", "yyyy-MM-dd");
if (district.getOpenDate().after(minDate) && district.getOpenDate().before(maxDate)) {
districtInfoDto
.setStartSellingTime(DateUtil.formatByPattern(district.getOpenDate(), "yyyy-MM-dd"));
}
} else {
districtInfoDto.setStartSellingTime("");
}
districtInfoDto.setFinishBuildTime(district.getFinishBuildingDate());
districtInfoDto.setMoveinTime(district.getLiveDate());
districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea()));
districtInfoDto.setConstructionSpace(district.getBuildArea());
districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent()));
districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent()));
districtInfoDto.setPreSalePermit(district.getLicence());
districtInfoDto.setDeveloper(district.getDeveloper());
districtInfoDto.setDiscount(district.getDiscount());
districtInfoDto.setPropertyManagement(district.getEstateCompany());
if (HouseValidator.fetchFitmentInfo(district.getFitment()) == null) {
districtInfoDto.setDecorationStandard("");
} else {
districtInfoDto.setDecorationStandard(HouseValidator.fetchFitmentInfo(district.getFitment()));
}
districtInfoDto.setBuildingStructure(district.getBuildStructure());
districtInfoDto.setSupportingFacilities(district.getFacilities());
districtInfoDto.setTransportation(district.getTraffic());
districtInfoDto.setRelatedInfo(district.getRelateInfo());
districtInfoDto.setDescription(district.getDescription());
districtInfoDto.setSalesOfficeAdd(district.getSellAddress());
districtInfoDto.setSalesPhoneNumber(district.getMobile());
districtInfoDto.setTownship(district.getTownship());
if (district.getLoudongCount() != null && district.getLoudongCount() > 0) {
districtInfoDto.setBuildingCount(district.getLoudongCount());
}
Integer finishYear = districtService.getBuildYear(district);
if (finishYear != null) {
districtInfoDto.setFinishYear(finishYear);
}
districtInfoDto.setFinishBuildTime(district.getFinishBuildingDate());
districtInfoDto.setMoveinTime(district.getLiveDate());
districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea()));
districtInfoDto.setConstructionSpace(district.getBuildArea());
districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent()));
districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent()));
districtInfoDto.setPreSalePermit(district.getLicence());
districtInfoDto.setDeveloper(district.getDeveloper());
districtInfoDto.setDiscount(district.getDiscount());
districtInfoDto.setPropertyManagement(district.getEstateCompany());
if (HouseValidator.fetchFitmentInfo(district.getFitment()) == null) {
districtInfoDto.setDecorationStandard("");
} else {
districtInfoDto.setDecorationStandard(HouseValidator.fetchFitmentInfo(district.getFitment()));
}
districtInfoDto.setBuildingStructure(district.getBuildStructure());
districtInfoDto.setSupportingFacilities(district.getFacilities());
districtInfoDto.setTransportation(district.getTraffic());
districtInfoDto.setRelatedInfo(district.getRelateInfo());
districtInfoDto.setDescription(district.getDescription());
districtInfoDto.setSalesOfficeAdd(district.getSellAddress());
districtInfoDto.setSalesPhoneNumber(district.getMobile());
districtInfoDto.setTownship(district.getTownship());
if (district.getLoudongCount() != null && district.getLoudongCount() > 0) {
districtInfoDto.setBuildingCount(district.getLoudongCount());
}
// Integer finishYear = districtService.getBuildYear(district);
Integer finishYear = districtService.getCreateYear(district);
if (finishYear != null) {
districtInfoDto.setFinishYear(finishYear);
}
mav.setModel(BasicModel.buildOKModel().setResult(districtInfoDto));
return mav;
}
+8 -2
View File
@@ -433,11 +433,17 @@
<artifactId>javadbf</artifactId>
<version>1.9.2</version>
</dependency>
<dependency>
<!-- <dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<version>0.1.41</version>
</dependency>
</dependency>-->
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<version>0.2.6</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
@@ -30,12 +30,12 @@ public class ExportAvgPrice {
DSCollection dataStoreConnection = null;
static main(args) {
def task = new ExportAvgPrice();
def weekends = ["2019-1-1","2019-2-1","2019-3-1","2019-4-1","2019-5-1","2019-6-1","2019-7-1","2019-8-1", "2019-9-1","2019-10-1","2019-11-1","2019-12-1","2020-1-1","2020-2-1","2020-3-1","2020-4-1","2020-5-1","2020-6-1","2020-7-1","2020-8-1", "2020-9-1","2020-10-1","2020-11-1","2020-12-1","2021-1-1","2021-2-1","2021-3-1","2021-4-1","2021-5-1","2021-6-1","2021-7-1","2021-8-1", "2021-9-1","2021-10-1"];
def citys = ["温州"];//findAllCity();
// task.exportAllCity(citys, weekends, "e:\\全国城市季度均价.csv");
task.exportRegion(citys, weekends, "d:\\温州区域季度均价.csv");
def weekends = ["2023-04-01","2023-03-01","2023-02-01","2023-01-01","2022-12-01","2022-11-01","2022-10-01","2022-09-01","2022-08-01","2022-07-01","2022-06-01","2022-05-01","2022-04-01","2022-03-01","2022-02-01","2022-01-01","2021-12-01","2021-11-01","2021-10-01","2021-09-01","2021-08-01","2021-07-01","2021-06-01","2021-05-01","2021-04-01"]
def citys = ["上海"];//findAllCity();
// task.exportAllCity(citys, weekends, "D:\\数据\\小区\\全国城市季度均价.csv");
// task.exportRegion(citys, weekends, "D:\\数据\\小区\\区域季度均价.csv");
// task.exportBlock(citys, weekends, "e:\\区域板块季度均价.csv");
// task.exportDistrict(citys, weekends, "e:\\小区季度均价.csv");
task.exportDistrict(citys, weekends, "D:\\数据\\小区\\小区季度均价.csv");
// task.exportRegionPrice("江苏");
}
@@ -182,6 +182,7 @@ public class ExportAvgPrice {
q.put("visible", 0);
q.put("cat", "district");
q.put("city", city);
// q.put("name", "龙柏二村")
DSCursor cursor = getDataStoreConnection().find(q).sort(new BasicDBObject('bscore', -1));
Iterator<DBObject> iters = cursor.iterator()
@@ -0,0 +1,36 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def d : datas){
def map = new BasicDBObject()
map.city = "深圳"
map.district_name = d['D']
map.region = d['B']
def roomStr = d['E']
if(roomStr){
def arrs = roomStr.split("-")
map.room = arrs[0].toString().toInteger()
map.hall = arrs[1].toString().toInteger()
map.toilet = arrs[3].toString().toInteger()
}
map.area = d['F'].toString().toDouble()
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].toString().toDouble()*10000)
def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd")
map.source = "深房家族"
map.create_date = new Date()
map.m_date = new Date()
DEAL_PRICE_COLL.save(map)
}
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
@@ -78,8 +78,8 @@ class BigdataCommunityBatch extends BaseBatch{
println "txt 生成->导Oracle"
delPgData()
saveToPgSql(rows)
// delPgData()
// saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
@@ -67,50 +67,50 @@ class BigdataDealBatch extends BaseBatch{
}
println "curDate:"+curDate
// for(def city : citys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
}
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
saveToPgSql(rows)
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def index = 0
// for(def city : citys){
// index ++
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def ppeDists = [:]
//
// def statMap = [:]//getDistrictStatByCity(city, weekend)
// if(city=="上海"){
// def youDataTrade = getYouDaData(statMap)
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
// rows.addAll(youdaDatas)
// println city +" "+youdaDatas.size()
// }
//
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
//
// }
//
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
// if(CommonDatautils.isWindow()){
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
// }
//
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
// println "txt 生成 " + rows.size()
//
// saveToPgSql(rows)
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -430,6 +430,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉
// q.put("name","锦南浦三小区")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -375,12 +375,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202302
def month = 202303
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202303)
def rets = findPeGteThirdParty2(city, 202304)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -33,8 +33,8 @@ class FjwPricePkCcbPrice {
def task = new FjwPricePkCcbPrice()
// task.run()
// task.updatePrice()
task.addFjwPrice()
// task.addCcbPrice()
// task.addFjwPrice()
// task.addCcbPrice()//oracle
// task.addCcbPriceFromGuided()
// task.addMarketPrice()
@@ -42,21 +42,36 @@ class FjwPricePkCcbPrice {
// task.exportFjwPriceAndCcbPrice()
// task.addNewDistPrice()
// task.addCcbPrice2()
// task.addCcbPrice3()
task.addCcbPrice2()//按id查
// task.addCcbPrice3()//按city,region,name查
// task.standRegion()
// task.addGuidPrice()
}
def addCcbPrice2(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\测试地址-估家科技V1 (6).xlsx", { d ->
if(d['R']){
def map = findAvgPrice(d['R'], cityCodeMap.get(d['S']))
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_补充.xlsx", { d ->
if(d['C']){
def map = findAvgPrice(d['C'], cityCodeMap.get(d['E']))
if(map){
d['AG'] = map.avg_price
d['AH'] = map.calc_type
d['AI'] = map.credit_value
d['AJ'] = map.month
d['K'] = map.avg_price
d['L'] = map.calc_type
// d['AI'] = map.credit_value
d['M'] = map.month
}
}
})
}
def addGuidPrice(){
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\指导价\\深圳政府指导价_230421.xlsx", { d ->
if(d['M']){
def map = findGuidePrice(d['C'], d['E'])
if(map){
d['M'] = map.avg_price
d['N'] = map.date.toString()
}
}
})
@@ -474,4 +489,14 @@ class FjwPricePkCcbPrice {
})
return map
}
def findGuidePrice(def commid, def city){
def sql = "SELECT gujia_bds_drc_prc,gujia_drc_dt from base.t0831_guided_price_gj_h where urbn_nm='${city}' and gujia_bds_id='${commid}' ORDER BY gujia_drc_dt desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_drc_prc
map.date = r.gujia_drc_dt
})
return map
}
}
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-21")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-26")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//更新静态数据表
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -71,33 +71,33 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
//
//})
def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
def allCitys = CcbCommonUtils.citys_all
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def c : allCitys){
def cityCode = cityCodeMap.get(c)
def count = null;
index++
for(def r : countCitys){
if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
count = r['B']
break
}
}
println index +" "+ c+" "+count
}
def findCcbPriceCredit(sqlInstance, cityid, commid){
def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.month = r.year_month
map.credit = r.CONFIDENCE
map.price = r.avg_price
});
return map
}
//def countCitys = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\pgsql-city.xlsx",0,1)
//def allCitys = CcbCommonUtils.citys_all
//def cityCodeMap = CcbCommonUtils.findCityCodeValue()
//def index = 0
//for(def c : allCitys){
// def cityCode = cityCodeMap.get(c)
// def count = null;
// index++
// for(def r : countCitys){
// if(cityCode.toString() == r['A'].toString().toDouble().toInteger().toString()){
// count = r['B']
// break
// }
// }
// println index +" "+ c+" "+count
//}
//
//def findCcbPriceCredit(sqlInstance, cityid, commid){
// def sql = "select * from (select CONFIDENCE,year_month,avg_price from community_base_price_gj where cityid='${cityid}' and commid='${commid}' order by year_month desc) WHERE ROWNUM <= 1"
// def map = [:]
// sqlInstance.eachRow(sql,{r->
// map.month = r.year_month
// map.credit = r.CONFIDENCE
// map.price = r.avg_price
// });
// return map
//}
def findThirdPriceBySource(sqlInstance, cityid, commid){
def sql = "select * from (select data_source, price_date, community_price,commid from bigdata_third_party_price_gj where cityid='${cityid}' and commid='${commid}' and data_source in ('±´¿Ç','Á´¼Ò') \n" +
@@ -1314,6 +1314,24 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "C:\\Users\\gaojuan\\Desktop\\·¿¼ÛÍøÐÂÔöÐ¡ÇøÊý¾Ý_0427.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
if(count && count==1){
def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
d['E'] = dist.city
}
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['B'] = dist.region
// d['C'] = dist.name
// d['J'] = dist.status
// }
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// d['F'] = newRegion
},0)
//ExcelUtils.modify(fileName, {d->
// if(d['A']){
// def sql = "update guided_price_gj set dealdate=to_date('2022-09-20','yyyy-MM-dd') where id='${d['A']}'"
@@ -92,13 +92,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","12月总量","23年1月总量","23年2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-12-01"
def date = "2023-01-01"
def endDate = "2023-02-01"
def newDate = "2023-03-01"
def startDate = "2023-01-01"
def date = "2023-02-01"
def endDate = "2023-03-01"
def newDate = "2023-04-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
// cityRun(citys, startDate, endDate, cityCodeMap)
// regionRun(citys, startDate, endDate, cityCodeMap)
cityRun(citys, startDate, endDate, cityCodeMap)
regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
@@ -34,6 +34,7 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
def houseStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
citys = CcbCommonUtils.getRunningCitys(args)
@@ -76,7 +77,7 @@ class ExportDistrictRongHe extends BaseBatch{
// exportBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
exportPgBigdataForsale("2023-03-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
}
if(curDate.toString().endsWith("28")){
@@ -1012,7 +1013,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgThirdPartyPrice2(price_date, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
@@ -1184,7 +1185,7 @@ class ExportDistrictRongHe extends BaseBatch{
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
@@ -1273,7 +1274,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
@@ -1381,7 +1382,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
@@ -1417,7 +1418,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
@@ -1532,7 +1533,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
@@ -622,7 +622,10 @@ class CommonDatautils {
public static def findDistrictCount(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(city){
q.put("city", city);
}
if(region){
q.put("region", region);
}
@@ -639,7 +642,10 @@ class CommonDatautils {
public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(city){
q.put("city", city);
}
if(region){
q.put("region", region);
}
@@ -25,7 +25,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
// task.regionPriceTrendRun()
task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202302.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -24,7 +24,13 @@ class AddAvgpriceBatch {
static main(args) {
def task = new AddAvgpriceBatch();
task.run1();
// task.run1();
task.test()
}
def test(){
//发重算均价命令
addPrice("上海","闵行","君莲闵都雅苑", "普通住宅", DateUtils.parseFromTightByPattern('2023-04-23','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-30','yyyy-MM-dd'))
}
def run(){
@@ -2,6 +2,7 @@ package com.fangjia.script.gj
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
@@ -263,8 +264,7 @@ class FiledStrand {
if(all_area){
def v = getValue(all_area)
if(v>=1000 && v<=10000000){
return v.toString()
return v.toString().toDouble().toInteger()
}
}
return null
@@ -273,7 +273,7 @@ class FiledStrand {
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
if(v>=1 && v<=100000){
if(v>5 && v<=100000){
try {
return v.toString().toInteger();
}catch(Exception e){
@@ -284,6 +284,70 @@ class FiledStrand {
return null
}
static def processCarSiteDesc(car_site){
if(car_site && car_site.size()<=20){
return car_site.toString()
}
return null
}
static def processPropertyLife(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
if([40,50,70].contains(v)){
try {
return v.toString()+"Äê";
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processAlias(esVal, name, alias){
if(esVal){
if(esVal!=name){
if(!alias){
alias = []
}
def arr = alias.split(",")
for(def s : arr){
if(!alias.contains(s)){
alias << esVal
}
}
if(alias){
return alias
}
}
}
return null
}
static def processAddress(def esAddress, def city){
if(esAddress){
def formatAddress = AddressUtil.formatAddress(city, esAddress)
if(formatAddress && isRoad(formatAddress)){
return formatAddress
}
}
return null
}
static def isRoad(def s){
def pattern = Pattern.compile("[·|½Ö|´óµÀ|Ïï]");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
static def addBuildAreaToData(def dist, def updateFile){
def build_area = null;
if(dist.build_area){
@@ -324,18 +388,12 @@ class FiledStrand {
return updateFile;
}
public static def processCubagePercent(cubage_percent, estate_type2){
public static def processCubagePercent(cubage_percent){
if(cubage_percent){
def v = getValue(cubage_percent)
if(v){
if(estate_type2 == "±ðÊû"){
if(v>=0.3 && v<=1){
return v.toString()
}
}else{
if(v>1 && v<=10){
return v.toString()
}
if(v>=0.3 && v<=10){
return v.toString()
}
}
}
@@ -418,26 +476,26 @@ class FiledStrand {
estateCharege = estateCharege.toDouble()
if(estateType2=="普通住宅"){
if(city=="上海" || city=="北京" || city=="广州" || city=="杭州"){
if(estateCharege>=0.6 && estateCharege<=3.5){
if(estateCharege>=0.3 && estateCharege<=40){
return estateCharege
}else{
return null;
}
}else{
if(estateCharege>=0.2 && estateCharege<=3.5){
if(estateCharege>=0.2 && estateCharege<=20){
return estateCharege
}else{
return null;
}
}
}else if(estateType2=="别墅"){
if(estateCharege>=3){
if(estateCharege>=1){
return estateCharege;
}else{
return null;
}
}else if(estateType2=="商住"){
if(estateCharege>=4){
if(estateCharege>=2){
return estateCharege;
}else{
return null;
@@ -454,13 +512,13 @@ class FiledStrand {
if(household_count){
household_count = household_count.toInteger()
if(estateType2=="普通住宅" || estateType2=="商住"){
if(household_count>=50 && household_count<=100000){
if(household_count>=15 && household_count<=100000){
return household_count;
}else{
return null;
}
}else if(estateType2=="别墅"){
if(household_count>=20 && household_count<=100000){
if(household_count>=10 && household_count<=100000){
return household_count;
}else{
return null;
@@ -470,6 +528,20 @@ class FiledStrand {
return null;
}
static def processLoudongCount(loudongCount){
def v = getValue(loudongCount.toString());
if(v){
if(v>=1 && v<=10000){
try {
return v.toString().toInteger();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processFinishBuildingDate(def finishBuildingDate){
if(finishBuildingDate){
Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}");
@@ -493,6 +565,39 @@ class FiledStrand {
}
static def processCompany(val){
if(val){
if(val.length()>20){
return null
}else{
if(val.contains("ÎÞ")){
return null
}
def fiterChars = [",",".","?","/","Êý¾Ý",'-','?','&#160;','&nbsp;']
for(def s : fiterChars){
val = val.replace(s, '')
}
if(val.length()<1){
return null
}
if(val.toString().isNumber()){
return null
}
Pattern pattern = Pattern.compile("\\d{4}Äê");
if (pattern.matcher(val).matches()) {
return null
}
return val
}
}
return null;
}
def converList(def dbCursor){
def list = []
for(def dbo : dbCursor){
@@ -143,17 +143,8 @@ class OpenApiUtils {
def result = queryMaxSimilarAddress(city, name)
if(result){
for(def ret : result){
if(ret && ret.lat){
try{
def gmap = GeocodeUtils.getGaodeToBaidu(ret.lng.toDouble(), ret.lat.toDouble())
ret.lng2 = gmap.get("lng")
ret.lat2 = gmap.get("lat")
ret.estateType = ret.type
ret.source = "¸ßµÂ"
}catch (Exception e){
}
}
ret.estateType = ret.type
ret.source = "¸ßµÂ"
}
}
return result
@@ -161,11 +152,17 @@ class OpenApiUtils {
public static def queryMaxSimilarAddress(String city, String address, boolean isFullMatch = false){
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
// if(StringUtils.isEmpty(city)){
// return null
// }
if(StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
// String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=120000&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
//key:8796b9dfb3889b57c38e2fff1b7e51bd »¨Ç®ÂòµÄ
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&city=" + city + "&output=json&offset=20&page=1&key=8796b9dfb3889b57c38e2fff1b7e51bd&extensions=all&citylimit=true"//citylimitÏÞÖÆ³ÇÊÐ
List<com.alibaba.fastjson.JSONObject> poiList = null;
com.alibaba.fastjson.JSONObject result = null;
@@ -553,6 +550,35 @@ class OpenApiUtils {
return null;
}
static def similarRank(proCode, cityCode, regionCode, name, area, floor, height){
def url = "http://localhost:8080/fangjia-open/house/similar?aboveFloorAmount=${height}&buildingName=${name}&city=${cityCode}&district=${regionCode}&" +
"propertyTypeCode=01&province=${proCode}&floorNumber=${floor}&strctureArea=${area}"
println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=2C5B1EDE22A17EDA191B1863B8BD319308E3D89AFBDC76DCD511B639DDF1FC404DA43FE52CB5D8BC666444D9753B39378041962D82A67C4BB45A0E481EE0CA3FF8438C2696B9B57F85B7C110E088361345E89BFEA62FB65442C4ED4508DCEAA931B6C2C05402A3A3"
String content = new HttpRequest().get(url, "utf-8");
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
def dtos = doc.result
return dtos
}
}
return null;
}
}
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-01-01"
static def statDate = "2023-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年2月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -3,6 +3,7 @@ package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
@@ -12,29 +13,106 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
//def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
//def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
//Calendar calendar = Calendar.getInstance()
//calendar.setTime(startDate)
//def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
//def rMap = [:]
//def index = 0
//OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
// index++
// def day = doc.c_date.format("yyyy-MM-dd")
// def count = rMap.get(day)
// if (!count) {
// rMap.put(day, 1)
// } else {
// rMap.put(day, count + 1)
// }
// if(index%10000==0){
// println index+" "+new Date().toString()
// }
//}
//
//rMap.each {k,val->
// println k+" "+val
//}
//def startDate = DateUtil.parseDate("20230416", "yyyyMMdd")
//def endDate = DateUtil.parseDate("20230418", "yyyyMMdd")
//Calendar calendar = Calendar.getInstance()
//calendar.setTime(startDate)
//def count = 0
//def hashSet = []
//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
//while (startDate.before(endDate)) {
// calendar.add(Calendar.DATE, 1)
// def nextDate = calendar.getTime()
// count++
// def total = 0
// OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
// total++
// def result = JSONObject.fromObject(doc.res_map.result)
//// if (!hashSet.contains(result)) {
// def item = [doc._id.toString(), "neiping2017@czbank", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
// data << item
// hashSet << result
// println total+" "+ nextDate
//// } else {
//// println("已存在")
//// }
// }
// if (count % 7 == 0) {
// hashSet.clear()
// }
//
// startDate = nextDate
//
//}
//
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
//ExcelUtils.write("D:\\浙商调用详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
def startDate = DateUtil.parseDate("20230501", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230508", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def data = [[ "公司名称","访问链接", "访问时间", "总量"]]
def rMap = [:]
def index = 0
OPEN_URI_LOG.find(and(eq("user_id", "1007569"), eq("request_uri", "/address/risk"), eq("code", 200))).each { Document doc ->
index++
def day = doc.c_date.format("yyyy-MM-dd")
def count = rMap.get(day)
if (!count) {
rMap.put(day, 1)
} else {
rMap.put(day, count + 1)
}
if(index%10000==0){
println index+" "+new Date().toString()
}
}
def count = 0
def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
rMap.each {k,val->
println k+" "+val
OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << result
}
//FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
ExcelUtils.write("D:\\辽宁卓信普惠征信${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
//def startDate = DateUtil.parseDate("20221201", "yyyyMMdd")
//def endDate = DateUtil.parseDate("20230506", "yyyyMMdd")
//Calendar calendar = Calendar.getInstance()
//calendar.setTime(startDate)
//def count = 0
//def hashSet = []
//def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
//
//OPEN_URI_LOG.find(and(eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", endDate), gte("c_date", startDate))).each { Document doc ->
// def result = JSONObject.fromObject(doc.res_map.result)
// def item = [doc._id.toString(), doc.user_id, doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
// data << item
// hashSet << result
// count++
// println count
//}
//
//
////FileUtils.batchCreateFile(data, "d:\\浙商调用详情1.csv",null, "utf-8", false);
//ExcelUtils.write("D:\\小区详情${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
@@ -33,11 +33,14 @@ class AnjukeDistrictMatch {
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
// def name = "陆家嘴滨江中心N2座"
// name = name.replaceAll("-?\\w+(号写字楼|号楼|楼|座|栋)","")
// println name
def task = new AnjukeDistrictMatch();
// task.run();
// task.runCsv();
// task.updateCredit()
// task.matchCommunity()//查询es小区数据
task.matchCommunity()//查询es小区数据
// task.run1()
// task.updateGreenRateField()//根据第三方数据填补库中字段
// task.updteGreenRateFieldByCsv()
@@ -50,7 +53,7 @@ class AnjukeDistrictMatch {
// task.updteBuildAreaFieldByCsv()
// task.updadteAllAreaFieldByCsv()
// task.updatePlotRateFieldByCsv()
task.updateFinishDateFieldByCsv()
// task.updateFinishDateFieldByCsv()
}
def updateCredit(){
@@ -728,16 +731,17 @@ class AnjukeDistrictMatch {
def run(){
def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\全国绿化率.xlsx"
// def datas = readTxt("D:\\数据\\建行\\二期\\第三方小区字段补充\\第三方网站绿化率 (3).csv")
def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)0406.xlsx"
def index = 0
ExcelUtils.multiThreadingModify(filePath,{row->
index++
if(row['A'] && row['B']){
def city = row['A']
def name = row['C']
def region = row['B']
def name = row['B']
def region = null
// def region = row['B']
if(name && name.contains("·")){
name = name.replace("·", "")
@@ -785,27 +789,27 @@ class AnjukeDistrictMatch {
row['AC'] = jsonObject.get("credit")
}
}
println index +" "+row['Y']
// if(row['G']){
// def dbo = CommonDatautils.findDistrictById(row['G'])
// if(dbo){
// row['H'] = dbo.estate_type2
// row['I'] = dbo.name_cat
//
// def count = findBigdataCommunity(row['G'])
// if(count){
// row['E'] = "是"
// }
// }
// }
// println index +" "+row['Y']
if(row['G']){
def dbo = CommonDatautils.findDistrictById(row['G'])
if(dbo){
row['H'] = dbo.estate_type2
row['I'] = dbo.name_cat
def count = findBigdataCommunity(row['G'])
if(count){
row['E'] = "是"
}
}
}
}
})
}
def matchCommunity(){
def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)3.24_2.xlsx"
def filePath = "D:\\数据\\小区\\押品发现新增小区确认(结果)0406-1.xlsx"
ExcelUtils.modify(filePath, { row ->
if (row['A'] && row['B'] && row['I']) {
if (row['A'] && row['B'] && !row['O']) {
def rets = findAllEsData(row['A'], row['B'])
if(rets){
def dbo = rets[0]
@@ -480,7 +480,7 @@ class OfficeProcess {
}
def checkRegion(){
def citys = ["北京","广州","重庆","天津","成都","南京"]//"上海","深圳",
def citys = ["杭州","苏州"]//"上海","深圳","北京","广州","重庆","成都",
for(def city : citys){
def regionPointsMap = findRegionRange(city)
def addDatas = findAddDatasByClickhouse(city)
@@ -16,7 +16,140 @@ class FieldStat {
// task.run()
// task.run1()
// task.run2()
task.run3()
// task.run3()
// task.offircRun()
task.distRun()
task.shopRun()
}
def distRun(){
def citys = CommonDatautils.findAllCity()
def cityMap = [:]
def fileds = ["region","name","alias","address","addr_alias","estate_type2","build_type","block","loop_line_info","street_community","household_count","curr_household_count","virescence_percent","cubage_percent","all_area","build_area","estate_charge","car_site","car_site_count","developer","estate_company","finish_building_date","live_date","start_building_date","open_date","property_life","floor_info","fitment","fitment_materials","build_structure","traffic","facilities","characteristic","sell_address","relate_info","description","total_price","mobile","building_status","licence","build_designer","garden_designer","business_quarter","bay_area","payment_pattern","land_plan_license","prj_plan_license","land_use_license","cons_license","elevators_count","build_date","divisible","status","metro_info","metro_names","cons_company","sells_company","contact_people","parking_fee","delivery_level","fitment_level","land_car_site","underground_car_site","avg_price","sell_price","standard_area","pics","loudong_count","committee","house_structure","usage_card","mainTypeArea"]
for(def c : citys){
def dbo = findCoverPrecent(c)
if(dbo && dbo.containsField("district_detail")){
def officeDetail = dbo.district_detail
cityMap.put(c, officeDetail)
}else{
cityMap.put(c, [:])
}
}
def rets = []
rets << ["城市","数量","区域","名称","别名","地址","地址别名","物业类型","建筑类型","板块","环线位置","街道社区","户数","当前户数","绿化率","容积率","占地面积","建筑面积","物业费","停车位","停车位数","开发商","物业公司","竣工时间","入住时间","开工时间","开盘时间","产权年限","楼层情况","装修情况","装修材料","建筑结构","交通情况","周边配","小区特征","售楼地址","相关信息","描述","总价","电话","建筑状态","预售许可证","建筑单位","园艺设计单位","商圈","户型面积","支付方式","用地规划许可证","工程规划许可证","土地使用证","施工许可证","电梯数量","建造年代","是否可分割","状态","地铁沿线","地铁站名","施工单位","销售企业","联系人","停车费","交付标准","装修标准","地上车位数","地下车位数","均价","出售价格","标准层面积","图片","楼栋总数","居委会","房屋结构","土地使用权证号","主力户型"]
def ret = []
ret << "city"
ret << "count"
for(def f : fileds){
ret << f
}
rets << ret
cityMap.each {k,v->
def row = []
row << k
row << CommonDatautils.findAllDistrictCount("district", k)
for(def filed : fileds){
if(v){
def val = v.get(underlineToCamel(filed))
if(val){
row << val.toDouble()*100
}else{
row << ""
}
}
}
rets << row
}
ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "小区")
}
def shopRun(){
def citys = CommonDatautils.findAllCity()
def cityMap = [:]
def fileds = ["region","name","alias","address","business_quarter","property_status","rent_type","lease_term","payment_pattern","sell_price","household_count","mobile","floor","floor_info","bay_area","finish_building_date","live_date","divisible","unified_manage","car_site_count","car_site","all_area","build_area","estate_charge","developer","estate_company","metro_info","metro_names","facilities","property_info","traffic","surrounding","business_status","url","description","open_date","sell_address","block","height","loop_line_info","cubage_percent","virescence_percent","characteristic","sells_company","licence","build_designer","cons_company","property_life","insidepos","delivery_level","start_building_date","land_plan_license","prj_plan_license","land_use_license","cons_license","fitment","build_structure","enter_company","build_type","elevators_count","estate_type2","status","standard_area","relate_info","garden_designer","fitment_level","build_date","fitment_materials","rent_price","out_pics","pics","land_car_site","underground_car_site","addr_alias"]
for(def c : citys){
def dbo = findCoverPrecent(c)
if(dbo && dbo.containsField("shop_detail")){
def officeDetail = dbo.shop_detail
cityMap.put(c, officeDetail)
}else{
cityMap.put(c, [:])
}
}
def rets = []
rets << ["城市","数量","区域","名称","别名","地址","商圈","建筑状态","租售类型","租赁年限","付款方式","出售价格","总户数","电话","建筑楼层","楼层状况","开间面积","竣工时间","开业时间","可分割","统一管理","车位数量","车位描述","占地面积","建筑面积","物业费","开发商","物业公司","地铁沿线","地铁站名","配套设施","物业概况","交通情况","周边情况","经营业态","来源url","描述","开盘时间","售楼地址","板块","楼高","环线位置","容积率","绿化率","商业特色","销售企业","销售许可证","设计单位","施工单位","产权年限","交通位置","交付标准","开工时间","用地规划许可证","工程规划许可证","土地使用证","施工许可证","装修","建筑结构","入驻企业","建筑类型","电梯数量","物业类型","状态","标准层面积","相关信息","景观设计单位","装修标准","建造年代","建材装修","出租价格","外景图","其它图","地上车位数","地下车位数","地址别名"]
def ret = []
ret << "city"
ret << "count"
for(def f : fileds){
ret << f
}
rets << ret
cityMap.each {k,v->
def row = []
row << k
row << CommonDatautils.findAllDistrictCount("shop", k)
for(def filed : fileds){
if(v){
def val = v.get(underlineToCamel(filed))
if(val){
row << val.toDouble()*100
}else{
row << ""
}
}
}
rets << row
}
ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "商铺")
}
def offircRun(){
def citys = CommonDatautils.findAllCity()
def cityMap = [:]
def fileds = ["region","name","alias","latLng","latLng2","address","addrAlias","bedKind","estateType2","buildType","block","businessQuarter","loopLineInfo","licence","sideset","realUsePercent","buildDesigner","consCompany","estateCharge","carSiteCount","carSite","undergroundCarSite","landCarSite","parkingFee","sight","propertyLife","lobbyHeight","standardHeight","clearHeight","standardArea","foreign","register","lightingVentilation","conditionerTime","","elevatorsCount","personElevatorsCount","goodsElevatorCount","openDate","finishBuildingDate","liveDate","allArea","buildArea","cubagePercent","virescencePercent","developer","estateCompany","networkWay","fitmentLevel","buildStructure"]
for(def c : citys){
def dbo = findCoverPrecent(c)
if(dbo && dbo.containsField("office_detail")){
def officeDetail = dbo.office_detail
cityMap.put(c, officeDetail)
}else{
cityMap.put(c, [:])
}
}
def rets = []
rets << ["城市","数量","区域","名称","名称别名","经纬度(百度)","经纬度(高德)","地址","地址别名","写字楼等级","写字楼类型","建筑类别","板块","商圈","环线位置","预售许可证","是否可分割","得房率","设计单位","施工单位","物业费","停车位数","停车描述","地上车位数","地下车位数","车位租金","开间面积","产权年限","大堂层高","标准层高","净高","标准层面积","是否涉外","是否可注册","空调","空调开放时间","空调费用","电梯数量","客梯数","货梯数","开盘时间","竣工时间","入住时间","占地面积","建筑面积","容积率","绿化率","开发商","物业公司","网络","装修状况","结构"]
def ret = []
ret << "城市"
ret << "数量"
for(def f : fileds){
ret << f
}
rets << ret
cityMap.each {k,v->
def row = []
row << k
row << CommonDatautils.findAllDistrictCount("office", k)
for(def filed : fileds){
if(v){
def val = v.get(filed)
if(val){
row << val.toDouble()*100
}else{
row << ""
}
}
}
rets << row
}
ExcelUtils.write("d:\\数据\\写字楼\\写字楼覆盖率.xlsx", rets, "写字楼")
}
def run3(){
@@ -136,4 +269,24 @@ class FieldStat {
}
return obj
}
public static String underlineToCamel(String param){
if (param==null||"".equals(param.trim())){
return "";
}
int len=param.length();
StringBuilder sb=new StringBuilder(len);
for (int i = 0; i < len; i++) {
char c=param.charAt(i);
if (c=="_"){
if (++i<len){
sb.append(Character.toUpperCase(param.charAt(i)));
}
}else{
sb.append(c);
}
}
return sb.toString();
}
}
@@ -14,19 +14,55 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class priceRank {
class ApiTest {
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new priceRank()
task.run()
def task = new ApiTest()
// task.rankRunTest()
// task.similarRunTest()
task.loupanLevelTest()
}
def run(){
def loupanLevelTest(){
ExcelUtils.modify("D:\\数据\\卓信\\上海 - 副本.xlsx", { d ->
if (d['A']) {
def results = com.fangjia.script.gj.OpenApiUtils.financialRating(d['A'], d['B']+d['C'])
if(results){
d['D'] = results.consumptionRating
}
}
})
}
def similarRunTest(){
ExcelUtils.modify("D:\\数据\\江南农商行\\住宅信息V2-差异案例(反馈).xlsx", { d ->
if(d['A']){
def cityCode = d['U'].toString().toDouble().toInteger()
def proCode = d['V'].toString().toDouble().toInteger()
def regionCode = d['T'].toString().toDouble().toInteger()
def name = d['D']
def area = d['F']
def floor = d['H'].toString().toDouble().toInteger()
def height = d['G'].toString().toDouble().toInteger()
def results = com.fangjia.script.gj.OpenApiUtils.similarRank(proCode, cityCode,regionCode, name, area, floor, height)
if(results){
def str = ""
for(def r : results){
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"\n"
}
d['Y'] = str
}
}
})
}
def rankRunTest(){
def dists = CommonDatautils.findAllDistrictByCity("ÉϺ£","district", ["city","region","name"])
//def dists = CommonDatautils.findDistrictByCityAndRegions("ÉϺ£","district", ["ÆÖ¶«"], ["city","region","name"])
def rets = createCityDatas(dists)
@@ -0,0 +1,417 @@
package com.fangjia.script.gj.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 小区字段入库(自动入库)
*/
@BatchOwner(Watcher.GaoJuan)
class DistrictFieldAddBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018)
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源
//"alias":"alias",
def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段
static main(args) {
runBatch(new DistrictFieldAddBatch(), args)
}
@Override
void process(Object args) {
def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京"
def analysRets = []
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
// addFieldFromRonghe(c, totalCount)
analysRets = addFieldFromMatch(c, totalCount, analysRets)
}
println analysRets.toString()
ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
}
def addFieldFromMatch(c, totalCount, analysRets){
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource)
def matchRets1 = matchApi(esRets)
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
def updateRets = processFieldsFromMatch(c , matchRets2)
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
if(esRets){
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
}
analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2]
}
return analysRets;
}
def addFieldFromRonghe(c, totalCount){
sourceMap.each{seaweedmapSource,esWebSource->
def rhRets = findRongheDists(c, seaweedmapSource)
def esRets = findEsDistrict(c, esWebSource)
def updateRets = processFieldsFromRh(c, rhRets, esRets)
println "【通过融合关系填充字段】网站:"+seaweedmapSource +" 第三方融合表:"+rhRets.size() +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, rhRets.size()*100/totalCount)+"%"
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "----------------------------------------------------------------------------------"
}
}
def processFieldsFromRh(c, rhRets, esRets){
def fields = fieldsMap.keySet() as List
def fieldValueMap = [:]
for(def rhDist : rhRets){
def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields)
if(!dist){
println c+" "+rhDist.commid+" 不存在"
continue
}
def esDbo = esRets.get(c + "#" + rhDist.id)
if(!esDbo){
println c+" "+rhDist.id+" 不存在[ES]"
continue
}
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = esDbo.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
if(newValues){
fieldValueMap.put(dist._id.toString(), newValues)
}
}
println fieldValueMap.size()
return fieldValueMap;
}
def processFieldsFromMatch(c,matchRets){
def fields = fieldsMap.keySet() as List
fields << "city"
def fieldValueMap = [:]
for(def matchDist : matchRets){
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
continue
}
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = matchDist.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
newVal = FiledStrand.processCompany(esVal)
}
if(seaweedfield =="virescence_percent"){
newVal = FiledStrand.processVirescencePercent(esVal)
}
if(seaweedfield =="cubage_percent"){
newVal = FiledStrand.processCubagePercent(esVal)
}
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
newVal = FiledStrand.processArea(esVal)
newVal = newVal.toString()
}
if(seaweedfield =="household_count"){
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
}
if(seaweedfield =="loudong_count"){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(seaweedfield =="car_site_count"){
newVal = FiledStrand.processCarSite(esVal)
}
if(seaweedfield =="car_site"){
newVal = FiledStrand.processCarSiteDesc(esVal)
}
if(seaweedfield =="property_life"){
newVal = FiledStrand.processPropertyLife(esVal)
}
if(seaweedfield =="alias"){
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
}
if(seaweedfield =="address"){
newVal = FiledStrand.processAddress(esVal, dist.city)
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
if(newValues){
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
}
}
return fieldValueMap;
}
def findRongheDists(def city, def source){
def keys = new BasicDBObject()
keys.put("commid", 1)
keys.put("city", 1)
keys.put("id", 1)
keys.put("source", 1)
def dbcursor = SEAWEEDMAP_COLL.find(new BasicDBObject().append("city", city).append("source", source), keys)
def rets = []
for(def d : dbcursor){
rets << ["commid":d.commid, "city":d.city, "source":d.source, "id":d.id]
}
return rets;
}
def findEsDistrict(c, esWebSource){
def map = [:]
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
if(r.id && !map.containsKey(c+"#"+r.id)){
map.put(c+"#"+r.id, r)
}
})
return map
}
def findEsDatas(c, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
list << r
})
return list
}
def fieldUpdate(total,updateRets){
def countMap = [:]
for(def rm : updateRets){
def valMap = rm.value
fieldsMap.each{seaweedfield,esField->
if(valMap.containsKey(seaweedfield)) {
if(countMap.containsKey(seaweedfield)) {
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
}else{
countMap.put(seaweedfield, 1)
}
}
}
}
println countMap.toString()
def precentMap = [:]
fieldsMap.each() {key,skey->
if(countMap.containsKey(key)) {
precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
}else {
precentMap.put(skey, 0)
}
}
return precentMap;
}
def matchApi(esRets){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(esRets.size())
for(def i=0;i<esRets.size();i++){
final def data = esRets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = matching(data)
if(ret){
// println totalCount.get() +" "+data.city+" "+data.name+" [Match] "+data.match_region+" "+data.match_name +" "+data.flag+" "+data.credit
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def selectRestByCredit(matchRets){
def matchRets1 = []
for(def r : matchRets){
if(r.credit && r.credit>=0.9){
matchRets1 << r
}
}
return matchRets1
}
def matching(data){
def name = data.name
if(data.name && data.name.contains("·")){
name = data.name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(data.city, data.region, name)
if(!dist){
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district")
flag = "映射别名匹配"
}
if(!dist) {
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district")
if(rets && rets.size()==1){
flag = "别名匹配"
dist = rets[0]
}
}
if(dist){
data.flag = flag
data.match_id = dist._id.toString()
data.match_region = dist.region
data.match_name = dist.name
data.credit = 0.95
}else{
def jsonObject = OpenApiClient.addressMatch(data.city, name, "property", false,data.region, null)
if(jsonObject!=null && jsonObject.precise==-1){
if(data.address && isStandardRoad(data.address)){
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "property", false,data.region, name)
}else{
data.flag = "匹配接口-地址"
}
}else{
data.flag = "匹配接口-名称"
}
if(jsonObject != null && jsonObject.precise!=-1) {
data.match_id = jsonObject.get("id")
data.match_region = jsonObject.get("district")
data.match_name = jsonObject.get("name")
data.credit = jsonObject.get("credit")
}
}
if(data.match_id){
return data
}
return null
}
def updateDB(c, updateRets){
for(def rm : updateRets) {
def id = rm.key
def valMap = rm.value
BasicDBObject udata = new BasicDBObject()
valMap.each {k,v->
udata.put(k,v)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
}
}
def matchDist(city, region, name){
def dist = CommonDatautils.findDistrict(city, region, name, "district")
return dist;
}
def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
}
@@ -0,0 +1,71 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class SameDistName {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
static main(args) {
def task = new SameDistName()
task.run()
}
def run(){
def dists = CommonDatautils.findAllDistrictByCity("常州", "district", ["city","region", "name", "address","lat", "lng"])
def map = [:]
for(def d : dists){
def list = map.get(d.name)
if(!list){
list = []
map.put(d.name, list)
}
list << d
}
def datas = []
map.each {k,v->
if(v && v.size()>=2){
datas.addAll(v)
}
}
def retDats = []
retDats << ["城市","区域","名称","地址","区域状况"]
if(datas){
def regionPointsMap = findRegionRange("常州")
for(def d : datas){
def lng = d.lng/10000000000.0
def lat = d.lat/10000000000.0
def flag = isInArea(lng, lat, regionPointsMap.get(d.region))
retDats << [d.city, d.region, d.name, d.address, flag]
}
}
ExcelUtils.write("D:\\数据\\小区\\相同小区不同区域.xlsx", retDats, "数据")
}
def findRegionRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline)
}
return regionMap
}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
}
@@ -1,6 +1,8 @@
package com.fangjia.script.gj.es
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
@@ -8,16 +10,39 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
findLogs()
//findLogs()
//findFangjiaLogs()
stat()
def stat(){
def datas = ExcelUtils.read("D:\\数据\\fjw-提交.xlsx",0,1)
def map=[:]
for(def d : datas){
def count = map.get(d['C'])
if(!count){
map.put(d['C'], 1)
}else{
map.put(d['C'], count+1)
}
}
map.each {k,v->
println k +" "+v
}
}
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
// def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
// def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
// QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
def q = QueryBuilders.fuzzyQuery("domain", ".fangjia.com")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size()
def set = new HashSet()
@@ -39,4 +64,48 @@ def findLogs(){
}
return total
}
def findFangjiaLogs(){
def startDate = DateUtil.parseDate("20230329", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230429", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def list = []
def data = [["mid", "ip", "访问时间"]]
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
// def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
// def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("@timestamp").from(startDate).to(nextDate).includeLower(true).includeUpper(false);
def q = QueryBuilders.fuzzyQuery("request", "/doAdd2")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q).must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.86", "fangjia-log*", queryBuilder);
println houses.size()
if (houses) {
for (def d : houses) {
def reqStr = d.request.toString()
def mid = null
if (reqStr.contains("mid=")) {
mid = reqStr.substring(reqStr.indexOf("mid=") + 4, reqStr.indexOf("mid=") + 8)
}
def time = d.get("@timestamp").toString()
time = time.substring(0, time.indexOf("T"))
def ip = d.remote_addr
data << [mid, ip, time]
}
}
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\fjw-提交.xlsx", data, "查询日志")
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.es
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -9,14 +10,18 @@ import org.elasticsearch.xpack.sql.jdbc.EsDataSource
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.25:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
findLogs()
//findLogs()
findFangjiaLogs()
def findLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
def q = QueryBuilders.fuzzyQuery("request", "/jnBank")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder);
@@ -53,3 +58,35 @@ def findLogs(){
return total
}
def findFangjiaLogs(){
def total = 0
try{
// String sql = "select * from fangjia-log* where request like '/jnBank%'"
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
def startDate = DateUtil.parseDate("20230427", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230428", "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("time_local").from(startDate).to(endDate).includeLower(true).includeUpper(false);
def q = QueryBuilders.fuzzyQuery("request", "/enquire")
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(q)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.25", "fangjia-log*", queryBuilder);
println houses.size()
def set = new HashSet()
if(houses) {
for (def d : houses) {
}
}
println set
}catch(Exception e){
e.printStackTrace()
}
return total
}
File diff suppressed because one or more lines are too long
@@ -32,7 +32,7 @@ public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
static main(args) {
def task = new BeiKeCrawlByChrome();
@@ -44,7 +44,7 @@ public class BeiKeCrawlByChrome {
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
@@ -112,18 +112,20 @@ public class BeiKeCrawlByChrome {
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
for(def m : citys){
driver.get(m.url+"/chengjiao/")
def city = m[0]
def url = m[1]
driver.get(url+"/chengjiao/")
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(λÖÃ)+dl a", doc)
if(regions){
for(def r : regions){
def regionText = r.text()
def href = m.url+ r.attr("href")
def href = url+ r.attr("href")
println "开始抓取【${m.city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, m.city, href, regionText)
println m.url +" 详情链接:"+detailUrls.size()
println "开始抓取【${city}】【${regionText}】【${href}】"
def detailUrls = getPageDeail(driver, city, href, regionText)
println url +" 详情链接:"+detailUrls.size()
}
}
@@ -183,7 +185,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, region)
@@ -398,16 +400,13 @@ public class BeiKeCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
def list = []
def citys = ["咸阳","九江","昆山","郑州","廊坊","漳州","长春"]
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
// cityMap.put(e.text(), "https:"+href)
// }
def href = "https:"+e.attr("href")
if (citys.contains(e.text())){
list << [e.text(), href]
}
}
}
return list
@@ -432,6 +431,7 @@ public class BeiKeCrawlByChrome {
elemPassword.sendKeys("12qwaszx");
btn.submit(); // µã»÷ÐÐΪ£»
Thread.sleep(10000)
}
}
@@ -31,7 +31,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -117,10 +117,11 @@ public class LianjiaCrawlByChrome {
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
// cityMap = cityMap.reverse()
getLogin(driver)
crawlData(driver, cityMap)
@@ -184,12 +185,15 @@ public class LianjiaCrawlByChrome {
}
}
def crawlData(driver, cityMap){
def crawlData(driver, cityList){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
for(def ret : cityList){
def city = ret[0]
def cityUrl = ret[1]
//def detailUrls = getPageUrls(driver, cityUrl)
driver.get(cityUrl+"chengjiao/")
Thread.sleep(10000)
def doc = Jsoup.parse(driver.getPageSource());
def regions = JQueryUtils.select(".position dl:contains(λÖÃ)+dl a", doc)
if(regions){
@@ -251,7 +255,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e, regionText)
@@ -473,17 +477,17 @@ public class LianjiaCrawlByChrome {
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["ËÕÖÝ"]
def cityList = []
def citys= ["À¥É½","¾Å½­","Ö£ÖÝ","ÀÈ·»","ÕÄÖÝ","ÏÌÑô","³¤´º","ÖØÇì"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityMap.put(e.text(), href)
cityList << [e.text(), href]
}
}
}
return cityMap
return cityList
}
def getLogin(driver){
@@ -505,6 +509,7 @@ public class LianjiaCrawlByChrome {
elemPassword.sendKeys("12qwaszx");
btn.submit(); // µã»÷ÐÐΪ£»
Thread.sleep(20000)
}
}
@@ -21,10 +21,10 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\招商\\招商.xlsx"
// addGXDPrice(filePath)
def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
addThirdPrice(filePath)
// addThirdPrice(filePath)
}
@@ -104,12 +104,13 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['M']) {
def dist = CommonDatautils.findDistrictById(d['M'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
d['AO'] = gxdPrice
}
if (d['B']) {
// def dist = CommonDatautils.findDistrictById(d['B'])
// if(dist){
// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['G'] = gxdPrice
// }
}
})
}
@@ -23,11 +23,16 @@ class ChongQingBlankTest {
static main(args) {
def task = new ChongQingBlankTest()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row->
allRegionMap.each{k,v->
println k+" "+v.size()
}
/* ExcelUtils.modify("D:\\数据\\重庆农商行\\重庆农商行测试样本(房价网v.1.2).xlsx",{row->
if(row['B'].toString()=="重庆市九龙坡区建工三村7号"){
def city = CommonUtils.parseCityInAddress(row["B"])
if(!city){
@@ -197,7 +202,7 @@ class ChongQingBlankTest {
}
}
}
}, 0)
}, 0)*/
}
@@ -27,8 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产"]
static main(args) {
runBatch(new SynchDistrictPrice(), args);
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202301);
def count2 = findTotalCount(source, 202302);
def count1 = findTotalCount(source, 202302);
def count2 = findTotalCount(source, 202303);
println source +" "+count1 +" "+count2
}
}
@@ -563,6 +563,8 @@ public interface DistrictService extends ModeService<DistrictKey, DistrictBo> {
*/
Integer getBuildYear(District district);
Integer getCreateYear(District district);
/**
* in查询小区信息
*
@@ -2667,6 +2667,15 @@ public class DistrictServiceImpl implements DistrictService {
}
}
public Integer getCreateYear(District district) {
Integer buildYear = getYearFormDate(district.getCreateYear());
int minYear = Integer.MAX_VALUE;
if (buildYear != null && buildYear < minYear) {
minYear = buildYear;
}
return minYear;
}
public Integer getBuildYear(District district) {
Integer finishYear = getYearFormDate(district.getFinishBuildingDate());