Merge branch 'script_gj' into 'master'
Script gj See merge request !673
This commit is contained in:
+4
@@ -102,6 +102,7 @@ public class ApiUserController {
|
||||
bo.setPurpose(apiUser.getPurpose());
|
||||
bo.setDeadTime(apiUser.getDeadTime());
|
||||
bo.setManager(apiUser.getManager());
|
||||
bo.setIsMonitor(apiUser.getIsMonitor());
|
||||
|
||||
apiUserBoList.add(bo);
|
||||
}
|
||||
@@ -183,6 +184,7 @@ public class ApiUserController {
|
||||
apiUser.setModificationDate(date);
|
||||
apiUser.setDeadTime(model.getDeadTime());
|
||||
apiUser.setManager(model.getManager());
|
||||
apiUser.setIsMonitor(model.getIsMonitor());
|
||||
this.apiUserService.save(apiUser);
|
||||
if(model.getEvaluateRate() != null && model.getEvaluateRate() != 0){
|
||||
ApiUserCfg apiUserCfg = new ApiUserCfg();
|
||||
@@ -222,6 +224,7 @@ public class ApiUserController {
|
||||
bo.setEmail(apiUser.getEmail());
|
||||
bo.setDeadTime(apiUser.getDeadTime());
|
||||
bo.setManager(apiUser.getManager());
|
||||
bo.setIsMonitor(apiUser.getIsMonitor());
|
||||
ApiUserCfg apiUserCfg_rate = apiUserCfgService.findByUserId(apiUser.getId(), ApiUserCfg.ApiUserCfgKey.EVALUATE_RATE);
|
||||
if(apiUserCfg_rate != null){
|
||||
bo.setEvaluateRate((Double) apiUserCfg_rate.getCfgValue());
|
||||
@@ -272,6 +275,7 @@ public class ApiUserController {
|
||||
dataApiUser.put("purpose", model.getPurpose());
|
||||
dataApiUser.put("deadTime", model.getDeadTime());
|
||||
dataApiUser.put("manager", model.getManager());
|
||||
dataApiUser.put("isMonitor", model.getIsMonitor());
|
||||
this.apiUserService.update(dataApiUser, userId);
|
||||
|
||||
//ApiUserCfg.ApiUserCfgKey.EVALUATE_RATE
|
||||
|
||||
@@ -38,6 +38,7 @@ public class ApiUserModel extends BaseModel {
|
||||
private Integer creditLevelStatus;
|
||||
private String priceType;
|
||||
private Map<String,Double> priceRatingMap;
|
||||
private Integer isMonitor;
|
||||
|
||||
public String getPriceType() {
|
||||
return priceType;
|
||||
@@ -606,4 +607,12 @@ public class ApiUserModel extends BaseModel {
|
||||
public void setManager(String manager) {
|
||||
this.manager = manager;
|
||||
}
|
||||
|
||||
public Integer getIsMonitor() {
|
||||
return isMonitor;
|
||||
}
|
||||
|
||||
public void setIsMonitor(Integer isMonitor) {
|
||||
this.isMonitor = isMonitor;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,13 +57,20 @@
|
||||
<td>账户用途</td>
|
||||
<td>
|
||||
<select name="purpose">
|
||||
<option value="" >请选择</option>
|
||||
<option value="" >请选择</option>
|
||||
<option value="1" >API</option>
|
||||
<option value="2" >PDS</option>
|
||||
<option value="4" >房生活</option>
|
||||
</select>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>是否监控</td>
|
||||
<td>
|
||||
<input type="radio" name="isMonitor" value="0" checked="checked"/>不监控
|
||||
<input type="radio" name="isMonitor" value="1" style="margin-left: 15px;"/>监控
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Email</td>
|
||||
<td><input name="email" value="${email!}" /></td>
|
||||
|
||||
@@ -72,6 +72,13 @@
|
||||
</select>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>是否监控</td>
|
||||
<td>
|
||||
<input type="radio" <#if apiUserBo?? && apiUserBo.isMonitor?? && apiUserBo.isMonitor == 0>checked="checked"</#if> name="isMonitor" value="0" />不监控
|
||||
<input type="radio" <#if apiUserBo?? && apiUserBo.isMonitor?? && apiUserBo.isMonitor == 1>checked="checked"</#if> name="isMonitor" value="1" style="margin-left: 15px;"/>监控
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Email</td>
|
||||
<td><input name="email" value="${apiUserBo.email!}"/></td>
|
||||
|
||||
@@ -53,6 +53,9 @@
|
||||
<th align="center" width="60px">
|
||||
账户用途
|
||||
</th>
|
||||
<th align="center" width="60px">
|
||||
是否监控
|
||||
</th>
|
||||
<th align="center" width="120px">
|
||||
到期时间
|
||||
</th>
|
||||
@@ -114,6 +117,13 @@
|
||||
房生活
|
||||
</#if>
|
||||
</td>
|
||||
<td>
|
||||
<#if bo.isMonitor==1>
|
||||
是
|
||||
<#else>
|
||||
否
|
||||
</#if>
|
||||
</td>
|
||||
<td align="center">
|
||||
<#if bo.deadTime??>
|
||||
${bo.deadTime?string("yyyy年MM月dd日")}
|
||||
|
||||
@@ -39,6 +39,8 @@ public class ApiUser implements SqlEntity {
|
||||
|
||||
private String manager;
|
||||
|
||||
private Integer isMonitor;
|
||||
|
||||
@Id
|
||||
@Column(name = "ID")
|
||||
public long getId() {
|
||||
@@ -175,4 +177,12 @@ public class ApiUser implements SqlEntity {
|
||||
public void setManager(String manager) {
|
||||
this.manager = manager;
|
||||
}
|
||||
@Column(name = "IS_MONITOR")
|
||||
public Integer getIsMonitor() {
|
||||
return isMonitor;
|
||||
}
|
||||
|
||||
public void setIsMonitor(Integer isMonitor) {
|
||||
this.isMonitor = isMonitor;
|
||||
}
|
||||
}
|
||||
|
||||
+60
-17
@@ -19,9 +19,9 @@ class CcbDataStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDataStat()
|
||||
// task.run()
|
||||
task.run(args)
|
||||
// task.tradeRun()
|
||||
task.regionDistStat()
|
||||
// task.regionDistStat()
|
||||
}
|
||||
|
||||
def regionDistStat(){
|
||||
@@ -124,22 +124,37 @@ class CcbDataStat {
|
||||
return count
|
||||
}
|
||||
|
||||
def run(){
|
||||
def run(args){
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
|
||||
if(d['A']) {
|
||||
def citycode = cityCodeDataMap.get(d['A'])
|
||||
def total = findTotalCount(citycode)
|
||||
def f_count = findDistFinishDate(citycode)
|
||||
def b_count = findBuildTypeCount(citycode)
|
||||
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
|
||||
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
|
||||
d['C'] = f_pre
|
||||
d['D'] = b_pre
|
||||
d['E'] = total
|
||||
println d['A'] +" "+ total +" "+f_count+" "+b_count
|
||||
}
|
||||
})
|
||||
def rets = []
|
||||
rets << ["城市","经纬度","物业类型","建筑年代","建筑类型","绿化率","容积率"]
|
||||
for(def c : citys){
|
||||
def ret = []
|
||||
ret << c
|
||||
def citycode = cityCodeDataMap.get(c)
|
||||
def total = findTotalCount(citycode)
|
||||
def lat_count = findLocationCount(citycode)
|
||||
def f_count = findDistFinishDate(citycode)
|
||||
def b_count = findBuildTypeCount(citycode)
|
||||
def g_count = findVirescencePercent(citycode)
|
||||
def c_count = findCubagePercentPercent(citycode)
|
||||
def location_pre = DataHandleUtils.getDoubleRound(2, lat_count*100/total)
|
||||
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
|
||||
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
|
||||
def g_pre = DataHandleUtils.getDoubleRound(2, g_count*100/total)
|
||||
def c_pre = DataHandleUtils.getDoubleRound(2, c_count*100/total)
|
||||
ret << location_pre
|
||||
ret << 100
|
||||
ret << b_pre
|
||||
ret << f_pre
|
||||
ret << g_pre
|
||||
ret << c_pre
|
||||
ret << total
|
||||
rets << ret
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\建行供数重要字段覆盖率.xlsx", rets, "统计")
|
||||
}
|
||||
|
||||
def findTotalCount(citycode){
|
||||
@@ -168,4 +183,32 @@ class CcbDataStat {
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findLocationCount(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and longitude>0";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findVirescencePercent(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and GREENINGRATE is not null";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def findCubagePercentPercent(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and PLOTRATIO is not null";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -61,9 +61,9 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
|
||||
|
||||
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
|
||||
cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
// cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
// regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
}
|
||||
|
||||
def cityRun(citys, startDate, endDate, cityCodeDataMap){
|
||||
|
||||
+44
-44
@@ -45,27 +45,27 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
//按天导(每天一个文件夹)
|
||||
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
// exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
|
||||
if(curDate.toString().endsWith("10")){
|
||||
exportBigdataCommunityGJRH()//
|
||||
exportBigdataBuildingFJW()//
|
||||
exportBigdataCommfeeGj()//
|
||||
|
||||
exportBigdataCommbuildstruGJ()//
|
||||
exportBigdataCommhousetypeGJ()//
|
||||
exportBigdataCommbuildyearGj()//
|
||||
exportBigdataCommhouseuseGJ()//
|
||||
exportBigdataPositionGJ()//
|
||||
exportBigdataDistrictFJW()//
|
||||
exportBigdataAliasFJW()//
|
||||
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
exportEvaluateRate()
|
||||
|
||||
exportBigdataCityPrice(queryDate)
|
||||
exportBigdataDistPrice(queryDate)
|
||||
// if(curDate.toString().endsWith("10")){
|
||||
// exportBigdataCommunityGJRH()//
|
||||
// exportBigdataBuildingFJW()//
|
||||
// exportBigdataCommfeeGj()//
|
||||
//
|
||||
// exportBigdataCommbuildstruGJ()//
|
||||
// exportBigdataCommhousetypeGJ()//
|
||||
// exportBigdataCommbuildyearGj()//
|
||||
// exportBigdataCommhouseuseGJ()//
|
||||
// exportBigdataPositionGJ()//
|
||||
// exportBigdataDistrictFJW()//
|
||||
// exportBigdataAliasFJW()//
|
||||
//
|
||||
// exportBigdataBuildprojFjw()
|
||||
// exportBigdataCommlayoutpicFjw()
|
||||
// exportEvaluateRate()
|
||||
//
|
||||
// exportBigdataCityPrice(queryDate)
|
||||
// exportBigdataDistPrice(queryDate)
|
||||
exportBigdataCommPrice(queryDate)
|
||||
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
|
||||
// exportCommunityBasePrice(lastMonth)
|
||||
@@ -75,25 +75,25 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
// exportBigdataModelingBase(lastMonth)
|
||||
|
||||
//2月8号 临时导出苏州一年的数据
|
||||
exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
|
||||
}
|
||||
// exportPgBigdataForsale(houseStartDate,DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
|
||||
// }
|
||||
//
|
||||
// if(curDate.toString().endsWith("28")){
|
||||
// exportPgBigdataSupplierStatistics(curMonth)
|
||||
// //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
|
||||
// def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
//// exportThirdPartyPrice2(thirdPartyMonth, new_dists)
|
||||
// exportPgThirdPartyPrice2(thirdPartyMonth, new_dists)
|
||||
// exportPgCommunityBasePrice(curMonth)
|
||||
// exportPgCommunityPdescribeGj(curMonth)
|
||||
// exportPgBigdataModelingBase(curMonth)
|
||||
// exportPgGuideStatistics(curMonth)
|
||||
// }
|
||||
|
||||
if(curDate.toString().endsWith("28")){
|
||||
exportPgBigdataSupplierStatistics(curMonth)
|
||||
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
|
||||
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
// exportThirdPartyPrice2(thirdPartyMonth, new_dists)
|
||||
exportPgThirdPartyPrice2(thirdPartyMonth, new_dists)
|
||||
exportPgCommunityBasePrice(curMonth)
|
||||
exportPgCommunityPdescribeGj(curMonth)
|
||||
exportPgBigdataModelingBase(curMonth)
|
||||
exportPgGuideStatistics(curMonth)
|
||||
}
|
||||
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
|
||||
exportOk(curDate)
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
//
|
||||
// exportOk(curDate)
|
||||
}
|
||||
|
||||
//
|
||||
@@ -1184,15 +1184,15 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
|
||||
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
|
||||
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
@@ -1273,9 +1273,9 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
def allRows = []
|
||||
|
||||
+3
-2
@@ -29,10 +29,11 @@ class AddAvgpriceBatch {
|
||||
}
|
||||
|
||||
def test(){
|
||||
def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"])
|
||||
def districts = CommonDatautils.findAllDistricts("福州","district", ["city","region","name", "estate_type2"])
|
||||
// def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"])
|
||||
for(def dist : districts){
|
||||
//发重算均价命令
|
||||
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd'))
|
||||
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-05','yyyy-MM-dd'))
|
||||
}
|
||||
}
|
||||
def run(){
|
||||
|
||||
+1
-1
@@ -164,7 +164,7 @@ class FangjiaClientUtils {
|
||||
// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
|
||||
// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
|
||||
|
||||
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
|
||||
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
|
||||
if (json.status == 0) {
|
||||
return "成功"
|
||||
}else if(json.status == 1){
|
||||
|
||||
+368
-11
@@ -4,14 +4,23 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 写字楼字段清洗入库(自动入库)
|
||||
@@ -19,6 +28,16 @@ import java.util.concurrent.Executors
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
|
||||
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area",
|
||||
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address",
|
||||
"live_date":"live_date", "loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段
|
||||
|
||||
def sourceMap = ["房天下":"房天下","办办网":"办办网","点点租":"点点租","安居客":"安居客","楼盘网":"楼盘网"]
|
||||
|
||||
static DBCollection BLOCKPOLYLINECOLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border2", "mongo.db.fangjia.com")
|
||||
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static main(args) {
|
||||
@@ -27,11 +46,13 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
for(def c : citys){
|
||||
def offices = findAllOffice(c)
|
||||
strandField(offices)
|
||||
}
|
||||
// def citys = CommonDatautils.findAllCity().reverse()
|
||||
// for(def c : citys){
|
||||
// def offices = findAllOffice(c)
|
||||
// strandField(offices)
|
||||
// }
|
||||
|
||||
addBlock()
|
||||
|
||||
}
|
||||
|
||||
@@ -65,15 +86,351 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def findAllOffice(city){
|
||||
def sql = "select * from office where city='${city}'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << d
|
||||
def findEsDatas(c, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
|
||||
list << r
|
||||
})
|
||||
return rets
|
||||
return list
|
||||
}
|
||||
|
||||
|
||||
def matchDist(city, region, name){
|
||||
def dist = CommonDatautils.findDistrict(city, region, name, "office")
|
||||
return dist;
|
||||
}
|
||||
|
||||
def matchApi(esRets){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(esRets.size())
|
||||
for(def i=0;i<esRets.size();i++){
|
||||
final def data = esRets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = matching(data)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def matching(data){
|
||||
def name = data.name
|
||||
if(data.name && data.name.contains("·")){
|
||||
name = data.name.replace("·", "")
|
||||
}
|
||||
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(data.city, data.region, name)
|
||||
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"office")
|
||||
flag = "映射别名匹配"
|
||||
}
|
||||
if(!dist) {
|
||||
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "office")
|
||||
if(rets && rets.size()==1){
|
||||
flag = "别名匹配"
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
|
||||
if(dist){
|
||||
data.flag = flag
|
||||
data.match_id = dist._id.toString()
|
||||
data.match_region = dist.region
|
||||
data.match_name = dist.name
|
||||
data.credit = 0.95
|
||||
}else{
|
||||
def jsonObject = OpenApiClient.addressMatch(data.city, name, "office", false,data.region, null)
|
||||
if(jsonObject!=null && jsonObject.precise==-1){
|
||||
if(data.address && isStandardRoad(data.address)){
|
||||
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "office", false,data.region, name)
|
||||
}else{
|
||||
data.flag = "匹配接口-地址"
|
||||
}
|
||||
}else{
|
||||
data.flag = "匹配接口-名称"
|
||||
}
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
data.match_id = jsonObject.get("id")
|
||||
data.match_region = jsonObject.get("district")
|
||||
data.match_name = jsonObject.get("name")
|
||||
data.credit = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
if(data.match_id){
|
||||
return data
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
def selectRestByCredit(matchRets){
|
||||
def matchRets1 = []
|
||||
for(def r : matchRets){
|
||||
if(r.credit && r.credit>=0.9){
|
||||
matchRets1 << r
|
||||
}
|
||||
}
|
||||
return matchRets1
|
||||
}
|
||||
|
||||
|
||||
def addFieldFromMatch(c, totalCount, analysRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findEsDatas(c, esWebSource)
|
||||
def matchRets1 = matchApi(esRets)
|
||||
def matchRets2 = selectRestByCredit(matchRets1)
|
||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2)
|
||||
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+ DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
if(esRets){
|
||||
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
|
||||
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
|
||||
}
|
||||
def analyRet = []
|
||||
analyRet << seaweedmapSource
|
||||
analyRet << c
|
||||
fields.each{key->
|
||||
analyRet << precentMap.get(key)
|
||||
}
|
||||
analyRet << esRets.size()
|
||||
analyRet << matchRets1.size()
|
||||
analyRet << matchRets2.size()
|
||||
analyRet << pre1
|
||||
analyRet << pre2
|
||||
analysRets << analyRet
|
||||
}
|
||||
return analysRets;
|
||||
}
|
||||
|
||||
def fieldUpdate(total,updateRets){
|
||||
|
||||
def countMap = [:]
|
||||
|
||||
for(def rm : updateRets){
|
||||
def valMap = rm.value
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
if(valMap.containsKey(seaweedfield)) {
|
||||
if(countMap.containsKey(seaweedfield)) {
|
||||
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
|
||||
}else{
|
||||
countMap.put(seaweedfield, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println countMap.toString()
|
||||
def precentMap = [:]
|
||||
fieldsMap.each() {key,skey->
|
||||
if(countMap.containsKey(key)) {
|
||||
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
|
||||
}else {
|
||||
precentMap.put(key, 0)
|
||||
}
|
||||
}
|
||||
return precentMap;
|
||||
}
|
||||
|
||||
|
||||
def processFieldsFromMatch(c,matchRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
fields << "city"
|
||||
fields << "estate_type2"
|
||||
def fieldValueMap = [:]
|
||||
for(def matchDist : matchRets){
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
println c+" "+matchDist.match_id+" 不存在"
|
||||
continue
|
||||
}
|
||||
def newValues = strandField(c, dist, matchDist)
|
||||
if(newValues){
|
||||
println dist._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
}
|
||||
|
||||
return fieldValueMap;
|
||||
}
|
||||
|
||||
def strandField(c,dist, matchDist){
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = dist.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = matchDist.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
if(seaweedfield =="virescence_percent"){
|
||||
newVal = FiledStrand.processVirescencePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="cubage_percent"){
|
||||
newVal = FiledStrand.processCubagePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
|
||||
newVal = FiledStrand.processArea(esVal)
|
||||
if(newVal){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(seaweedfield =="household_count"){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
if(seaweedfield =="loudong_count"){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="car_site_count"){
|
||||
newVal = FiledStrand.processCarSite(esVal)
|
||||
}
|
||||
if(seaweedfield =="car_site"){
|
||||
newVal = FiledStrand.processCarSiteDesc(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="property_life"){
|
||||
newVal = FiledStrand.processPropertyLife(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="alias"){
|
||||
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
|
||||
}
|
||||
|
||||
if(seaweedfield =="address"){
|
||||
newVal = FiledStrand.processAddress(esVal, dist.city)
|
||||
}
|
||||
if(seaweedfield =="block"){
|
||||
newVal = FiledStrand.processBlock(dist.city, dist.region, esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_type"){
|
||||
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
if(seaweedfield == "land_property"){
|
||||
newVal = FiledStrand.processlandProperty(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_structure"){
|
||||
newVal = FiledStrand.processBuildStructure(esVal)
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return newValues;
|
||||
}
|
||||
|
||||
def addBlock(){
|
||||
def rets = findAllBlockRange()
|
||||
|
||||
def offices = CommonDatautils.findAllDistricts("上海", "office", ["city","region","name","block","lat2", "lng2"])
|
||||
for(def i=0;i<=offices.size();i++){
|
||||
final def office = offices[i]
|
||||
if(office && !office.block){
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
|
||||
def lastObj = null
|
||||
for (def r : rets) {
|
||||
def flag = InsidePolygon.isInArea(office.lng2 / 10000000000.0, office.lat2 / 10000000000.0, r.points)
|
||||
if (flag) {
|
||||
lastObj = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lastObj) {
|
||||
def udata = [:]
|
||||
udata.put("block", lastObj.name)
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "office")
|
||||
udata.put("username", "高娟")
|
||||
udata.put("id", office._id.toString())
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println s
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findAllBlockRange(){
|
||||
def rets = []
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = BLOCKPOLYLINECOLL.find(q)
|
||||
for(def d : dbcursor) {
|
||||
def pointList = InsidePolygon.createPoint3(d.border);
|
||||
d.points = pointList
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name, datas){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category","office");
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", "update")
|
||||
obj.put("data", datas)
|
||||
obj.put("user_name", "高娟")
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -26,6 +26,7 @@ public class ApiUserBo implements Serializable {
|
||||
private Date deadTime;
|
||||
private String manager;
|
||||
private String priceType;
|
||||
private Integer isMonitor;
|
||||
|
||||
public String getPriceType() {
|
||||
return priceType;
|
||||
@@ -162,4 +163,12 @@ public class ApiUserBo implements Serializable {
|
||||
public void setManager(String manager) {
|
||||
this.manager = manager;
|
||||
}
|
||||
|
||||
public Integer getIsMonitor() {
|
||||
return isMonitor;
|
||||
}
|
||||
|
||||
public void setIsMonitor(Integer isMonitor) {
|
||||
this.isMonitor = isMonitor;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user