diff --git a/deliverables/k8s/fangjia-bkoff.yaml b/deliverables/k8s/fangjia-bkoff.yaml index 768fd80cd..a577d40bd 100644 --- a/deliverables/k8s/fangjia-bkoff.yaml +++ b/deliverables/k8s/fangjia-bkoff.yaml @@ -21,6 +21,10 @@ spec: project: fangjia-bkoff app: fangjia spec: + hostAliases: + - ip: "192.168.0.38" + hostnames: + - "seaweed.db.fangjia.com" imagePullSecrets: - name: nexus-registry-secret nodeSelector: diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java index 29ee2de45..52b1ecb2d 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/EvaluationAuditController.java @@ -106,7 +106,13 @@ public class EvaluationAuditController { if(eval!=null){ bo.setHouseArea(eval.getHouseArea()); bo.setDistrictName(eval.getDistrictName()); - bo.setAddress(eval.getAddress()); + String address = ""; + if(StringUtils.isNotBlank(eval.getCertificateAddress())){ + address = eval.getCertificateAddress(); + }else{ + address = eval.getAddress(); + } + bo.setAddress(address); bo.setHouseNum(eval.getHouseNum()); bo.setRoomNum(eval.getRoomNum()); bo.setEvaluationDate(eval.getLastUpdated()); @@ -115,7 +121,7 @@ public class EvaluationAuditController { bo.setFloorDesc(eval.getFloorDesc()); bos.add(bo); - List evaluations = fjtEvaluationService.findAllFjtEvaluationByConditions(eval.getCity(), eval.getRegion(), eval.getAddress(), eval.getHouseNum(), eval.getRoomNum()); + List evaluations = fjtEvaluationService.findAllFjtEvaluationByConditions(eval.getCity(), eval.getRegion(), bo.getAddress(), eval.getHouseNum(), eval.getRoomNum()); if(evaluations!=null && evaluations.size()>=2){ List ids = new ArrayList(); for(FjtEvaluation fjtEval : evaluations){ @@ -148,7 +154,13 @@ public class EvaluationAuditController { BeanUtils.copyProperties(audit, bo); bo.setHouseArea(eval.getHouseArea()); bo.setDistrictName(eval.getDistrictName()); - bo.setAddress(eval.getAddress()); + String address = ""; + if(StringUtils.isNotBlank(eval.getCertificateAddress())){ + address = eval.getCertificateAddress(); + }else{ + address = eval.getAddress(); + } + bo.setAddress(address); bo.setHouseNum(eval.getHouseNum()); bo.setRoomNum(eval.getRoomNum()); bo.setEvaluationDate(eval.getLastUpdated()); @@ -157,7 +169,7 @@ public class EvaluationAuditController { bo.setFloorDesc(eval.getFloorDesc()); bos.add(bo); //根据地址,楼栋号判断是否是同一套房 - List evaluations = fjtEvaluationService.findAllFjtEvaluationByConditions(eval.getCity(), eval.getRegion(), eval.getAddress(), eval.getHouseNum(), eval.getRoomNum()); + List evaluations = fjtEvaluationService.findAllFjtEvaluationByConditions(eval.getCity(), eval.getRegion(), bo.getAddress(), eval.getHouseNum(), eval.getRoomNum()); if(evaluations!=null && evaluations.size()>=2){ List ids = new ArrayList(); for(FjtEvaluation fjtEval : evaluations){ diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/OperationLogController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/OperationLogController.java index 0c786ceec..e9d7794ba 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/OperationLogController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/OperationLogController.java @@ -6,8 +6,6 @@ import com.fangjia.core.util.DateUtil; import com.fangjia.dal.mongo.model.OperationLog; import com.fangjia.service.mongo.bkoff.OperationLogService; import org.sothis.core.util.Pager; - -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -28,7 +26,6 @@ public class OperationLogController{ put( "block" , "板块" ); put( "address" , "地址" ); put( "loop_line_info" , "环线" ); - put( "loop_line_info" , "环线" ); put( "estate_type2" , "物业类型" ); put( "street_community" , "街道社区" ); put( "household_count" , "户数" ); @@ -58,7 +55,6 @@ public class OperationLogController{ put( "house_height" , "楼高" ); put( "lease_term","租赁年限"); put( "payment_pattern","付款方式"); - put( "rent_price","出租价格"); put( "sell_price","出售价格"); put( "floor","建筑楼层"); put( "floor_info","楼层状况"); @@ -66,7 +62,6 @@ public class OperationLogController{ put( "characteristic","项目特色"); put( "bay_area","开间面积"); put( "finish_building_date","竣工时间"); - put( "live_date","开业时间"); put( "divisible","是否可分割"); put( "unified_manage","是否统一管理"); put( "metro_info","地铁沿线"); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/RentPriceUpdateController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/RentPriceUpdateController.java new file mode 100644 index 000000000..7964e3813 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/RentPriceUpdateController.java @@ -0,0 +1,1166 @@ +package com.fangjia.bkoff.controllers; + +import com.fangjia.bkoff.model.AvgPriceUpdateModel; +import com.fangjia.bkoff.model.GuidingAvgPriceUpdateModel; +import com.fangjia.bkoff.model.JsonModel; +import com.fangjia.bkoff.model.RentPriceUpdateModel; +import com.fangjia.core.RegionData; +import com.fangjia.core.util.AnalyticsUtil; +import com.fangjia.core.util.DataUtil; +import com.fangjia.core.util.DateUtil; +import com.fangjia.core.util.RandomUtil; +import com.fangjia.dal.common.DistrictKey; +import com.fangjia.dal.common.ServerAreaData; +import com.fangjia.dal.mongo.dao.*; +import com.fangjia.dal.mongo.fangjia.model.ServerArea; +import com.fangjia.dal.mongo.model.DistrictStat; +import com.fangjia.dal.mongo.model.RegionStat; +import com.fangjia.dal.mongo.model.SqmetreRent; +import com.fangjia.dal.mongo.model.bkoff.*; +import com.fangjia.dal.seawater.model.Seawater; +import com.fangjia.dal.seaweed.model.District; +import com.fangjia.service.bo.AvgUpdateBo; +import com.fangjia.service.bo.DistrictStatBo; +import com.fangjia.service.datastore.DistrictService; +import com.fangjia.service.datastore.RegionStatService; +import com.fangjia.service.district.DistrictStatService; +import com.fangjia.service.district.SqmetreRentService; +import com.fangjia.service.mongo.bkoff.*; +import com.fangjia.service.mongo.fangjia.DealPriceService; +import com.fangjia.service.mongo.fangjia.SeawaterService; +import com.ibm.icu.text.MessageFormat; +import com.mongodb.BasicDBObject; +import com.mongodb.DBObject; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.time.DateUtils; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.client.solrj.util.ClientUtils; +import org.bson.types.ObjectId; +import org.sothis.core.util.Pager; +import org.sothis.web.mvc.annotation.Param; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpSession; +import java.util.*; + +/** + * 负责租金更新任务的管理工作 + */ +public class RentPriceUpdateController { + + private RegionStatService regionStatService; + private CalcCommandService calcCommandService; + private DistrictStatService districtStatService; + private SqmetreRentService sqmetreRentService; + private CalcDistrictStatDao calcDistrictStatDao; + private AvgBaseDao avgBaseDao; + private DistrictService districtService; + private AverageStatisticsDao averageStatisticsDao; + private DealPriceService dealPriceService; + private GuidingPriceService guidingPriceService; + private CalcConfigService calcConfigService; + private AveragerQualityReportService averagerQualityReportService; + private SampleQualityReportService sampleQualityReportService; + private ComparePriceDao comparePriceDao; + private ComparePriceCityDao comparePriceCityDao; + private SeawaterService seawaterService; + private static final List IPS = Arrays.asList("fangjia-priceanalysis-0"); + + private static final List HOT_CITIES = Arrays.asList("上海", "北京", "广州", "深圳", "天津", "重庆", "成都", "杭州", "南京", "长沙", + "武汉", "西安", "沈阳"); + + public Object listQRAction(AvgPriceUpdateModel model) { + if(StringUtils.isBlank(model.getCityDisplay())) { + model.setCityDisplay("全国"); + } + + if(model.getSdateStart() <= 0 || model.getSdateEnd() <= 0) { + Date sunday = DateUtil.getThisSunday(new Date()); + model.setSdateStart(Integer.parseInt(DateUtil.formatToTightYYYYMMDD(DateUtil.getBeforeDay(sunday, 13 * 7)))); //最近一季度 + model.setSdateEnd(Integer.parseInt(DateUtil.formatToTightYYYYMMDD(sunday))); + } + + if(StringUtils.isBlank(model.getStype())) { + model.setStype("1"); + } + + if(StringUtils.isBlank(model.getVts())) { + model.setVts("1"); + } + + return model; + } + + public Object listSQRAction(AvgPriceUpdateModel model) { + if(StringUtils.isBlank(model.getCityDisplay())) { + model.setCityDisplay("全国"); + } + + if(model.getSdateStart() <= 0 || model.getSdateEnd() <= 0) { + Date sunday = DateUtil.getThisSunday(new Date()); + model.setSdateStart(Integer.parseInt(DateUtil.formatToTightYYYYMMDD(DateUtil.getBeforeDay(sunday, 13 * 7)))); //最近一季度 + model.setSdateEnd(Integer.parseInt(DateUtil.formatToTightYYYYMMDD(sunday))); + } + + if(StringUtils.isBlank(model.getStype())) { + model.setStype("1"); + } + + if(StringUtils.isBlank(model.getVts())) { + model.setVts("1"); + } + return model; + } + + public Object loadQRDataAction(AvgPriceUpdateModel model) { + if(StringUtils.isBlank(model.getCityDisplay())) { + model.setCityDisplay("全国"); + } + + if(model.getSdateStart() <= 0 || model.getSdateEnd() <= 0) { + Date sunday = DateUtil.getThisSunday(new Date()); + model.setSdateStart(Integer.parseInt(DateUtil.formatToTightYYYYMMDD(DateUtil.getBeforeDay(sunday, 13)))); //最近一季度 + model.setSdateEnd(Integer.parseInt(DateUtil.formatToTightYYYYMMDD(sunday))); + } + + if(StringUtils.isBlank(model.getStype())) { + model.setStype("1"); + } + + JSONArray series = createQualitySeries(model.getCityDisplay(), model.getStype(), model.getVts(), DateUtil.parseDate(String.valueOf(model.getSdateStart()), "yyyyMMdd"), DateUtil.parseDate(String.valueOf(model.getSdateEnd()), "yyyyMMdd")); + model.setSeries(series.toString().replace("\"name\"", "name").replace("\"data\"", "data").replace("\"visible\"", "visible")); + return model; + } + + public Object loadSQRDataAction(AvgPriceUpdateModel model) { + if(StringUtils.isBlank(model.getCityDisplay())) { + model.setCityDisplay("全国"); + } + + if(model.getSdateStart() <= 0 || model.getSdateEnd() <= 0) { + Date sunday = DateUtil.getThisSunday(new Date()); + model.setSdateStart(Integer.parseInt(DateUtil.formatToTightYYYYMMDD(DateUtil.getBeforeDay(sunday, 13)))); //最近一季度 + model.setSdateEnd(Integer.parseInt(DateUtil.formatToTightYYYYMMDD(sunday))); + } + + if(StringUtils.isBlank(model.getStype())) { + model.setStype("1"); + } + + JSONArray series = createSampleSeries(model.getCityDisplay(), model.getStype(), model.getVts(), DateUtil.parseDate(String.valueOf(model.getSdateStart()), "yyyyMMdd"), DateUtil.parseDate(String.valueOf(model.getSdateEnd()), "yyyyMMdd")); + model.setSeries(series.toString().replace("\"name\"", "name").replace("\"data\"", "data").replace("\"visible\"", "visible")); + return model; + } + + private JSONArray createSampleSeries(String city, String stype, String vts, Date startDate, Date endDate) { + List sampleQualityReports = sampleQualityReportService.findQualityReports(city, Integer.parseInt(DateUtil.formatToTightYYYYMMDD(startDate)), + Integer.parseInt(DateUtil.formatToTightYYYYMMDD(endDate))); + + JSONArray ret = new JSONArray(); + JSONObject jo1 = new JSONObject(), jo2 = new JSONObject(), jo3 = new JSONObject(),jo4 = new JSONObject(), + jo5 = new JSONObject(), jo6 = new JSONObject(), jo7 = new JSONObject(), jo8 = new JSONObject(); + JSONArray pt1 = new JSONArray(), pt2 = new JSONArray(), pt3 = new JSONArray(), pt4 = new JSONArray(), + pt5 = new JSONArray(), pt6 = new JSONArray(), pt7 = new JSONArray(), pt8 = new JSONArray(); + + if("1".equals(stype)) { + jo1.put("name", "一个月<10小区量"); + jo2.put("name", "一个月<30小区量"); + jo3.put("name", "一个月<100小区量"); + for(SampleQualityReport sqr : sampleQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(sqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + sqr.getMonthCount1() + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + sqr.getMonthCount2() + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + sqr.getMonthCount3() + "]")); + } + }else if("2".equals(stype)) { + jo1.put("name", "一个月<10比例"); + jo2.put("name", "一个月<30比例"); + jo3.put("name", "一个月<100比例"); + for(SampleQualityReport sqr : sampleQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(sqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getMonthCount1() / (double)sqr.getDistrictCount() * 1000.0) / 10.0 + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getMonthCount2() / (double)sqr.getDistrictCount() * 1000.0) / 10.0 + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getMonthCount3() / (double)sqr.getDistrictCount() * 1000.0) / 10.0+ "]")); + } + }else if("3".equals(stype)) { + jo1.put("name", "季度<10小区量"); + jo2.put("name", "季度<30小区量"); + jo3.put("name", "季度<100小区量"); + for(SampleQualityReport sqr : sampleQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(sqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + sqr.getQuarCount1() + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + sqr.getQuarCount2() + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + sqr.getQuarCount3() + "]")); + } + }else if("4".equals(stype)) { + jo1.put("name", "季度<10比例"); + jo2.put("name", "季度<30比例"); + jo3.put("name", "季度<100比例"); + for(SampleQualityReport sqr : sampleQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(sqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getQuarCount1() / (double)sqr.getDistrictCount() * 1000.0) / 10.0 + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getQuarCount2() / (double)sqr.getDistrictCount() * 1000.0) / 10.0 + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getQuarCount3() / (double)sqr.getDistrictCount() * 1000.0) / 10.0+ "]")); + } + }else if("5".equals(stype)) { + jo1.put("name", "半年<10小区量"); + jo2.put("name", "半年<30小区量"); + jo3.put("name", "半年<100小区量"); + for(SampleQualityReport sqr : sampleQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(sqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + sqr.getHalfCount1() + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + sqr.getHalfCount2() + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + sqr.getHalfCount3() + "]")); + } + }else if("6".equals(stype)) { + jo1.put("name", "半年<10比例"); + jo2.put("name", "半年<30比例"); + jo3.put("name", "半年<100比例"); + for(SampleQualityReport sqr : sampleQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(sqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getHalfCount1() / (double)sqr.getDistrictCount() * 1000.0) / 10.0 + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getHalfCount2() / (double)sqr.getDistrictCount() * 1000.0) / 10.0 + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getHalfCount3() / (double)sqr.getDistrictCount() * 1000.0) / 10.0+ "]")); + } + }else if("7".equals(stype)) { + jo1.put("name", "一年<10小区量"); + jo2.put("name", "一年<30小区量"); + jo3.put("name", "一年<100小区量"); + for(SampleQualityReport sqr : sampleQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(sqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + sqr.getYearCount1() + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + sqr.getYearCount2() + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + sqr.getYearCount3() + "]")); + } + }else if("8".equals(stype)) { + jo1.put("name", "一年<10比例"); + jo2.put("name", "一年<30比例"); + jo3.put("name", "一年<100比例"); + for(SampleQualityReport sqr : sampleQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(sqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getYearCount1() / (double)sqr.getDistrictCount() * 1000.0) / 10.0 + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getYearCount2() / (double)sqr.getDistrictCount() * 1000.0) / 10.0 + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + (int)((double)sqr.getYearCount3() / (double)sqr.getDistrictCount() * 1000.0) / 10.0+ "]")); + } + } + + + jo1.put("color", "#79434D"); + jo2.put("color", "#BBBD81"); + jo3.put("color", "#C877C8"); + + jo1.put("data", pt1); + jo2.put("data", pt2); + jo3.put("data", pt3); + + ret.add(jo1); + ret.add(jo2); + ret.add(jo3); + + return ret; + } + + private JSONArray createQualitySeries(String city, String stype, String vts, Date startDate, Date endDate) { + List averagerQualityReports = averagerQualityReportService.findQualityReports(city, Integer.parseInt(DateUtil.formatToTightYYYYMMDD(startDate)), + Integer.parseInt(DateUtil.formatToTightYYYYMMDD(endDate))); + JSONArray ret = new JSONArray(); + JSONObject jo1 = new JSONObject(), jo2 = new JSONObject(), jo3 = new JSONObject(),jo4 = new JSONObject(), + jo5 = new JSONObject(), jo6 = new JSONObject(), jo7 = new JSONObject(), jo8 = new JSONObject(); + JSONArray pt1 = new JSONArray(), pt2 = new JSONArray(), pt3 = new JSONArray(), pt4 = new JSONArray(), + pt5 = new JSONArray(), pt6 = new JSONArray(), pt7 = new JSONArray(), pt8 = new JSONArray(); + if("1".equals(stype)) { + jo1.put("name", "小区量(全部)"); + jo2.put("name", "小区量(无法计算)"); + jo3.put("name", "小区量(指导价)"); + jo4.put("name", "小区量(自身样本)"); + jo5.put("name", "小区量(第三方)"); + jo6.put("name", "小区量(成交数据)"); + jo7.put("name", "小区量(周边小区)"); + jo8.put("name", "小区量(推算)"); + for(AveragerQualityReport aqr : averagerQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(aqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + aqr.getDistrictCount() + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + aqr.getNoneCount() + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + aqr.getGuidingCount() + "]")); + pt4.add(JSONArray.fromObject("[" + tms + "," + aqr.getOwnCount() + "]")); + pt5.add(JSONArray.fromObject("[" + tms + "," + aqr.getThirdCount() + "]")); + pt6.add(JSONArray.fromObject("[" + tms + "," + aqr.getDealCount() + "]")); + pt7.add(JSONArray.fromObject("[" + tms + "," + aqr.getAroundCount() + "]")); + pt8.add(JSONArray.fromObject("[" + tms + "," + aqr.getDeriCount() + "]")); + } + }else if("2".equals(stype)) { + jo1.put("name", "平均可信度(全部)"); + jo2.put("name", "平均可信度(无法计算)"); + jo3.put("name", "平均可信度(指导价)"); + jo4.put("name", "平均可信度(自身样本)"); + jo5.put("name", "平均可信度(第三方)"); + jo6.put("name", "平均可信度(成交数据)"); + jo7.put("name", "平均可信度(周边小区)"); + jo8.put("name", "平均可信度(推算)"); + for(AveragerQualityReport aqr : averagerQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(aqr.getWeekend()), "yyyyMMdd").getTime(); + pt1.add(JSONArray.fromObject("[" + tms + "," + (int)(aqr.getAvgCredit() * 100000) + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + (int)(aqr.getNoneCredit() * 100000) + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + (int)(aqr.getGuidingCredit() * 100000) + "]")); + pt4.add(JSONArray.fromObject("[" + tms + "," + (int)(aqr.getOwnCredit() * 100000) + "]")); + pt5.add(JSONArray.fromObject("[" + tms + "," + (int)(aqr.getThirdCredit() * 100000) + "]")); + pt6.add(JSONArray.fromObject("[" + tms + "," + (int)(aqr.getDealCredit() * 100000) + "]")); + pt7.add(JSONArray.fromObject("[" + tms + "," + (int)(aqr.getAroundCredit() * 100000) + "]")); + pt8.add(JSONArray.fromObject("[" + tms + "," + (int)(aqr.getDeriCredit() * 100000) + "]")); + } + }else if("3".equals(stype)) { + jo1.put("name", "测试案例数量(全部)"); + jo2.put("name", "测试案例数量(无法计算)"); + jo3.put("name", "测试案例数量(指导价)"); + jo4.put("name", "测试案例数量(自身样本)"); + jo5.put("name", "测试案例数量(第三方)"); + jo6.put("name", "测试案例数量(成交数据)"); + jo7.put("name", "测试案例数量(周边小区)"); + jo8.put("name", "测试案例数量(推算)"); + for(AveragerQualityReport aqr : averagerQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(aqr.getWeekend()), "yyyyMMdd").getTime(); + int allTest = aqr.getNoneTest() + aqr.getGuidingTest() + aqr.getOwnTest() + aqr.getThirdTest() + aqr.getDealTest() + aqr.getAroundTest() + aqr.getDeriTest(); + pt1.add(JSONArray.fromObject("[" + tms + "," + allTest + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + aqr.getNoneTest() + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + aqr.getGuidingTest() + "]")); + pt4.add(JSONArray.fromObject("[" + tms + "," + aqr.getOwnTest() + "]")); + pt5.add(JSONArray.fromObject("[" + tms + "," + aqr.getThirdTest() + "]")); + pt6.add(JSONArray.fromObject("[" + tms + "," + aqr.getDealTest() + "]")); + pt7.add(JSONArray.fromObject("[" + tms + "," + aqr.getAroundTest() + "]")); + pt8.add(JSONArray.fromObject("[" + tms + "," + aqr.getDeriTest() + "]")); + } + }else if("4".equals(stype)) { + jo1.put("name", "案例差比<10%量(全部)"); + jo2.put("name", "案例差比<10%量(无法计算)"); + jo3.put("name", "案例差比<10%量(指导价)"); + jo4.put("name", "案例差比<10%量(自身样本)"); + jo5.put("name", "案例差比<10%量(第三方)"); + jo6.put("name", "案例差比<10%量(成交数据)"); + jo7.put("name", "案例差比<10%量(周边小区)"); + jo8.put("name", "案例差比<10%量(推算)"); + for(AveragerQualityReport aqr : averagerQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(aqr.getWeekend()), "yyyyMMdd").getTime(); + int allTest = aqr.getNoneTest() + aqr.getGuidingTest() + aqr.getOwnTest() + aqr.getThirdTest() + aqr.getDealTest() + aqr.getAroundTest() + aqr.getDeriTest(); + int allAccu1 = aqr.getNoneAccu1() + aqr.getGuidingAccu1() + aqr.getOwnAccu1() + aqr.getThirdAccu1() + aqr.getDealAccu1() + aqr.getAroundAccu1() + aqr.getDeriAccu1(); + pt1.add(JSONArray.fromObject("[" + tms + "," + (int)((double)allAccu1 / (double)allTest * 10000) / 100.0 + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getNoneAccu1() / (double)allTest * 10000) / 100.0 + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getGuidingAccu1() / (double)allTest * 10000) / 100.0 + "]")); + pt4.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getOwnAccu1() / (double)allTest * 10000) / 100.0 + "]")); + pt5.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getThirdAccu1() / (double)allTest * 10000) / 100.0 + "]")); + pt6.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getDealAccu1() / (double)allTest * 10000) / 100.0 + "]")); + pt7.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getAroundAccu1() / (double)allTest * 10000) / 100.0 + "]")); + pt8.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getDeriAccu1() / (double)allTest * 10000) / 100.0 + "]")); + } + }else if("5".equals(stype)) { + jo1.put("name", "案例差比<20%量(全部)"); + jo2.put("name", "案例差比<20%量(无法计算)"); + jo3.put("name", "案例差比<20%量(指导价)"); + jo4.put("name", "案例差比<20%量(自身样本)"); + jo5.put("name", "案例差比<20%量(第三方)"); + jo6.put("name", "案例差比<20%量(成交数据)"); + jo7.put("name", "案例差比<20%量(周边小区)"); + jo8.put("name", "案例差比<20%量(推算)"); + for(AveragerQualityReport aqr : averagerQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(aqr.getWeekend()), "yyyyMMdd").getTime(); + int allTest = aqr.getNoneTest() + aqr.getGuidingTest() + aqr.getOwnTest() + aqr.getThirdTest() + aqr.getDealTest() + aqr.getAroundTest() + aqr.getDeriTest(); + int allAccu2 = aqr.getNoneAccu2() + aqr.getGuidingAccu2() + aqr.getOwnAccu2() + aqr.getThirdAccu2() + aqr.getDealAccu2() + aqr.getAroundAccu2() + aqr.getDeriAccu2(); + pt1.add(JSONArray.fromObject("[" + tms + "," + (int)((double)allAccu2 / (double)allTest * 10000) / 100.0 + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getNoneAccu2() / (double)allTest * 10000) / 100.0 + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getGuidingAccu2() / (double)allTest * 10000) / 100.0 + "]")); + pt4.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getOwnAccu2() / (double)allTest * 10000) / 100.0 + "]")); + pt5.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getThirdAccu2() / (double)allTest * 10000) / 100.0 + "]")); + pt6.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getDealAccu2() / (double)allTest * 10000) / 100.0 + "]")); + pt7.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getAroundAccu2() / (double)allTest * 10000) / 100.0 + "]")); + pt8.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getDeriAccu2() / (double)allTest * 10000) / 100.0 + "]")); + } + }else if("6".equals(stype)) { + jo1.put("name", "案例差比<30%量(全部)"); + jo2.put("name", "案例差比<30%量(无法计算)"); + jo3.put("name", "案例差比<30%量(指导价)"); + jo4.put("name", "案例差比<30%量(自身样本)"); + jo5.put("name", "案例差比<30%量(第三方)"); + jo6.put("name", "案例差比<30%量(成交数据)"); + jo7.put("name", "案例差比<30%量(周边小区)"); + jo8.put("name", "案例差比<30%量(推算)"); + for(AveragerQualityReport aqr : averagerQualityReports) { + long tms = DateUtil.parseDate(String.valueOf(aqr.getWeekend()), "yyyyMMdd").getTime(); + int allTest = aqr.getNoneTest() + aqr.getGuidingTest() + aqr.getOwnTest() + aqr.getThirdTest() + aqr.getDealTest() + aqr.getAroundTest() + aqr.getDeriTest(); + int allAccu3 = aqr.getNoneAccu3() + aqr.getGuidingAccu3() + aqr.getOwnAccu3() + aqr.getThirdAccu3() + aqr.getDealAccu3() + aqr.getAroundAccu3() + aqr.getDeriAccu3(); + pt1.add(JSONArray.fromObject("[" + tms + "," + (int)((double)allAccu3 / (double)allTest * 10000) / 100.0 + "]")); + pt2.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getNoneAccu3() / (double)allTest * 10000) / 100.0 + "]")); + pt3.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getGuidingAccu3() / (double)allTest * 10000) / 100.0 + "]")); + pt4.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getOwnAccu3() / (double)allTest * 10000) / 100.0 + "]")); + pt5.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getThirdAccu3() / (double)allTest * 10000) / 100.0 + "]")); + pt6.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getDealAccu3() / (double)allTest * 10000) / 100.0 + "]")); + pt7.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getAroundAccu3() / (double)allTest * 10000) / 100.0 + "]")); + pt8.add(JSONArray.fromObject("[" + tms + "," + (int)((double)aqr.getDeriAccu3() / (double)allTest * 10000) / 100.0 + "]")); + } + } + + jo1.put("color", "#79434D"); + jo2.put("color", "#BBBD81"); + jo3.put("color", "#C877C8"); + jo4.put("color", "#3A696F"); + jo5.put("color", "#6BB7C5"); + jo6.put("color", "#729827"); + jo7.put("color", "#DD9F19"); + jo8.put("color", "#CAB0A2"); + + jo1.put("data", pt1); + jo2.put("data", pt2); + jo3.put("data", pt3); + jo4.put("data", pt4); + jo5.put("data", pt5); + jo6.put("data", pt6); + jo7.put("data", pt7); + jo8.put("data", pt8); + + List vtss = Arrays.asList(vts.split("-")); + if(vtss.contains("1")) { + ret.add(jo1); + } + if(vtss.contains("2")) { + ret.add(jo2); + } + if(vtss.contains("3")) { + ret.add(jo3); + } + if(vtss.contains("4")) { + ret.add(jo4); + } + if(vtss.contains("5")) { + ret.add(jo5); + } + if(vtss.contains("6")) { + ret.add(jo6); + } + if(vtss.contains("7")) { + ret.add(jo7); + } + if(vtss.contains("8")) { + ret.add(jo8); + } + return ret; + } + + public Object doCreatTaskAction(AvgPriceUpdateModel model) { + // model.setAvgUpdateBos(regionStatService.findAvgPriceData(null,null,null)); + + Map> avgUpdateBoMap = new LinkedHashMap>(); + List avgUpdateBoList = regionStatService.findAvgPriceData(null, null, null); + + Map regionStatMap = new LinkedHashMap(); + for (AvgUpdateBo bo : avgUpdateBoList) { + regionStatMap.put(bo.getCity(), bo); + } + + List serverAreas = ServerAreaData.getServerAreaList(); + Collections.sort(serverAreas, new Comparator() { + @Override + public int compare(ServerArea o1, ServerArea o2) { + return ((Integer)o1.getCityGrade()).compareTo(o2.getCityGrade()); + } + }); + + for (ServerArea serverArea : serverAreas) { + String w = String.valueOf(DataUtil.hanyuJianpin(serverArea.getDisplay()).toCharArray()[0]).toUpperCase(); + if (HOT_CITIES.contains(serverArea.getDisplay())) { + w = "HOT"; + } + + AvgUpdateBo bo = regionStatMap.get(serverArea.getDisplay()); + if(bo == null) { + bo = new AvgUpdateBo(); + bo.setCity(serverArea.getDisplay()); + } + if (avgUpdateBoMap.containsKey(w)) { + List list = avgUpdateBoMap.get(w); + list.add(bo); + avgUpdateBoMap.put(w, list); + } else { + List list = new ArrayList(); + list.add(bo); + avgUpdateBoMap.put(w, list); + } + } + + model.setAvgUpdateBoMap(avgUpdateBoMap); + return model; + } + + public Object reComputeAction(@Param(name = "data") String data, HttpServletRequest request) { + + JsonModel model = new JsonModel(); + Map json = new HashMap(); + model.setJsonModel(json); + JSONObject jsonObject = JSONObject.fromObject(data); + String calcRange = null; + String _startDate = null; + String _endDate = null; + if(jsonObject.containsKey("calcRange")) { + calcRange = jsonObject.getString("calcRange"); + } + if(jsonObject.containsKey("startDate")) { + _startDate = jsonObject.getString("startDate"); + } + if(jsonObject.containsKey("endDate")) { + _endDate = jsonObject.getString("endDate"); + } + String realName = (String) request.getSession().getAttribute("realName"); + String ip = AnalyticsUtil.getIpAddress(request); + String remark = MessageFormat.format("userName:{0},ip:{1}", new String[] { realName, ip }); + + JSONArray commands = jsonObject.getJSONArray("commands"); + + Date startDate = null; + Date endDate = null; + + List cmds = new ArrayList(); + + if(StringUtils.isNotBlank(calcRange)) { + Date currentDate = new Date(); + endDate = new Date(); + if("week".equals(calcRange)) { + startDate = DateUtil.getBeforeDay(currentDate, 7); + }else if("month".equals(calcRange)) { + startDate = DateUtil.getBeforeDay(currentDate, 30); + }else if("quarter".equals(calcRange)) { + startDate = DateUtil.getBeforeDay(currentDate, 30 * 3); + }else if("year".equals(calcRange)) { + startDate = DateUtil.getBeforeDay(currentDate, 365); + }else if("all".equals(calcRange)) { +// for (int i = 0, len = commands.size(); i < len; i++) { +// JSONObject command = (JSONObject) commands.get(i); +// cmds.add(commandToCalcCommand(command, null, remark)); +// } + // startDate取createYear或buildYear最小值 + JSONObject command = (JSONObject) commands.get(0); + District district = districtService.findByName(command.getString("city"), command.getString("region"), command.getString("name"), "district"); + Integer createYear = null; + if (StringUtils.isNotBlank(district.getCreateYear())) { + createYear = Integer.parseInt(district.getCreateYear().substring(0, 4)); + } + if (StringUtils.isNotBlank(district.getFinishBuildingDate())) { + Integer finishYear = Integer.parseInt(district.getFinishBuildingDate().substring(0, 4)); + if ((createYear != null && createYear > finishYear) || createYear == null) { + createYear = finishYear; + } + } + if (createYear == null) { + // 重算一年 + createYear = Integer.valueOf(DateUtil.formatByPattern(DateUtils.addYears(new Date(), -1), "yyyy")); + } else if (Integer.valueOf(DateUtil.formatByPattern(new Date(), "yyyy")) - createYear > 2) { + // 对于大于两年取两年 + createYear = Integer.valueOf(DateUtil.formatByPattern(DateUtils.addYears(new Date(), -2), "yyyy")); + } + startDate = DateUtil.parseDate(String.valueOf(createYear), "yyyy"); + } + }else if (StringUtils.isNotBlank(_startDate) || StringUtils.isNotBlank(_endDate)) { + if (StringUtils.isNotBlank(_startDate)) { + startDate = DateUtil.parseDate(_startDate, "yyyyMMdd"); + } else { + startDate = DateUtil.parseDate("20070101", "yyyyMMdd"); + } + if (StringUtils.isNotBlank(_endDate)) { + endDate = DateUtil.parseDate(_endDate, "yyyyMMdd"); + } else { + endDate = new Date(); + } + }else { + for (int i = 0, len = commands.size(); i < len; i++) { + JSONObject command = (JSONObject) commands.get(i); + cmds.add(commandToCalcCommand(command, null, remark)); + } + } + + if(startDate != null && endDate != null) { + List months = DateUtil.getLastDayBetweenDates(startDate, endDate); + if (months.isEmpty()) { + json.put("msg", "你选择的日期不包含周末,请重新选择"); + return model; + } + for (Date month : months) { + for (int i = 0, len = commands.size(); i < len; i++) { + JSONObject command = (JSONObject) commands.get(i); + CalcCommand cmd = commandToCalcCommand(command, month, remark); + if (cmd != null) { + cmds.add(cmd); + } + + } + } + } + + calcCommandService.save(cmds); + json.put("msg", "任务保存成功,共保存" + cmds.size() + "条任务"); + return model; + } + + private CalcCommand commandToCalcCommand(JSONObject command, Date weekend, String remark) { + CalcCommand cmd = new CalcCommand(); + cmd.setCity(command.getString("city")); + if (command.containsKey("region")) { + cmd.setRegion(command.getString("region")); + } else { + cmd.setRegion(null); + } + if (command.containsKey("block")) { + cmd.setBlock(command.getString("block")); + } else { + cmd.setBlock(null); + } + if (command.containsKey("name")) { + cmd.setName(command.getString("name")); + } else { + cmd.setName(null); + } + if (command.containsKey("estateType")) { + cmd.setEstateType(command.getString("estateType")); + } else { + cmd.setEstateType(null); + } + if (weekend == null) { + cmd.setCalcType(1); + } else { + if (command.containsKey("startDate") && StringUtils.isNotBlank(command.getString("startDate"))) { + Date startDate = DateUtil.parseDate(command.getString("startDate"), "yyyy-MM-dd");// 均价开始时间 + if (weekend.before(startDate)) { + return null; + } + } + cmd.setCalcType(0); + } + cmd.setStatus(0); + cmd.setPriority(2); + cmd.setCalcDate(weekend); + cmd.setRemark(remark); + cmd.setCategory("square_rent"); + cmd.setIp(IPS.get(RandomUtil.nextInt(IPS.size()))); + cmd.setCreationDate(new Date()); + return cmd; + } + + public Object deleteAction(AvgPriceUpdateModel model) { + JsonModel ret = new JsonModel(); + if(model.getId() != null) { + regionStatService.deleteById(model.getId()); + } + return ret; + } + + public Object loadAvgPriceDataAction(AvgPriceUpdateModel model) { + JsonModel json = new JsonModel(); + json.setJsonModel(regionStatService.findAvgPriceData(model.getCityName(), model.getRegion(), model.getBlock())); + return json; + } + + public Object loadAllAvgPriceAction(AvgPriceUpdateModel model) { + JsonModel json = new JsonModel(); + Pager pager = model.getPager(); + if (pager == null) { + pager = model.getPager(15); + } + Map data = new HashMap(); + data.put("data", + regionStatService.findAllAvgPriceData(model.getCityName(), model.getRegion(), model.getBlock(), model.getPager(), false)); + data.put("pager", pager); + json.setJsonModel(data); + return json; + } + + public Object updateAvgPriceAction(AvgPriceUpdateModel model) { + JsonModel jsonModel = new JsonModel(); + RegionStat regionStat = regionStatService.findById(model.getId()); + if (regionStat != null) { + regionStat.setAvgPrice(model.getAvgPrice()); + regionStatService.save(regionStat); + } + return jsonModel; + } + + + public Object addAvgPriceAction(AvgPriceUpdateModel model) { + JsonModel jsonModel = new JsonModel(); + Map resMap = new HashMap(); + if(model.getAvgPrice() <= 0) { + resMap.put("msg", "价格错误!"); + jsonModel.setJsonModel(resMap); + return jsonModel; + } + + if(model.getWeekend() == null || model.getWeekend().getDay() > 0) { + resMap.put("msg", "日期错误!"); + jsonModel.setJsonModel(resMap); + return jsonModel; + } + + String city = null; + String region = null; + String block = null; + String cat = null; + if(StringUtils.isNotBlank(model.getCityDisplay()) && StringUtils.isNotBlank(model.getRegion()) && StringUtils.isNotBlank(model.getBlock())) { + city = model.getCityDisplay(); + region = model.getRegion(); + block = model.getBlock(); + cat = "板块"; + }else if(StringUtils.isNotBlank(model.getCityDisplay()) && StringUtils.isNotBlank(model.getRegion()) && StringUtils.isBlank(model.getBlock())) { + city = model.getCityDisplay(); + region = model.getRegion(); + cat = "区域"; + }else if(StringUtils.isNotBlank(model.getCityDisplay()) && StringUtils.isBlank(model.getRegion()) && StringUtils.isBlank(model.getBlock())) { + city = model.getCityDisplay(); + cat = "城市"; + } + + if(city == null || region == null || block == null) { + resMap.put("msg", "城市区域板块错误!"); + jsonModel.setJsonModel(resMap); + return jsonModel; + } + + RegionStat regionStat = regionStatService.findRegionStat(city, region, block, model.getWeekend()); + if (regionStat != null) { + resMap.put("msg", "均价数据已存在!"); + jsonModel.setJsonModel(resMap); + return jsonModel; + } + + RegionStat entity = new RegionStat(); + entity.setCity(city); + entity.setRegion(region); + entity.setBlock(block); + entity.setWeekend(model.getWeekend()); + entity.setAvgPrice(model.getAvgPrice()); + entity.setCalcValue(model.getAvgPrice()); + entity.setCoverage(0); + entity.setCredit(0.6); + entity.setCat(cat); + entity.setWeekendMarkup(0); + regionStatService.save(entity); + return jsonModel; + } + + public Object districtListAction(AvgPriceUpdateModel model) { + Pager pager = model.getPager(15); + String sort = model.getSort(); + if (StringUtils.isNotBlank(sort)) { + String[] sortArr = sort.split("_"); + if (sortArr.length == 2) { + model.setAvgUpdateBos(districtStatService.findAvgPriceData(model.getCityDisplay(), model.getRegion(), + model.getBlock(), model.getName(), model.getMinCredit(), model.getMaxCredit(), model.getValueType(), + pager,sortArr[0], sortArr[1])); + } + } else { + model.setAvgUpdateBos(districtStatService.findAvgPriceData(model.getCityDisplay(), model.getRegion(), + model.getBlock(), model.getName(), model.getMinCredit(), model.getMaxCredit(), model.getValueType(), pager, + "","")); + } + return model; + } + + public Object loadAllDistrictRentPriceAction(RentPriceUpdateModel model) { + JsonModel json = new JsonModel(); + Map data = new HashMap(); + DistrictKey key = new DistrictKey(model.getCityName(), model.getRegion(), model.getName()); + List bos = sqmetreRentService.findAllAvgPriceByKeys(key, "district"); + data.put("data", bos); + json.setJsonModel(data); + return json; + } + + /**a + * 加载某个小区的房源信息 + * + * @param model + * @return + */ + public Object loadHouseInfoAction(AvgPriceUpdateModel model) { + JsonModel json = new JsonModel(); + Map data = new HashMap(); + District district = this.districtService.findDistrict(new DistrictKey(model.getCityName(), model.getRegion(), model.getName()), false, new String[]{"city", "region", "name", "alias"}); + List districtAlias = null; + if(district != null && district.getAlias() != null) { + districtAlias = district.getAlias(); + } + + List regionAlias = RegionData.getRegionAlias(model.getCityName(), model.getRegion()); + List sells = seawaterService.findSeawatersFromEs("rent", model.getCityName(), model.getRegion(), model.getName(), districtAlias, regionAlias, null,null,null,null,"p_date",false,0,1000,null,"latest_seawater"); + data.put("data", sells); + json.setJsonModel(data); + return json; + + } + + + private long findSeawatersCount(String city, String region, String name, Date startDate, Date endDate) { + District district = this.districtService.findDistrict(new DistrictKey(city, region, name), false, new String[]{"city", "region", "name", "alias"}); + List districtAlias = null; + if(district != null && district.getAlias() != null) { + districtAlias = district.getAlias(); + } + List regionAlias = RegionData.getRegionAlias(city, region); + Integer sd = null; + Integer ed = null; + if(startDate != null) { + sd = Integer.parseInt(DateUtil.formatToTightYYYYMMDD(startDate)); + } + + if(endDate != null) { + ed = Integer.parseInt(DateUtil.formatToTightYYYYMMDD(endDate)); + } + + return seawaterService.findSeawatersCountFromEs("rent", city, region, name, districtAlias, regionAlias, sd, ed, null, null,"seawater"); + } + + private DBObject buildQueryDBObject(District district, Date startDate, Date endDate, boolean isCaseRegion, boolean isUseRegionAlias, boolean isUseAlias) { + Set namesList = new LinkedHashSet(); + namesList.add(district.getName()); + + if (district != null && district.getAlias() != null) { + namesList.addAll(district.getAlias()); + } + + Set regionsList = new LinkedHashSet(); + regionsList.add(district.getRegion()); + //只有进行区域区分,才可以使用区域别名 + if (isCaseRegion && isUseRegionAlias) { + List regionAlias = RegionData.getRegionAlias(district.getCity(), district.getRegion()); + if (regionAlias != null) { + for (String p : regionAlias) { + if (StringUtils.isNotBlank(p)) { + regionsList.add(p); + } + } + } + } + + BooleanQuery srQuery = new BooleanQuery(); + if (isCaseRegion) { + BooleanQuery regionQuery = new BooleanQuery(); + for (String region : regionsList) { + regionQuery.add(new TermQuery(new Term("region", region)), BooleanClause.Occur.SHOULD); + } + srQuery.add(regionQuery, BooleanClause.Occur.MUST); + } + + BooleanQuery nameQuery = new BooleanQuery(); + for (String name : namesList) { + name = name.trim(); + if (StringUtils.isNotBlank(name) && name.length() < 20) { + nameQuery.add(new TermQuery(new Term("district_name", ClientUtils.escapeQueryChars(name))), BooleanClause.Occur.SHOULD); + } + } + + if (StringUtils.isBlank(nameQuery.toString())) { + return null; + } + srQuery.add(nameQuery, BooleanClause.Occur.MUST); + + DBObject q = new BasicDBObject(); + q.put("cat", "rent"); + q.put("city", district.getCity()); + q.put("$sr", new BasicDBObject("sr.query", srQuery.toString())); + if (startDate != null && endDate != null) { + q.put("p_date", new BasicDBObject("$gt", Integer.parseInt(DateUtil.formatToTightYYYYMMDD(startDate))) + .append("$lt", Integer.parseInt(DateUtil.formatToTightYYYYMMDD(endDate)))); + } else if (startDate != null && endDate == null) { + q.put("p_date", new BasicDBObject("$gt", Integer.parseInt(DateUtil.formatToTightYYYYMMDD(startDate)))); + } else if (startDate == null && endDate != null) { + q.put("p_date", new BasicDBObject("$lt", Integer.parseInt(DateUtil.formatToTightYYYYMMDD(endDate)))); + } + + return q; + } + + public Object loadAllAvgPriceForChartAction(AvgPriceUpdateModel model) { + JsonModel json = new JsonModel(); + Map data = new HashMap(); + List bos = regionStatService.findAllAvgPriceData(model.getCityName(), model.getRegion(), model.getBlock(), + null, true); + data.put("data", bos); + json.setJsonModel(data); + return json; + + } + + public Object detailAction(AvgPriceUpdateModel model) { + return model; + } + + public Object districtDetailAction(RentPriceUpdateModel model) { + District district = null; + if (StringUtils.isNotBlank(model.getId())) { + district = districtService.findById(model.getId()); + } else { + district = districtService.findDistrict(new DistrictKey(model.getCityDisplay(), model.getRegion(), model.getName()), + "district"); + } + + + if (district == null) { + return null; + } + + if (district.getEstateType2() != null && district.getEstateType2().indexOf("别墅") > -1) { + district.setEstateType2("别墅"); + } else { + district.setEstateType2("普通住宅"); + } + + model.setId(district.getId()); + Date thisMonth = DateUtil.getFirstDayOfMonth(new Date()); + Date lastMonth = DateUtil.getFirstDayOfLastMonth(DateUtil.getFirstDayOfLastMonth(new Date())); + Date thisWeekend = DateUtil.getThisSunday(new Date()); + + DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + + model.setCalcDistrictStat(this.calcDistrictStatDao.findCalcDistrictStat(district.getCity(), district.getRegion(), + district.getName(), district.getEstateType2(), thisWeekend)); + + List guidingPrices = guidingPriceService.findByDistrictKey(districtKey); + if(guidingPrices!=null && guidingPrices.size()>0) { + List t = new ArrayList(); + t.add(guidingPrices.get(guidingPrices.size()-1)); + model.setGuidingPrices(t); + }else { + model.setGuidingPrices(new ArrayList()); + } + Pager pager = new Pager(1); + pager.setCurrentPage(1); + model.setDealPriceCount(dealPriceService.findDealPriceCountByDistrictKey(districtKey)); + model.setAverageStatistics(averageStatisticsDao.findDistrictAverageStatistics(district.getCity(), district.getRegion(), + district.getName(), Integer.parseInt(DateUtil.formatByPattern(lastMonth, "yyyyMM")), + Integer.parseInt(DateUtil.formatByPattern(thisMonth, "yyyyMM")))); + model.setAvgBases(avgBaseDao.findAll(district.getCity(), district.getRegion(),district.getName(),0,null,null,null)); + Map distribution = new HashMap(); + /*distribution.put("QiTian", findSeawatersCount(district.getCity(), district.getRegion(), district.getName(), DateUtil.add(new Date(), Calendar.DATE, -7), null)); + distribution.put("YiYue", findSeawatersCount(district.getCity(), district.getRegion(), district.getName(), DateUtil.add(new Date(), Calendar.MONTH, -1), null)); + distribution.put("SanYue", findSeawatersCount(district.getCity(), district.getRegion(), district.getName(), DateUtil.add(new Date(), Calendar.MONTH, -3), null)); + distribution.put("BanNian", findSeawatersCount(district.getCity(), district.getRegion(), district.getName(), DateUtil.add(new Date(), Calendar.MONTH, -6), null)); + distribution.put("YiNian", findSeawatersCount(district.getCity(), district.getRegion(), district.getName(), DateUtil.add(new Date(), Calendar.YEAR, -1), null)); + distribution.put("All", findSeawatersCount(district.getCity(), district.getRegion(), district.getName(), null, null));*/ + List calcConfigs = calcConfigService.findDateByQuery(districtKey.getCity(), districtKey.getRegion(), + districtKey.getName()); + if (!calcConfigs.isEmpty()) { + model.setCalcConfig(calcConfigs.get(0)); + } + model.setDistrict(district); + model.setDistribution(distribution); + + Date week = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); + SqmetreRent sqmetreRent = sqmetreRentService.findSqmetreRent(districtKey.getCity(), districtKey.getRegion(), districtKey.getName(), "district", week); + model.setSqmetreRent(sqmetreRent); + + return model; + } + + public Object addGuidingPriceAction(@Param(name = "guidingDate") String guidingDate, + @Param(name = "guidingPrice") int guidingPrice, @Param(name = "id") String id,HttpServletRequest request) { + HttpSession session=request.getSession(); + District district = districtService.findById(id); + GuidingPrice guiding = new GuidingPrice(); + guiding.setGuidingPerson(session.getAttribute("realName").toString()); + guiding.setCat("district"); + guiding.setCreationDate(new Date()); + guiding.setGuidingValue(guidingPrice); + guiding.setGuidingDate(DateUtil.parseDate(guidingDate, "yyyyMMdd")); + guiding.setCity(district.getCity()); + guiding.setRegion(district.getRegion()); + guiding.setName(district.getName()); + guiding.setEstateType(district.getEstateType2().indexOf("别墅") > -1 ? "别墅" : "普通住宅"); + guidingPriceService.save(guiding); + + JsonModel model = new JsonModel(); + return model; + } + public Object guidingListAction(GuidingAvgPriceUpdateModel model) { + District district = districtService.findById(model.getId()); + model.setDistrict(district); + DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + List guidingPrices = guidingPriceService.findByDistrictKey(districtKey); + model.setGuidingPrices(guidingPrices); + return model; + } + public Object guidingPriceAction(GuidingAvgPriceUpdateModel model) { + District district = districtService.findById(model.getId()); + if(StringUtils.isNotBlank(model.getGid())){ + GuidingPrice gp = guidingPriceService.findById(model.getGid()); + String gd = DateUtil.formatToTightYYYYMMDD(gp.getGuidingDate()); + model.setGuidingDate(gd); + model.setGuidingPrice(gp.getGuidingValue()); + } + model.setDistrict(district); + return model; + } + + public Object deleteGuidingPriceNewAction(GuidingAvgPriceUpdateModel model) { + JsonModel json = new JsonModel(); + guidingPriceService.deleteById(model.getGid()); + return json; + } + + public Object saveGuidingPriceAction(HttpServletRequest request,GuidingAvgPriceUpdateModel model) { + JsonModel json = new JsonModel(); + HttpSession session=request.getSession(); + District district = districtService.findById(model.getId()); + GuidingPrice guiding = new GuidingPrice(); + if(StringUtils.isNotBlank(model.getGid())){ + guiding = guidingPriceService.findById(model.getGid()); + } + guiding.setGuidingPerson(session.getAttribute("realName").toString()); + guiding.setCat("district"); + guiding.setCreationDate(new Date()); + guiding.setGuidingValue(model.getGuidingPrice()); + guiding.setGuidingDate(DateUtil.parseDate(model.getGuidingDate(), "yyyyMMdd")); + guiding.setCity(district.getCity()); + guiding.setRegion(district.getRegion()); + guiding.setName(district.getName()); + guiding.setEstateType(district.getEstateType2().indexOf("别墅") > -1 ? "别墅" : "普通住宅"); + guidingPriceService.save(guiding); + json.setJsonModel(model); + return json; + } + + public Object updateCalcConfigAction(@Param(name = "maxPrice") Integer maxPrice, @Param(name = "minPrice") Integer minPrice, + @Param(name = "id") String id, @Param(name = "configId") String configId) { + CalcConfig calcConfig; + if (StringUtils.isNotBlank(configId) && ObjectId.isValid(configId)) { + calcConfig = calcConfigService.findById(configId); + if (calcConfig.getLogicalMaxValue() != null && calcConfig.getLogicalMaxValue().equals(maxPrice)) { + calcConfig.getMarkMap().put("logical_max_value", 1); + } + if (calcConfig.getLogicalMinValue() != null && calcConfig.getLogicalMinValue().equals(minPrice)) { + calcConfig.getMarkMap().put("logical_min_value", 1); + } + } else { + District district = districtService.findById(id); + calcConfig = new CalcConfig(); + calcConfig.setCategory("district"); + calcConfig.setCreationDate(new Date()); + calcConfig.setCity(district.getCity()); + calcConfig.setRegion(district.getRegion()); + calcConfig.setName(district.getName()); + calcConfig.setEstateType(district.getEstateType2()); + } + calcConfig.setLogicalMaxValue(maxPrice); + calcConfig.setLogicalMinValue(minPrice); + calcConfigService.save(calcConfig); + JsonModel model = new JsonModel(); + return model; + } + + public Object updateDistrictCalcConfigAction(AvgPriceUpdateModel model) { + JsonModel ret = new JsonModel(); + + + return ret; + } + + public Object taskListAction(AvgPriceUpdateModel model) { + Pager pager = model.getPager(15); + pager.setTotalRows(calcCommandService.findAllCommondCount(model.getStype(), model.getCityDisplay(), model.getKeyword(), + model.getStatus(), model.getStartDate(), model.getEndDate())); + model.setCalcCommands(calcCommandService.findCommandByquery(model.getStype(), model.getCityDisplay(), model.getKeyword(), + model.getStatus(), model.getStartDate(), model.getEndDate(), pager.getStartRow(), pager.getPageSize())); + return model; + } + + public Object deleteGuidingPriceAction(@Param(name = "id") String id) { + JsonModel model = new JsonModel(); + guidingPriceService.deleteById(id); + return model; + } + + public RegionStatService getRegionStatService() { + return regionStatService; + } + + public void setRegionStatService(RegionStatService regionStatService) { + this.regionStatService = regionStatService; + } + + public CalcCommandService getCalcCommandService() { + return calcCommandService; + } + + public void setCalcCommandService(CalcCommandService calcCommandService) { + this.calcCommandService = calcCommandService; + } + + public DistrictStatService getDistrictStatService() { + return districtStatService; + } + + public void setDistrictStatService(DistrictStatService districtStatService) { + this.districtStatService = districtStatService; + } + + public void setDistrictService(DistrictService districtService) { + this.districtService = districtService; + } + + public SqmetreRentService getSqmetreRentService() { + return sqmetreRentService; + } + + public void setSqmetreRentService(SqmetreRentService sqmetreRentService) { + this.sqmetreRentService = sqmetreRentService; + } + + public void setAverageStatisticsDao(AverageStatisticsDao averageStatisticsDao) { + this.averageStatisticsDao = averageStatisticsDao; + } + + public void setDealPriceService(DealPriceService dealPriceService) { + this.dealPriceService = dealPriceService; + } + + public void setGuidingPriceService(GuidingPriceService guidingPriceService) { + this.guidingPriceService = guidingPriceService; + } + + public void setCalcConfigService(CalcConfigService calcConfigService) { + this.calcConfigService = calcConfigService; + } + + public void setCalcDistrictStatDao(CalcDistrictStatDao calcDistrictStatDao) { + this.calcDistrictStatDao = calcDistrictStatDao; + } + + public void setAveragerQualityReportService(AveragerQualityReportService averagerQualityReportService) { + this.averagerQualityReportService = averagerQualityReportService; + } + + public void setSampleQualityReportService(SampleQualityReportService sampleQualityReportService) { + this.sampleQualityReportService = sampleQualityReportService; + } + + public ComparePriceDao getComparePriceDao() { + return comparePriceDao; + } + + public void setComparePriceDao(ComparePriceDao comparePriceDao) { + this.comparePriceDao = comparePriceDao; + } + + public ComparePriceCityDao getComparePriceCityDao() { + return comparePriceCityDao; + } + + public void setComparePriceCityDao(ComparePriceCityDao comparePriceCityDao) { + this.comparePriceCityDao = comparePriceCityDao; + } + + public void setSeawaterService(SeawaterService seawaterService) { + this.seawaterService = seawaterService; + } + + public void setAvgBaseDao(AvgBaseDao avgBaseDao) { + this.avgBaseDao = avgBaseDao; + } +} diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SeawaterController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SeawaterController.java index a223c1674..4db96b47b 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SeawaterController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SeawaterController.java @@ -3,6 +3,7 @@ package com.fangjia.bkoff.controllers; import com.fangjia.bkoff.model.JsonModel; import com.fangjia.bkoff.model.SeawaterModel; import com.fangjia.core.RegionData; +import com.fangjia.dal.SeawaterQuery; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seaweed.dao.DistrictDao; @@ -13,6 +14,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; import org.sothis.core.util.Pager; import org.sothis.web.mvc.annotation.Param; +import org.springframework.beans.BeanUtils; import java.io.UnsupportedEncodingException; import java.util.Arrays; @@ -49,6 +51,8 @@ public class SeawaterController { pager.setTotalRows(0); } }else { + SeawaterQuery query = new SeawaterQuery(); + BeanUtils.copyProperties(model, query, new String[] {"sortField", "regionAlias", "districtAlias", "keys", "table"}); String sort = model.getSort(); String sortField = null; boolean order = false; @@ -62,6 +66,8 @@ public class SeawaterController { if (sortField == null) { sortField = "p_date"; } + query.setOrder(order); + query.setSortField(sortField); List regionAlias = null; List districtAlias = null; @@ -80,9 +86,15 @@ public class SeawaterController { regionAlias = RegionData.getRegionAlias(city, model.getRegion()); } + query.setDistrictAlias(districtAlias); + query.setRegionAlias(regionAlias); + query.setTable("seawater"); + +// pager.setTotalRows((int) seawaterService.findSeawatersCountFromEs(model.getCat(), city, model.getRegion(),model.getDistrictName(), districtAlias, regionAlias, model.getStartDate(), model.getEndDate(), model.getRoom(), model.getRootName(),"seawater")); +// List seawaterList = seawaterService.findSeawatersFromEs(model.getCat(), city, model.getRegion(),model.getDistrictName(), districtAlias, regionAlias, model.getStartDate(), model.getEndDate(), model.getRoom(), model.getRootName(), sortField, order, pager.getStartRow(), pager.getPageSize(), null,"seawater"); + pager.setTotalRows((int)seawaterService.findSeawatersCountByQuery(query)); + List seawaterList = seawaterService.findSeawatersByQuery(query); - pager.setTotalRows((int) seawaterService.findSeawatersCountFromEs(model.getCat(), city, model.getRegion(),model.getDistrictName(), districtAlias, regionAlias, model.getStartDate(), model.getEndDate(), model.getRoom(), model.getRootName(),"seawater")); - List seawaterList = seawaterService.findSeawatersFromEs(model.getCat(), city, model.getRegion(),model.getDistrictName(), districtAlias, regionAlias, model.getStartDate(), model.getEndDate(), model.getRoom(), model.getRootName(), sortField, order, pager.getStartRow(), pager.getPageSize(), null,"seawater"); Map tagsMap = new HashMap(); for(Seawater seawater : seawaterList) { if(StringUtils.isBlank(seawater.getName())){ @@ -125,6 +137,58 @@ public class SeawaterController { return ret; } + /** + * 按条件批量删除数据(小区名称只匹配主名不匹配别名,查询时匹配别名) + * @param model + * @return + */ + public Object batchDeleteByParamsAction(SeawaterModel model) { + JsonModel ret = new JsonModel(); + + String city = model.getCityDisplay(); + if(StringUtils.isBlank(model.getCat())) { + model.setCat("sell"); + } + SeawaterQuery query = new SeawaterQuery(); + BeanUtils.copyProperties(model, query, new String[] {"sortField", "regionAlias", "districtAlias", "keys", "table"}); + List regionAlias = null; + model.setDistrictName(model.getDistrictName().trim()); + if(StringUtils.isNotBlank(model.getRegion())) { + regionAlias = RegionData.getRegionAlias(city, model.getRegion()); + } + query.setRegionAlias(regionAlias); + query.setTable("seawater"); + seawaterService.batchDeleteByParams(query); + + return ret; + } + + /** + * 按条件批量修改物业类型(小区名称只匹配主名不匹配别名,查询时匹配别名) + * @param model + * @return + */ + public Object batchUpdateEstateTypeAction(SeawaterModel model) { + JsonModel ret = new JsonModel(); + + String city = model.getCityDisplay(); + if(StringUtils.isBlank(model.getCat())) { + model.setCat("sell"); + } + SeawaterQuery query = new SeawaterQuery(); + BeanUtils.copyProperties(model, query, new String[] {"sortField", "regionAlias", "districtAlias", "keys", "table"}); + List regionAlias = null; + model.setDistrictName(model.getDistrictName().trim()); + if(StringUtils.isNotBlank(model.getRegion())) { + regionAlias = RegionData.getRegionAlias(city, model.getRegion()); + } + query.setRegionAlias(regionAlias); + query.setTable("seawater"); + seawaterService.batchUpdateEstateType(query); + + return ret; + } + public void setSeawaterService(SeawaterService seawaterService) { this.seawaterService = seawaterService; } diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SqMetreRentController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SqMetreRentController.java index 4c369289e..8e2662190 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SqMetreRentController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/SqMetreRentController.java @@ -102,7 +102,7 @@ public class SqMetreRentController { JSONObject row = new JSONObject(); row.put("x",DateUtil.add(rent.getWeekend(), 1).getTime()); row.put("y",rent.getAvgPrice()); - row.put("sampleEnum",rent.getSampleEnum()); + row.put("sampleEnum",rent.getValueType()); datas.add(row); } @@ -136,7 +136,7 @@ public class SqMetreRentController { Date statDate = DateUtil.parseFormatDate(model.getStatDate()); SqmetreRent stat = sqmetreRentService.findSqmetreRent(model.getCityDisplay(), model.getRegion(), model.getDistrictName(), model.getCat(), statDate); if(null != stat){ - stat.setAvgPrice(model.getValue()); + stat.setAvgPrice(model.getValue().intValue()); stat.setModificationDate(new Date()); sqmetreRentService.updateSqmetreRent(stat); } @@ -204,7 +204,7 @@ public class SqMetreRentController { rent.setDistrictName(model.getDistrictName()); rent.setWeekend(statDate); rent.setCat(model.getCat()); - rent.setAvgPrice(model.getValue()); + rent.setAvgPrice(model.getValue().intValue()); rent.setCreationDate(new Date()); rent.setModificationDate(new Date()); sqmetreRentService.save(rent); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java index 37e2d93f7..3a51dcc61 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/YugudanController.java @@ -362,8 +362,8 @@ public class YugudanController { if (model.getId()>0) { //FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(model.getId()); Map data = new HashMap(); - if(StringUtils.isNotBlank(model.getAddress())){ - data.put("address", model.getAddress()); + if(StringUtils.isNotBlank(model.getCertificateAddress())){ + data.put("certificateAddress", model.getCertificateAddress()); } // if(model.getPeopleEditDate()!=null || model.getPeopleEditPrice()!=null){ // data.put("peopleEdit", 1); diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/fjtEvaluationController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/fjtEvaluationController.java index 18e5f75ed..93eec6a06 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/fjtEvaluationController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/fjtEvaluationController.java @@ -113,7 +113,9 @@ public class fjtEvaluationController { FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(Integer.parseInt(bo.getEvaluationId())); if(evaluation != null){ String addr = ""; - if(evaluation.getAddress() != null){ + if(StringUtils.isNotBlank(evaluation.getCertificateAddress())){ + addr += evaluation.getCertificateAddress(); + }else if(evaluation.getAddress() != null){ addr += evaluation.getAddress(); } if(StringUtils.isNotBlank(evaluation.getHouseNum())){ diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/RentPriceUpdateModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/RentPriceUpdateModel.java new file mode 100644 index 000000000..5245c5e0b --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/RentPriceUpdateModel.java @@ -0,0 +1,341 @@ +package com.fangjia.bkoff.model; + +import com.fangjia.dal.mongo.model.*; +import com.fangjia.dal.mongo.model.bkoff.CalcCommand; +import com.fangjia.dal.mongo.model.bkoff.CalcConfig; +import com.fangjia.dal.mongo.model.bkoff.GuidingPrice; +import com.fangjia.dal.seaweed.model.District; +import com.fangjia.service.bo.AvgUpdateBo; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +public class RentPriceUpdateModel extends BaseModel { + + /** + * + */ + private static final long serialVersionUID = 1L; + + private List avgUpdateBos; + private Map> avgUpdateBoMap; + private String region; + private String block; + private String cityDisplay; + private String id; + private int avgPrice; + private Double minCredit; + private Double maxCredit; + private Integer valueType; + private String name; + private District district; + private List averageStatistics; + private List avgBases; + private CalcDistrictStat calcDistrictStat; + private DistrictStat districtStat; + private SqmetreRent sqmetreRent; + private long dealPriceCount; + private List guidingPrices; + private Map distribution; + private CalcConfig calcConfig; + private String stype; + private String keyword; + private Integer status; + private List calcCommands; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date startDate; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date endDate; + private String estateType; + private String sort; + private Date weekend; + private int sdateStart; + private int sdateEnd; + private String series; + private String vts; + private String category; + + public List getAvgUpdateBos() { + return avgUpdateBos; + } + + public void setAvgUpdateBos(List avgUpdateBos) { + this.avgUpdateBos = avgUpdateBos; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getBlock() { + return block; + } + + public void setBlock(String block) { + this.block = block; + } + + public String getCityDisplay() { + return cityDisplay; + } + + public void setCityDisplay(String cityDisplay) { + this.cityDisplay = cityDisplay; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public int getAvgPrice() { + return avgPrice; + } + + public void setAvgPrice(int avgPrice) { + this.avgPrice = avgPrice; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public District getDistrict() { + return district; + } + + public void setDistrict(District district) { + this.district = district; + } + + public List getAverageStatistics() { + return averageStatistics; + } + + public void setAverageStatistics(List averageStatistics) { + this.averageStatistics = averageStatistics; + } + + public long getDealPriceCount() { + return dealPriceCount; + } + + public void setDealPriceCount(long dealPriceCount) { + this.dealPriceCount = dealPriceCount; + } + + public List getGuidingPrices() { + return guidingPrices; + } + + public void setGuidingPrices(List guidingPrices) { + this.guidingPrices = guidingPrices; + } + + public Map getDistribution() { + return distribution; + } + + public void setDistribution(Map distribution) { + this.distribution = distribution; + } + + public CalcConfig getCalcConfig() { + return calcConfig; + } + + public void setCalcConfig(CalcConfig calcConfig) { + this.calcConfig = calcConfig; + } + + public String getStype() { + return stype; + } + + public void setStype(String stype) { + this.stype = stype; + } + + public String getKeyword() { + return keyword; + } + + public void setKeyword(String keyword) { + this.keyword = keyword; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Date getStartDate() { + return startDate; + } + + public void setStartDate(Date startDate) { + this.startDate = startDate; + } + + public Date getEndDate() { + return endDate; + } + + public void setEndDate(Date endDate) { + this.endDate = endDate; + } + + public List getCalcCommands() { + return calcCommands; + } + + public void setCalcCommands(List calcCommands) { + this.calcCommands = calcCommands; + } + + public String getEstateType() { + return estateType; + } + + public void setEstateType(String estateType) { + this.estateType = estateType; + } + + public Map> getAvgUpdateBoMap() { + return avgUpdateBoMap; + } + + public void setAvgUpdateBoMap(Map> avgUpdateBoMap) { + this.avgUpdateBoMap = avgUpdateBoMap; + } + + public String getSort() { + return sort; + } + + public void setSort(String sort) { + this.sort = sort; + } + + public Double getMinCredit() { + return minCredit; + } + + public void setMinCredit(Double minCredit) { + this.minCredit = minCredit; + } + + public Double getMaxCredit() { + return maxCredit; + } + + public void setMaxCredit(Double maxCredit) { + this.maxCredit = maxCredit; + } + + public Integer getValueType() { + return valueType; + } + + public void setValueType(Integer valueType) { + this.valueType = valueType; + } + + public CalcDistrictStat getCalcDistrictStat() { + return calcDistrictStat; + } + + public void setCalcDistrictStat(CalcDistrictStat calcDistrictStat) { + this.calcDistrictStat = calcDistrictStat; + } + + public DistrictStat getDistrictStat() { + return districtStat; + } + + public void setDistrictStat(DistrictStat districtStat) { + this.districtStat = districtStat; + } + + public Date getWeekend() { + return weekend; + } + + public void setWeekend(Date weekend) { + this.weekend = weekend; + } + + + public int getSdateStart() { + return sdateStart; + } + + public void setSdateStart(int sdateStart) { + this.sdateStart = sdateStart; + } + + public int getSdateEnd() { + return sdateEnd; + } + + public void setSdateEnd(int sdateEnd) { + this.sdateEnd = sdateEnd; + } + + public String getSeries() { + return series; + } + + public void setSeries(String series) { + this.series = series; + } + + public String getVts() { + return vts; + } + + public void setVts(String vts) { + this.vts = vts; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public List getAvgBases() { + return avgBases; + } + + public void setAvgBases(List avgBases) { + this.avgBases = avgBases; + } + + + public SqmetreRent getSqmetreRent() { + return sqmetreRent; + } + + public void setSqmetreRent(SqmetreRent sqmetreRent) { + this.sqmetreRent = sqmetreRent; + } +} diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/SeawaterModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/SeawaterModel.java index 9b923244b..35660d045 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/SeawaterModel.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/SeawaterModel.java @@ -35,12 +35,19 @@ public class SeawaterModel extends BaseModel { private Seawater seawater; private List seawaterList; private Map tagsMap; - private Integer order; + private Boolean order; private Integer StartDate; private Integer endDate; private String sort; private String url; private String rootName; + private String estateType; + private Double startArea; + private Double endArea; + private Integer startPrice; + private Integer endPrice; + private Integer status; + private String editEstateType; public String getId() { return id; } @@ -156,10 +163,10 @@ public class SeawaterModel extends BaseModel { public void setCityDisplay(String cityDisplay) { this.cityDisplay = cityDisplay; } - public Integer getOrder() { + public Boolean getOrder() { return order; } - public void setOrder(Integer order) { + public void setOrder(Boolean order) { this.order = order; } @@ -220,4 +227,60 @@ public class SeawaterModel extends BaseModel { public void setTagsMap(Map tagsMap) { this.tagsMap = tagsMap; } + + public String getEstateType() { + return estateType; + } + + public void setEstateType(String estateType) { + this.estateType = estateType; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public Integer getStartPrice() { + return startPrice; + } + + public void setStartPrice(Integer startPrice) { + this.startPrice = startPrice; + } + + public Integer getEndPrice() { + return endPrice; + } + + public void setEndPrice(Integer endPrice) { + this.endPrice = endPrice; + } + + public Double getStartArea() { + return startArea; + } + + public void setStartArea(Double startArea) { + this.startArea = startArea; + } + + public Double getEndArea() { + return endArea; + } + + public void setEndArea(Double endArea) { + this.endArea = endArea; + } + + public String getEditEstateType() { + return editEstateType; + } + + public void setEditEstateType(String editEstateType) { + this.editEstateType = editEstateType; + } } diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/YugudanModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/YugudanModel.java index d710efeed..10db294b2 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/YugudanModel.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/YugudanModel.java @@ -150,8 +150,10 @@ public class YugudanModel extends BaseModel{ private String reportNum; private Integer askPriceId;//人工询价id - - + + private String certificateAddress; + + public int getId() { return id; } @@ -861,7 +863,12 @@ public class YugudanModel extends BaseModel{ public void setUnitNum(String unitNum) { this.unitNum = unitNum; } - - - + + public String getCertificateAddress() { + return certificateAddress; + } + + public void setCertificateAddress(String certificateAddress) { + this.certificateAddress = certificateAddress; + } } diff --git a/projects/fangjia-bkoff/src/main/webapp/apiUser/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/apiUser/edit.ftl index 94b2730f3..caf99a2c7 100644 --- a/projects/fangjia-bkoff/src/main/webapp/apiUser/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/apiUser/edit.ftl @@ -90,12 +90,12 @@ - 浠锋牸绫诲瀷 + 浼颁环绫诲瀷 diff --git a/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl b/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl index 9ade46e39..a3cb6949e 100644 --- a/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/avgPriceUpdate/districtDetail.ftl @@ -227,7 +227,7 @@ 鍏斿崥澹� - <#if (it.centanetNewPrice)?? || (it.centanetEsfPrice)??> + <#if (it.zhongyuanNewPrice)?? || (it.zhongyuanEsfPrice)??> 涓師鍦颁骇 <#if (it.leyoujiaNewPrice)?? || (it.leyoujiaEsfPrice)??> @@ -247,7 +247,13 @@ <#if (it.yungujiaNewPrice)?? || (it.yungujiaEsfPrice)??> 浜戜及浠� - + + <#if (it.jiwuNewPrice)?? || (it.jiwuEsfPrice)??> + 鍚夊眿 + + <#if (it.tsfhouseNewPrice)?? || (it.tsfhouseEsfPrice)??> + 365鎴夸骇 + @@ -300,7 +306,7 @@ 浜屾墜鎴� - <#if (it.centanetNewPrice)?? || (it.centanetEsfPrice)??> + <#if (it.zhongyuanNewPrice)?? || (it.zhongyuanEsfPrice)??> 鏂版埧 浜屾墜鎴� @@ -329,10 +335,17 @@ <#if (it.yungujiaNewPrice)?? || (it.yungujiaEsfPrice)??> 鏂版埧 - 浜屾墜鎴� - + 浜屾墜鎴� - + <#if (it.jiwuNewPrice)?? || (it.jiwuEsfPrice)??> + 鏂版埧 + 浜屾墜鎴� + + <#if (it.tsfhouseNewPrice)?? || (it.tsfhouseEsfPrice)??> + 鏂版埧 + 浜屾墜鎴� + + @@ -392,9 +405,9 @@ ${it.tuboshiEsfPrice!} - <#if (it.centanetNewPrice)?? || (it.centanetEsfPrice)??> - ${it.centanetNewPrice!} - ${it.centanetEsfPrice!} + <#if (it.zhongyuanNewPrice)?? || (it.zhongyuanEsfPrice)??> + ${it.zhongyuanNewPrice!} + ${it.zhongyuanEsfPrice!} <#if (it.leyoujiaNewPrice)?? || (it.leyoujiaEsfPrice)??> @@ -422,7 +435,15 @@ ${it.yungujiaNewPrice!} ${it.yungujiaEsfPrice!} - + <#if (it.jiwuNewPrice)?? || (it.jiwuEsfPrice)??> + ${it.jiwuNewPrice!} + ${it.jiwuEsfPrice!} + + <#if (it.tsfhouseNewPrice)?? || (it.tsfhouseEsfPrice)??> + ${it.tsfhouseNewPrice!} + ${it.tsfhouseEsfPrice!} + + <#if averageStatistics?size == 0> @@ -724,19 +745,59 @@ }); var series = new Array(); series[0] = { - name:"鎴挎簮鏁e垪鍥�", + name:"鎴挎簮鏁e垪鍥�-鏅�氫綇瀹�", data:new Array() //color:"#660099" } + series[1] = { + name:"鎴挎簮鏁e垪鍥�-鍟嗕綇", + data:new Array(), + color:"#12990f" + } + series[2] = { + name:"鎴挎簮鏁e垪鍥�-鍒", + data:new Array(), + color:"#992632" + } + var series_arry_0 = [] + var series_arry_1 = [] + var series_arry_2 = [] $(data).each(function(i,item){ var pdate = item.pDate + ""; var ds = pdate.substring(0, 4) + '-' + pdate.substring(4, 6) + '-' + pdate.substring(6) - series[0].data.push({ - x:new Date(ds), - y:Math.round(item.totalPrice/item.area), - data:item - }) + if ((item.estateType!=null) && (item.estateType.indexOf('鍒') > -1)){ + series_arry_2.push({ + x:new Date(ds), + y:Math.round(item.totalPrice/item.area), + data:item + }) + }else if((item.estateType!=null) && ((item.estateType.indexOf('鍟嗕綇') > -1) || (item.estateType.indexOf('鍟嗕笟') > -1) || (item.estateType.indexOf('鍟嗗姟') > -1) || (item.estateType.indexOf('鍏瘬閰掑簵') > -1))){ + series_arry_1.push({ + x:new Date(ds), + y:Math.round(item.totalPrice/item.area), + data:item + }) + }else{ + series_arry_0.push({ + x:new Date(ds), + y:Math.round(item.totalPrice/item.area), + data:item + }) + } + // series[0].data.push({ + // x:new Date(ds), + // y:Math.round(item.totalPrice/item.area), + // data:item + // }) + // series[1].data.push({ + // x:new Date(ds), + // y:Math.round(item.totalPrice/item.area), + // data:item + // }) }); + series[0].data = series_arry_0 + series[1].data = series_arry_1 + series[2].data = series_arry_2 $('#houseChart').highcharts({ chart: { type: 'scatter', diff --git a/projects/fangjia-bkoff/src/main/webapp/data/city.ftl b/projects/fangjia-bkoff/src/main/webapp/data/city.ftl index 25959e242..c5d2e5a60 100644 --- a/projects/fangjia-bkoff/src/main/webapp/data/city.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/data/city.ftl @@ -74,7 +74,7 @@ - +<#-- -->
@@ -234,7 +234,7 @@ Mapabc缁忕含搴�
瑕嗙洊鐜� 鐧惧害缁忕含搴�
瑕嗙洊鐜� - 绉熼噾瑕嗙洊鐜� +<#-- 绉熼噾瑕嗙洊鐜�--> 缁熻鍛� @@ -269,7 +269,7 @@ <#if info.priceToEva??>${info.priceToEva * 100}%<#else>鏈煡 <#if info.districtDetail.latLng??>${info.districtDetail.latLng * 100}%<#else>鏈煡 <#if info.districtDetail.latLng2??>${info.districtDetail.latLng2 * 100}%<#else>鏈煡 - <#if info.districtDetail.rentPrice??>${info.districtDetail.rentPrice * 100}%<#else>鏈煡 +<#-- <#if info.districtDetail.rentPrice??>${info.districtDetail.rentPrice * 100}%<#else>鏈煡--> ${info.weekend?string('yyyy-MM-dd')} 璇︾粏 @@ -306,7 +306,7 @@ - +<#-- --> 瀵煎嚭 @@ -444,7 +444,7 @@ Mapabc缁忕含搴�
瑕嗙洊鐜� 鐧惧害缁忕含搴�
瑕嗙洊鐜� - 绉熼噾瑕嗙洊鐜� +<#-- 绉熼噾瑕嗙洊鐜�--> 缁熻鍛� @@ -478,7 +478,7 @@ <#if info.priceToEva??>${info.priceToEva * 100}%<#else>鏈煡 <#if info.shopDetail.latLng??>${info.shopDetail.latLng * 100}%<#else>鏈煡 <#if info.shopDetail.latLng2??>${info.shopDetail.latLng2 * 100}%<#else>鏈煡 - <#if info.shopDetail.rentPrice??>${info.shopDetail.rentPrice * 100}%<#else>0 +<#-- <#if info.shopDetail.rentPrice??>${info.shopDetail.rentPrice * 100}%<#else>0--> ${info.weekend?string('yyyy-MM-dd')} 璇︾粏 @@ -515,7 +515,7 @@ - +<#-- --> 瀵煎嚭 @@ -653,7 +653,7 @@ Mapabc缁忕含搴�
瑕嗙洊鐜� 鐧惧害缁忕含搴�
瑕嗙洊鐜� - 绉熼噾瑕嗙洊鐜� +<#-- 绉熼噾瑕嗙洊鐜�--> 缁熻鍛� @@ -687,7 +687,7 @@ <#if info.priceToEva??>${info.priceToEva * 100}%<#else>鏈煡 <#if info.officeDetail.latLng??>${info.officeDetail.latLng * 100}%<#else>鏈煡 <#if info.officeDetail.latLng2??>${info.officeDetail.latLng2 * 100}%<#else>鏈煡 - <#if info.officeDetail.rentPrice??>${info.officeDetail.rentPrice * 100}%<#else>0 +<#-- <#if info.officeDetail.rentPrice??>${info.officeDetail.rentPrice * 100}%<#else>0--> ${info.weekend?string('yyyy-MM-dd')} 璇︾粏 diff --git a/projects/fangjia-bkoff/src/main/webapp/district/list.ftl b/projects/fangjia-bkoff/src/main/webapp/district/list.ftl index a3ff35d00..60e42be0a 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district/list.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district/list.ftl @@ -686,6 +686,7 @@ 閰嶅 鍧囦环 + 绉熼噾 鏍锋湰 鏁版嵁瀵规瘮 澶嶅埗 diff --git a/projects/fangjia-bkoff/src/main/webapp/loudong/list.ftl b/projects/fangjia-bkoff/src/main/webapp/loudong/list.ftl index f131f5e6e..ff6b813b2 100644 --- a/projects/fangjia-bkoff/src/main/webapp/loudong/list.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/loudong/list.ftl @@ -152,7 +152,7 @@ <#list loudongList! as loudong> - + ${loudong.districtNo!''} diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/detail.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/detail.ftl new file mode 100644 index 000000000..2757fd165 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/detail.ftl @@ -0,0 +1,470 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + ${cityDisplay}<#if region?? && region!=''>-${region}<#if block?? && block!=''>-${block}</#if></#if> 鍩庡競绉熼噾璇︾粏淇℃伅 + + + + + + <#include "/common_js.ftl" /> + + + + + + + + + +<#include "/layout/header.ftl" /> +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
鏌ヨ鏉′欢绉熼噾鏇存柊绉熼噾鏂板
+ + + <@d_city defaultValue="${cityDisplay!}" callback="changeCity" callbackField="display" name="cityDisplay"/> + + 鏃ユ湡锛� + + + + 鏂板鏃ユ湡锛� + + +
+ + + + + 瑕嗙洊鐜�: + + + + 瑕嗙洊鐜�: + + +
+ + + + + 鍙俊搴�: + + + + 鍙俊搴�: + + +
+ + / + + + + + + + +
+ 鏇存柊绉熼噾 + 鍒犻櫎 + + 鏂板浠锋牸 +
+
+ + + + + + +
+
+
+ + + + + + + + + + + + + +
鏃ユ湡绉熼噾璁$畻绉熼噾鍛ㄥ悓姣�鍙俊搴�瑕嗙洊鐜�
+
+
+
+ + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/districtDetail.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/districtDetail.ftl new file mode 100644 index 000000000..ecab150c2 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/districtDetail.ftl @@ -0,0 +1,946 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + <#if district??>${district.name!}</#if> 绉熼噾璇︾粏淇℃伅 + + + + + <#include "/common_js.ftl" /> + + + + + + + + + + +<#include "/layout/header.ftl" /> +
+

+ + ${district.city!}- + ${district.region!}- + ${district.name!}(${district.estateType2!}) + 绉熼噾璇︾粏淇℃伅 + 閲嶇畻绉熼噾 +

+
+

+ <#if sqmetreRent??> + 鏈�鏂�(${t_util.formatDate(sqmetreRent.weekend!)}) + 绉熼噾锛�${sqmetreRent.avgPrice!}鍏�/骞崇背  +<#-- 2B绉熼噾锛�${districtStat.avg_price_2b!}鍏�/骞崇背 --> + 璁$畻鏂瑰紡锛� + <#if (sqmetreRent.valueType)??> + <#if sqmetreRent.valueType=='own'> + 鑷韩鏍锋湰 + + <#if sqmetreRent.valueType=='around'> + 鍛ㄨ竟 + + + 鍙俊搴︼細${(sqmetreRent.credit!)?string("#.#######")} + <#if sqmetreRent.markup??> +  鐜瘮涓婃湀锛�<#if sqmetreRent.markup gt 0>鈫�<#elseif sqmetreRent.markup lt 0>鈫�${(sqmetreRent.markup!/100.0)?string("#.##")}% + + <#else> + 鏈湀绉熼噾鏆傛棤 + +

+
+

${district.name!}-浠锋牸璧板娍鍥�

+ +
+ +<#--
--> +<#--

${district.name!}-鎸囧浠�

--> +<#--
--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> + +<#-- <#list guidingPrices as it>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#list guidingPrices as it>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#list guidingPrices as it>--> +<#-- --> +<#--
--> +<#--
--> +<#-- --> +<#-- --> +<#-- --> +<#--
--> +<#--
鏃ユ湡 ${it.guidingDate?string("yyyy-MM-dd")} --> +<#--
浠锋牸${it.guidingValue!}
娣诲姞浜�${it.guidingPerson!}
--> +<#--
--> +<#--
--> +<#--
--> +<#--

${district.name!}-鍙傝�冧环

--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (avgBases??&&avgBases?size>0)>--> +<#-- <#list avgBases as it>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if avgBases?size == 0>--> +<#-- --> +<#-- --> +<#-- --> +<#--
浠锋牸璋冩煡鏃堕棿璋冩煡浜�
--> +<#-- ${it.price}--> +<#-- --> +<#-- ${it.creationDate?string("yyyy-MM-dd")}--> +<#-- --> +<#-- <#if it.userName??>--> +<#-- ${it.userName}--> +<#-- --> +<#--
鏆傛棤鏁版嵁
--> +<#--
--> +<#--
--> +<#-- --> +<#--

${district.name!}-绗笁鏂逛环鏍煎姣�

--> +<#-- --> +<#-- <#list averageStatistics as it>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.xiaozijiaNewPrice)?? || (it.xiaozijiaEsfPrice)??> --> +<#-- --> +<#-- --> +<#-- <#if (it.lianjiaNewPrice)?? || (it.lianjiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.anjukeNewPrice)?? || (it.anjukeEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.zhugezhaofangNewPrice)?? || (it.zhugezhaofangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.beikeNewPrice)?? || (it.beikeEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.fangguguNewPrice)?? || (it.fangguguEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.fangdoudouNewPrice)?? || (it.fangdoudouEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.soufangNewPrice)?? || (it.soufangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.cityhouseNewPrice)?? || (it.cityhouseEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.fangtanNewPrice)?? || (it.fangtanEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.tuboshiNewPrice)?? || (it.tuboshiEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.centanetNewPrice)?? || (it.centanetEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.leyoujiaNewPrice)?? || (it.leyoujiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.qfangNewPrice)?? || (it.qfangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.sinaNewPrice)?? || (it.sinaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.cricNewPrice)?? || (it.cricEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.ganjiNewPrice)?? || (it.ganjiEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- <#if (it.yungujiaNewPrice)?? || (it.yungujiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.xiaozijiaNewPrice)?? || (it.xiaozijiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.lianjiaNewPrice)?? || (it.lianjiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.anjukeNewPrice)?? || (it.anjukeEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.zhugezhaofangNewPrice)?? || (it.zhugezhaofangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.beikeNewPrice)?? || (it.beikeEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.fangguguNewPrice)?? || (it.fangguguEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.fangdoudouNewPrice)?? || (it.fangdoudouEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.soufangNewPrice)?? || (it.soufangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.cityhouseNewPrice)?? || (it.cityhouseEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.fangtanNewPrice)?? || (it.fangtanEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.tuboshiNewPrice)?? || (it.tuboshiEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.centanetNewPrice)?? || (it.centanetEsfPrice)??> --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.leyoujiaNewPrice)?? || (it.leyoujiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.qfangNewPrice)?? || (it.qfangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.sinaNewPrice)?? || (it.sinaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.cricNewPrice)?? || (it.cricEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.ganjiNewPrice)?? || (it.ganjiEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.yungujiaNewPrice)?? || (it.yungujiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.xiaozijiaNewPrice)?? || (it.xiaozijiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.lianjiaNewPrice)?? || (it.lianjiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.anjukeNewPrice)?? || (it.anjukeEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.zhugezhaofangNewPrice)?? || (it.zhugezhaofangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.beikeNewPrice)?? || (it.beikeEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.fangguguNewPrice)?? || (it.fangguguEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.fangdoudouNewPrice)?? || (it.fangdoudouEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.soufangNewPrice)?? || (it.soufangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.cityhouseNewPrice)?? || (it.cityhouseEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.fangtanNewPrice)?? || (it.fangtanEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.tuboshiNewPrice)?? || (it.tuboshiEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.centanetNewPrice)?? || (it.centanetEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.leyoujiaNewPrice)?? || (it.leyoujiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.qfangNewPrice)?? || (it.qfangEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.sinaNewPrice)?? || (it.sinaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.cricNewPrice)?? || (it.cricEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.ganjiNewPrice)?? || (it.ganjiEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if (it.yungujiaNewPrice)?? || (it.yungujiaEsfPrice)??>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if averageStatistics?size == 0>--> +<#-- --> +<#-- --> +<#--
鏃ユ湡鎴夸环缃�灏忚祫瀹�閾惧瀹夊眳瀹�璇歌憶璐濆3鎴夸及浼�鎴垮厹鍏�鎼滄埧cityhouse鎴挎帰鍏斿崥澹�涓師鍦颁骇涔愭湁瀹�Q鎴�鏂版氮鍏嬪皵鐟�璧堕泦缃�浜戜及浠�
鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�鏂版埧浜屾墜鎴�
${(t_util.parseDate(it.shortDate?string, 'yyyyMM'))?string("yyyy骞碝M鏈�")}${it.fangjiaPrice!}${it.xiaozijiaNewPrice!}${it.xiaozijiaEsfPrice!}${it.lianjiaNewPrice!}${it.lianjiaEsfPrice!}${it.anjukeNewPrice!}${it.anjukeEsfPrice!}${it.zhugezhaofangNewPrice!}${it.zhugezhaofangEsfPrice!}${it.beikeNewPrice!}${it.beikeEsfPrice!}${it.fangguguNewPrice!}${it.fangguguEsfPrice!}${it.fangdoudouNewPrice!}${it.fangdoudouEsfPrice!}${it.soufangNewPrice!}${it.soufangEsfPrice!}${it.cityhouseNewPrice!}${it.cityhouseEsfPrice!}${it.fangtanNewPrice!}${it.fangtanEsfPrice!}${it.tuboshiNewPrice!}${it.tuboshiEsfPrice!}${it.centanetNewPrice!}${it.centanetEsfPrice!}${it.leyoujiaNewPrice!}${it.leyoujiaEsfPrice!}${it.qfangNewPrice!}${it.qfangEsfPrice!}${it.sinaNewPrice!}${it.sinaEsfPrice!}${it.cricNewPrice!}${it.cricEsfPrice!}${it.ganjiNewPrice!}${it.ganjiEsfPrice!}${it.yungujiaNewPrice!}${it.yungujiaEsfPrice!}
鏆傛棤鏁版嵁
--> +<#-- --> +<#--
--> +<#-- <#if !(calcDistrictStat??)>
鏈湀娌℃湁鍒╃敤绗笁鏂圭殑璇︾粏鏁版嵁锛�
--> +<#--
--> +<#-- <#if calcDistrictStat?? && calcDistrictStat.thirdpartDetails??>--> +<#-- <#macro greet pt> --> +<#-- <#assign ret=0>--> +<#-- <#list calcDistrictStat.thirdpartDetails as detail>--> +<#-- <#if detail['name'] == pt && detail['val']??>--> +<#-- <#assign ret>${detail['val']!}--> +<#-- --> +<#-- --> +<#-- ${ret}--> +<#-- --> +<#-- --> + +

+ ${district.name!}-鏍锋湰鏁e垪鍥� + 鎴挎簮鏍锋湰鍒楄〃 +<#-- 浜ゆ槗鏁版嵁鍒楄〃(${dealPriceCount!})--> +

+ <#-- + + +
涓冨ぉ涓�涓湀涓変釜鏈�鍗婂勾涓�骞�鍏ㄩ儴
${distribution['QiTian']}${distribution['YiYue']}${distribution['SanYue']}${distribution['BanNian']}${distribution['YiNian']}${distribution['All']}
--> +

${district.name!}-鎴挎簮鏁e垪鍥�

+
+<#--

${district.name!}-鏍锋湰杩囨护瑙勫垯

--> +<#--
--> +<#-- <#if calcConfig??>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if calcConfig??>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- --> +<#-- <#if calcConfig??>--> +<#-- --> +<#-- <#else>--> +<#-- --> +<#-- --> +<#-- (璇疯缃緝澶у尯闂�)--> +<#--
--> +<#--

--> +<#--
--> +<#-- --> +<#--
--> +<#--


--> +
+ + + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/districtList.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/districtList.ftl new file mode 100644 index 000000000..b5846cabb --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/districtList.ftl @@ -0,0 +1,344 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + ${cityDisplay}<#if region?? && region!=''>-${region}<#if block?? && block!=''>-${block}</#if></#if> 灏忓尯鍧囦环鎿嶄綔鍒楄〃 + + + + + <#include "/common_js.ftl" /> + + + + + + + + + + +<#include "/layout/header.ftl" /> +<#setting number_format="0.###########"/> +
+ + + + + + + +
鏌ヨ鏉′欢
+
+ <@d_city defaultValue="${cityDisplay!}" callback="changeCity" callbackField="display" name="cityDisplay"/> + + + + + + + - + + + + + + + + + +
+
+ 鏌ヨ鍒�${pager.totalRows!}鏉¤褰� + + + + + + + + + + + + + + <#if avgUpdateBos??> + <#list avgUpdateBos as it> + + + + + + + + + + + + + + + + + + + +
鍖哄煙灏忓尯鍚�鍒悕鐗╀笟绫诲瀷 + <#if sort?? && sort == 'price_desc'> + 鍧囦环鈫� + <#elseif sort?? && sort == 'price_asc'> + 鍧囦环鈫� + <#else> + 鍧囦环 + + 鏃堕棿 + <#if sort?? && sort == 'credit_desc'>鍙俊搴�鈫� + <#elseif sort?? && sort == 'credit_asc'>鍙俊搴�鈫� + <#else>鍙俊搴� + + 璁$畻绫诲瀷鎿嶄綔
${it.region!}${it.districtName!} + ${t_util.join(it.alias!, ',')} + ${it.estateType!}${it.avgPrice!}${it.weekend!}${it.credit} + <#if it.valueType?? && it.valueType == 1> + 鎸囧浠� + <#elseif it.valueType?? && it.valueType == 2> + 鑷韩鏍锋湰 + <#elseif it.valueType?? && it.valueType == 3> + 绗笁鏂� + <#elseif it.valueType?? && it.valueType == 4> + 浜ゆ槗鏁版嵁 + <#elseif it.valueType?? && it.valueType == 5> + 鍛ㄨ竟妤肩洏 + + + 璋冩煡 + 鏍锋湰 + 璁剧疆 +
+ <@d_pager action = "districtList" currentPage = pager.currentPage controller = "rentPriceUpdate" totalPages = pager.totalPages pagerCount=12 theme="full"/> +
+
+ + + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/doCreatTask.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/doCreatTask.ftl new file mode 100644 index 000000000..2cc00dfbe --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/doCreatTask.ftl @@ -0,0 +1,278 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 鍧囦环鎿嶄綔鍚庡彴 + + + + + + <#include "/common_js.ftl" /> + + + + +<#include "/layout/header.ftl" /> +
+ <#-- + + + + + + + + + + + + + + +
椤�鍊�
鍩庡競
鍖哄煙
鏉垮潡
璧锋鏃堕棿 + + - +
鎿嶄綔 +
+ --> +
+ <#assign seqIndex = 0 /> + + + + + + + + <#list ['HOT', 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z'] as key> + <#if avgUpdateBoMap[key]?? && avgUpdateBoMap[key]?size gt 0> + + + + <#list avgUpdateBoMap[key] as it> + + + + + + + + + + +
+ <#list ['HOT', 'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z'] as key> + ${key}     + +
搴忓彿鍚嶇О鏈�鏂板潎浠�璧峰鏃ユ湡鎿嶄綔
  ${key}
<#assign seqIndex = seqIndex +1 />${seqIndex} +  ${it.city} + ${it.avgPrice!}${it.startDate!} + 璇︾粏淇℃伅 + 鏌ョ湅灏忓尯 + 鏍锋湰鎴挎簮 +
+ + + + +<#-- + +--> + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/guidingList.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/guidingList.ftl new file mode 100644 index 000000000..e12ac9f88 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/guidingList.ftl @@ -0,0 +1,192 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 鎸囧浠锋暟鎹� + + + + + + +<#include "/common_js.ftl" /> +<#include "/layout/header.ftl" /> +<#setting number_format="0.####"> +
+ +
+
+
+
+ +
+ + + + + + + + + + + +
鍩庡競鍖哄煙灏忓尯鍚�鏃ユ湡娣诲姞浜�娣诲姞浠锋牸鎿嶄綔
+
+ + + <#list guidingPrices! as guid> + + + + + + + + + + + +
${guid.city!''}${guid.region!}${guid.name!}${guid.guidingDate?string("yyyy-MM-dd")}${guid.guidingPerson!}${guid.guidingValue!} + 缂栬緫 + 鍒犻櫎 + +
+
+
+ + + + + + + + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/guidingPrice.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/guidingPrice.ftl new file mode 100644 index 000000000..a6a5a6580 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/guidingPrice.ftl @@ -0,0 +1,111 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 娣诲姞鎸囧浠� + + + + +<#include "/common_js.ftl" /> + <#include "/layout/header.ftl" /> +
+
+
+ + + + + + + + + + + + + + + + + + +
+ 鍩庡競 + + +
+ 鍖哄煙 + + +
+ 灏忓尯鍚� + + +
+ 鏃ユ湡* + +
+ 鎸囧浠锋牸* + +
+ + + + + +
+ +
+ +
+
+
+ + + + + + + + + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/listQR.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/listQR.ftl new file mode 100644 index 000000000..781677cd0 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/listQR.ftl @@ -0,0 +1,125 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 鍧囦环绯荤粺-${cityDisplay!}浠锋牸璐ㄩ噺璺熻釜 + + + + + <#include "/common_js.ftl" /> + + + + + + + + + + + <#setting number_format="0.###########"/> + <#include "/layout/header.ftl" /> + +
+ + + + + + + +
鏌ヨ鏉′欢
+
+
+ <@d_city defaultValue="${cityDisplay!}" callback="changeCity" callbackField="display" showCountry="1" name="cityDisplay"/> +
+
+ + +  -  + + +
+ +
+
+ +
+ cur"> + 灏忓尯閲� + + cur"> + 骞冲潎鍙俊搴� + + cur"> + 娴嬭瘯妗堜緥鏁伴噺 + + cur"> + 妗堜緥宸瘮<10%閲� + + cur"> + 妗堜緥宸瘮<20%閲� + + cur"> + 妗堜緥宸瘮<30%閲� + + <#-- + class="cur" style="border: 1px solid #e0e0e0; padding: 8px 10px; margin-top: 10px;cursor:pointer;border-left:none;"> + 鎴挎簮鏍锋湰閲� + + --> +
+ +
+ +
+ + + + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/listSQR.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/listSQR.ftl new file mode 100644 index 000000000..2cd6e67b5 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/listSQR.ftl @@ -0,0 +1,110 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 鍧囦环绯荤粺-${cityDisplay!}鏍锋湰璐ㄩ噺璺熻釜 + + + + + <#include "/common_js.ftl" /> + + + + + + + + + + + <#setting number_format="0.###########"/> + <#include "/layout/header.ftl" /> + +
+ + + + + + + +
鏌ヨ鏉′欢
+
+
+ <@d_city defaultValue="${cityDisplay!}" callback="changeCity" callbackField="display" showCountry="1" name="cityDisplay"/> +
+
+ + +  -  + + +
+ +
+
+ + + +
+
+ + + + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/loadQRData.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/loadQRData.ftl new file mode 100644 index 000000000..c706f6731 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/loadQRData.ftl @@ -0,0 +1,102 @@ +
+ + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/loadSQRData.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/loadSQRData.ftl new file mode 100644 index 000000000..12e6095c6 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/loadSQRData.ftl @@ -0,0 +1,49 @@ +
+ + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/taskList.ftl b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/taskList.ftl new file mode 100644 index 000000000..f32d1be97 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/rentPriceUpdate/taskList.ftl @@ -0,0 +1,160 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 鍧囦环鏇存柊 + + + + + <#include "/common_js.ftl" /> + + + + + + + + + + +<#include "/layout/header.ftl" /> +
+ + + + + + + +
鏌ヨ鏉′欢
+
+ <@d_city defaultValue="${cityDisplay!}" callbackField="display" name="cityDisplay"/> + + + + + + <#if startDate??> + + <#else> + + + + <#if endDate??> + + <#else> + + + + 鍒涘缓浠诲姟 +
+
+ + + <#list calcCommands as it> + + + + + + + + + + + + + + + + +
鍩庡競鍖哄煙鏉垮潡灏忓尯鍚�璁$畻绫诲瀷璁$畻鏃堕棿鐘舵��浼樺厛绾�鐗╀笟绫诲瀷绉嶇被鍒涘缓鏃堕棿澶囨敞
${it.city!}${it.region!}${it.block!}${it.name} + <#if it.calcType ==0>鎸囧畾鏃ユ湡<#elseif it.calcType ==1 >鍏ㄩ儴璁$畻 + + <#if it.calcDate??> + ${it.calcDate?string("yyyy-MM-dd")} + <#if it.status ==0> + 鍒濆鐘舵�� + <#elseif it.status == 1> + 姝e湪澶勭悊 + <#elseif it.status == 2> + 澶勭悊瀹屾垚 + <#elseif it.status == 3> + 澶勭悊寮傚父 + + + <#if it.priority ==0> + 0 + <#elseif it.priority == 1> + 1 + <#elseif it.priority == 2> + 2 + + ${it.estateType!}${it.category!}${it.creationDate?string("yyyy-MM-dd HH:mm")}${it.remark!}
+ <@d_pager action = "taskList" currentPage = pager.currentPage controller = "rentPriceUpdate" totalPages = pager.totalPages/> +
+
+ + + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl b/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl index 69d4cd0f7..38771bc5d 100644 --- a/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/seawater/houseList.ftl @@ -72,7 +72,9 @@ - + + + - - + + + + + - + + - + + + - + + + + @@ -131,16 +149,18 @@ 闈㈢Н 鍗曚环 鎬讳环 - <#if sort?? && sort == 'p_date_desc'>鍙戝竷鏃堕棿鈫� - <#elseif sort?? && sort == 'p_date_asc'>鍙戝竷鏃堕棿鈫� - <#else>鍙戝竷鏃堕棿鈫� + <#if sort?? && sort == 'p_date_desc'>鍙戝竷鏃堕棿鈫� + <#elseif sort?? && sort == 'p_date_asc'>鍙戝竷鏃堕棿鈫� + <#else>鍙戝竷鏃堕棿鈫� 瀹ゅ巺鍗� 妤煎眰 鏉垮潡 瑁呬慨 鏈濆悜 + 鐗╀笟绫诲瀷 鏉ユ簮 + 鐘舵�� 鎿嶄綔 <#if seawaterList??> @@ -172,7 +192,7 @@ ${it.area!' '} - ${(it.totalPrice / it.area)?string('#')} + ${it.avgPrice!' '} ${it.totalPrice!' '} ${it.pDate!} ${it.room!'-'}/${it.hall!'-'}/${it.toilet!'-'} @@ -180,6 +200,7 @@ ${it.block!} ${it.fitment!} ${it.direction!} + ${it.estateType!} <#if it.url??> ${t_util.urlToSiteName(it.url!)!'鏈煡'} @@ -187,6 +208,13 @@ -- + + <#if it.status == 0> + 姝e父 + <#else> + 宸插垹闄� + + detail-name="${it.name}" <#if it.address??>detail-address="${it.address!}"<#if it.mobile??>detail-mobile="${it.mobile}"<#if it.category??>detail-category="${it.category}">璇︽儏 鍒犻櫎 @@ -217,7 +245,26 @@ - + + + @@ -383,6 +430,106 @@ } }); + $("#batchDeleteByParam").click(function(){ + if(confirm('纭瑕佸垹闄ゅ綋鍓嶆煡璇㈡潯浠朵笅鐨勬墍鏈夋暟鎹悧锛�')) { + var cityDisplay = $("input[name='cityDisplay']").val() + var region = $("select[name='region']").val() + var districtName = $("input[name='districtName']").val() + var estateType = $("input[name='estateType']").val() + var cat = $("select[name='cat']").val() + var status = $("select[name='status']").val() + var rootName = $("input[name='rootName']").val() + var startDate = $("input[name='startDate']").val() + var endDate = $("input[name='endDate']").val() + var startArea = $("input[name='startArea']").val() + var endArea = $("input[name='endArea']").val() + var startPrice = $("input[name='startPrice']").val() + var endPrice = $("input[name='endPrice']").val() + if(cityDisplay == "" || region == "" || districtName == ""){ + alert("鍩庡競銆佸尯鍩熴�佸皬鍖哄悕绉板繀濉�") + return false; + } + var data = {"cityDisplay": cityDisplay, "region": region, "districtName": districtName, + "estateType": estateType, "cat": cat, "status": status, + "rootName": rootName, "startDate": startDate, "endDate": endDate, + "startArea": startArea, "endArea": endArea, "startPrice": startPrice, + "endPrice": endPrice} + jQuery.ajax({ + type: 'POST', + url: '../seawater/batchDeleteByParams', + data: data, + dataType: 'json', + timeout: 100000, + success: function(model) { + alert("鎵归噺鍒犻櫎鎿嶄綔鎴愬姛"); + location.reload(); + }, + error : function() { + } + }); + } + }); + $("#batchUpdateEstateType").click(function(){ + $('#estateTypeModal').modal('show'); + }); + + $("#editSubmit").click(function(){ + var editEstateType = $("input[name='editEstateType']").val() + if(editEstateType == ""){ + alert("鐗╀笟绫诲瀷蹇呭~") + return false; + } + if(confirm('纭瑕佷慨鏀瑰綋鍓嶆煡璇㈡潯浠朵笅鐨勭墿涓氱被鍨嬪悧锛�')) { + var cityDisplay = $("input[name='cityDisplay']").val() + var region = $("select[name='region']").val() + var districtName = $("input[name='districtName']").val() + var estateType = $("input[name='estateType']").val() + var cat = $("select[name='cat']").val() + var status = $("select[name='status']").val() + var rootName = $("input[name='rootName']").val() + var startDate = $("input[name='startDate']").val() + var endDate = $("input[name='endDate']").val() + var startArea = $("input[name='startArea']").val() + var endArea = $("input[name='endArea']").val() + var startPrice = $("input[name='startPrice']").val() + var endPrice = $("input[name='endPrice']").val() + if(cityDisplay == "" || region == "" || districtName == ""){ + alert("鍩庡競銆佸尯鍩熴�佸皬鍖哄悕绉板繀濉�") + return false; + } + console.log(startArea) + console.log(endArea) + console.log(startPrice) + console.log(endPrice) + console.log(estateType) + if(startArea == "" && endArea == "" && startPrice == "" && endPrice == "" && estateType == ""){ + alert("闈㈢Н锛屼环鏍硷紝鐗╀笟绫诲瀷蹇呴』鑷冲皯涓�椤规湁鍊�") + return false; + } + + var data = {"cityDisplay": cityDisplay, "region": region, "districtName": districtName, + "estateType": estateType, "cat": cat, "status": status, + "rootName": rootName, "startDate": startDate, "endDate": endDate, + "startArea": startArea, "endArea": endArea, "startPrice": startPrice, + "endPrice": endPrice, "editEstateType": editEstateType} + jQuery.ajax({ + type: 'POST', + url: '../seawater/batchUpdateEstateType', + data: data, + dataType: 'json', + timeout: 100000, + success: function(model) { + alert("鎵归噺淇敼鎿嶄綔鎴愬姛"); + location.reload(); + }, + error : function() { + alert("鎵归噺淇敼鎿嶄綔澶辫触"); + location.reload(); + } + }); + $('#estateTypeModal').modal('hide'); + } + }); \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl b/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl index 53fd8f659..0d7cdf0e7 100644 --- a/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/yugudan/editEval.ftl @@ -241,10 +241,10 @@ - 鐗╀笟鍦板潃 + 浜ц瘉鍦板潃 - ${(evaluation.address)!} - + <#if evaluation?? && evaluation.certificateAddress??>${(evaluation.certificateAddress)!}<#else>${(evaluation.address)!} + diff --git a/projects/fangjia-bkoff/src/main/webapp/yugudan/evallist.ftl b/projects/fangjia-bkoff/src/main/webapp/yugudan/evallist.ftl index 26c02dc4c..c2fa9b4a8 100644 --- a/projects/fangjia-bkoff/src/main/webapp/yugudan/evallist.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/yugudan/evallist.ftl @@ -68,7 +68,7 @@ 鍩庡競 鍖哄煙 妤肩洏鍚嶇О - 浼颁环鍦板潃 + 浜ц瘉鍦板潃 甯愬彿 闈㈢Н 鎬讳环 @@ -83,7 +83,7 @@ ${it.city} ${it.region!} ${it.districtName!} - ${it.address!}<#if it.houseNum??>${it.houseNum!}鍙� + <#if it.certificateAddress??>${(it.certificateAddress)!}<#else>${(it.address)!}<#if it.houseNum??>${it.houseNum!}鍙� <#if it.unitNum?? && it.unitNum?length gt 1>${it.unitNum!}鍗曞厓 <#if it.buildType!=3 && it.buildType!=8 && it.buildType!=0> <#if it.floorDesc?? && it.floorDesc?length gt 1>${it.floorDesc!}灞� diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index e11cd2246..adfb2c7c8 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1847,6 +1847,9 @@ public class DataUtil { // // System.out.println(FourUpSixInto("258.393", 2)); + String href = encodePicUrl("62afd81d303c3f827833bb03", "o", "upload"); + String link = "http://static.fangjia.com/stc/fs/"+href; + System.out.println(link); } } diff --git a/projects/fangjia-coresite/Dockerfile b/projects/fangjia-coresite/Dockerfile index d0b8e61a1..18a266b75 100644 --- a/projects/fangjia-coresite/Dockerfile +++ b/projects/fangjia-coresite/Dockerfile @@ -1,2 +1,2 @@ -FROM 192.168.0.19:9012/tomcat6:latest +FROM 192.168.0.19:9012/tomcat6:220713 COPY target/*.war /home/goojia/webapps/ROOT.war \ No newline at end of file diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SaleLeadsController.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SaleLeadsController.java index 9596069f4..eab5dbd3c 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SaleLeadsController.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SaleLeadsController.java @@ -6,12 +6,20 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URLEncoder; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.logging.Handler; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import com.fangjia.core.util.DataUtil; +import com.fangjia.core.util.DingDingMessageUtil; +import com.fangjia.core.util.HttpRequest; +import com.fangjia.core.util.HttpRequestException; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import com.fangjia.coresite.model.JsonModel; @@ -27,9 +35,12 @@ import com.fangjia.service.common.CookieName; import com.fangjia.service.fangjia.NewsResourceService; import com.fangjia.service.fangjia.SaleLeadsService; import com.fangjia.service.mongo.fangjia.ServerAreaService; +import org.apache.log4j.Logger; public class SaleLeadsController { + private static final Logger LOG = Logger.getLogger(SaleLeadsController.class); + private SaleLeadsService saleLeadsService; private ServerAreaService serverAreaService; private NewsResourceService newsResourceService; @@ -41,6 +52,15 @@ public class SaleLeadsController { return model; } + /** + * 地数公众号咨询 + * @param model + * @return + */ + public Object didataApplyAction(SaleLeadsModel model) { + return model; + } + public Object doAddAction(SaleLeadsModel model, HttpServletRequest req, HttpServletResponse res) throws UnsupportedEncodingException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { @@ -129,8 +149,64 @@ public class SaleLeadsController { } saleLeads = converValue(saleLeads); + String body = + "业务需求:" + saleLeads.getBusiness() + " " + saleLeads.getOtherBusiness() + "\n\n" + + "姓名:" + saleLeads.getUname() + "\n\n" + + "城市:" + saleLeads.getCity() + "\n\n" + + "工作职能:" + saleLeads.getWork() + " " + saleLeads.getOtherWork() + "\n\n" + + "手机:" + saleLeads.getMobile() + "\n\n" + + "邮箱:" + saleLeads.getEmail() + "\n\n" + + "公司名称:" + saleLeads.getCompany() + "\n\n" + + "行业性质:" + saleLeads.getIndustryNature() + " " + saleLeads.getOtherIndustryNature() + "\n\n"; + body = body.replaceAll("null", "").replaceAll("其他,请填写", ""); + if (StringUtils.isNotBlank(refUrl) && refUrl.contains("bd_vid")) { + body += "线索来源:百度"; + } else if (StringUtils.isNotBlank(refUrl) && refUrl.contains("didata.wechat")) { + body += "线索来源: 地数公众号"; + } else { + body += "线索来源:房价网"; + } + try { + JSONObject requestBody = new JSONObject(); + requestBody.put("msgtype", "text"); + JSONObject markdown = new JSONObject(); + markdown.put("title", "销售线索"); + markdown.put("content", body); + requestBody.put("text", markdown); + HttpRequest request = new HttpRequest(); + final String ding_url = "https://oapi.dingtalk.com/robot/send?access_token=894650a12af9497bed599df1d5f93b21d6da5bbae45f35f18c5825580ebbdf64"; + Map headers = new HashMap(); + headers.put("Content-Type", "application/json; charset=utf-8"); + request.postWithBody(ding_url, requestBody, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace(); + } saleLeadsService.save(saleLeads); - + + // 对于百度广告过来的信息,进行回传 + try { + if (StringUtils.isNotBlank(refUrl) && refUrl.contains("bd_vid")) { + HttpRequest httpRequest = new HttpRequest(); + final String baiduUrl = "https://ocpc.baidu.com/ocpcapi/api/uploadConvertData"; + Map bdHeaders = new HashMap(); + bdHeaders.put("Content-Type", "application/json; charset=utf-8"); + + JSONObject requestBody = new JSONObject(); + // https://fengchao.baidu.com/fc/toolscenter/optimize/track/user/27002539 + requestBody.put("token", "EZGISwlOClRMEZErT2ThZTEjpLAzs2wE@9POwrGLqpRtQeGhvvCug1N3OK1tq3skk"); + JSONArray jsonArray = new JSONArray(); + JSONObject contentJson = new JSONObject(); + contentJson.put("logidUrl", refUrl); + contentJson.put("newType", "3"); + jsonArray.add(contentJson); + requestBody.put("conversionTypes", jsonArray); + String result = httpRequest.postWithBody(baiduUrl, requestBody, bdHeaders, "UTF-8"); + LOG.info("百度回传成功:" + result); + } + } catch (HttpRequestException e) { + e.printStackTrace(); + } + if(StringUtils.isNotBlank(saleLeads.getBusiness()) && saleLeads.getBusiness().contains("_")){ String[] businessArray = saleLeads.getBusiness().split("_"); for(String s : businessArray){ @@ -155,7 +231,12 @@ public class SaleLeadsController { if(StringUtils.isNotBlank(refUrl) && refUrl.contains("/resource/info")){ url = refUrl.substring(refUrl.indexOf("/resource/info"), refUrl.length()); - }else{ + } else if (StringUtils.isNotBlank(refUrl) && refUrl.contains("didata.wechat")) { // 地数公众号跳转 + url = "/enquire/didataSuccess"; + if (StringUtils.isNotBlank(refUrl)) { + url = url + "?backUrl=" + URLEncoder.encode(refUrl); + } + } else{ url = "/enquire/success"; if (StringUtils.isNotBlank(refUrl)) { url = url + "?backUrl=" + URLEncoder.encode(refUrl); @@ -224,6 +305,10 @@ public class SaleLeadsController { return model; } + public Object didataResultAction(SaleLeadsModel model) { + return model; + } + public SaleLeadsService getSaleLeadsService() { return saleLeadsService; } diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java index ef220df7d..b49974a26 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SolutionController.java @@ -1,6 +1,15 @@ package com.fangjia.coresite.controllers; +import com.fangjia.core.util.HttpRequest; +import com.fangjia.core.util.HttpRequestException; import com.fangjia.coresite.model.FangjiaModel; +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; +import org.apache.commons.lang3.StringUtils; + +import javax.servlet.http.HttpServletRequest; +import java.util.HashMap; +import java.util.Map; /** * 解决方案 控制器 @@ -14,7 +23,7 @@ public class SolutionController { * @param model * @return */ - public Object apiAction(FangjiaModel model) { + public Object apiAction(FangjiaModel model, HttpServletRequest request) { return model; } diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/filters/AntiSpiderFilter.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/filters/AntiSpiderFilter.java index b33d38545..db4f0a4db 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/filters/AntiSpiderFilter.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/filters/AntiSpiderFilter.java @@ -48,6 +48,11 @@ public class AntiSpiderFilter implements Filter { HttpServletRequest request = (HttpServletRequest) req; HttpServletResponse response = (HttpServletResponse) resp; + // 放开首页不做爬虫验证 + if (request.getRequestURL().toString().endsWith("fangjia.com") || request.getRequestURL().toString().endsWith("fangjia.com/")) { + chain.doFilter(req, resp); + return; + } if(request.getRequestURI().endsWith("/validate") || request.getRequestURI().contains("randomCode") || request.getRequestURI().contains("checkCode")|| request.getRequestURI().contains("yeepayCallBack")){ chain.doFilter(req, resp); return; diff --git a/projects/fangjia-coresite/src/main/resources/urlrewrite2.0.xml b/projects/fangjia-coresite/src/main/resources/urlrewrite2.0.xml index 5c8ab82c9..d96ca92fe 100644 --- a/projects/fangjia-coresite/src/main/resources/urlrewrite2.0.xml +++ b/projects/fangjia-coresite/src/main/resources/urlrewrite2.0.xml @@ -309,7 +309,17 @@ / - + + + + /didataApply + + + + + /didataSuccess + + diff --git a/projects/fangjia-coresite/src/main/webapp/company/about.ftl b/projects/fangjia-coresite/src/main/webapp/company/about.ftl index 778557075..5e222e0da 100644 --- a/projects/fangjia-coresite/src/main/webapp/company/about.ftl +++ b/projects/fangjia-coresite/src/main/webapp/company/about.ftl @@ -153,47 +153,47 @@
浠栦滑閮藉湪鍜屾垜浠悎浣�
- - - -<#-- --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
+<#-- --> +<#-- --> +<#-- --> +<#--<#– –>--> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- --> +<#-- -->
- +
diff --git a/projects/fangjia-coresite/src/main/webapp/evaluation/evalResult.ftl b/projects/fangjia-coresite/src/main/webapp/evaluation/evalResult.ftl index 425df6501..0c828feec 100644 --- a/projects/fangjia-coresite/src/main/webapp/evaluation/evalResult.ftl +++ b/projects/fangjia-coresite/src/main/webapp/evaluation/evalResult.ftl @@ -182,10 +182,10 @@ -
-


鏇村娆$殑璇勪及锛屾洿濂界殑浣撻獙

- -
+<#--
--> +<#--


鏇村娆$殑璇勪及锛屾洿濂界殑浣撻獙

--> +<#-- --> +<#--
--> diff --git a/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl b/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl index 293708039..37a1f6a1b 100644 --- a/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl +++ b/projects/fangjia-coresite/src/main/webapp/header/v5/_header.ftl @@ -18,7 +18,7 @@ -
  • 鏈�鏂拌祫璁�
  • +
  • 鏈�鏂拌祫璁�
  • -
  • APP涓嬭浇
  • +<#--
  • APP涓嬭浇
  • --> @@ -148,9 +148,9 @@ -
  • - APP涓嬭浇 -
  • +<#--
  • --> +<#-- APP涓嬭浇--> +<#--
  • --> diff --git a/projects/fangjia-coresite/src/main/webapp/index.ftl b/projects/fangjia-coresite/src/main/webapp/index.ftl index b5cb130e7..efec6e4b6 100644 --- a/projects/fangjia-coresite/src/main/webapp/index.ftl +++ b/projects/fangjia-coresite/src/main/webapp/index.ftl @@ -207,29 +207,29 @@ - +<#----> diff --git a/projects/fangjia-coresite/src/main/webapp/saleLeads/apply.ftl b/projects/fangjia-coresite/src/main/webapp/saleLeads/apply.ftl index c8c45e7cc..aab69abc0 100644 --- a/projects/fangjia-coresite/src/main/webapp/saleLeads/apply.ftl +++ b/projects/fangjia-coresite/src/main/webapp/saleLeads/apply.ftl @@ -555,27 +555,53 @@ $(".user_btn").click(function(){ scrollTo(0,0); return false; } - + + // 瑙e喅淇℃伅涓㈠け闂 + var work = $(".duty:visible").find("select").val() + if (work) { + work = work.replaceAll("璇烽�夋嫨鎮ㄧ殑宸ヤ綔鑱岃兘", "") + work = work.replaceAll("璇烽�夋嫨", "") + $("#work").val(work); + } + + var box = $(".demand p i.checked"); + if(box.length>0) { + $(".demand").parent().find(".error span").text("") + $(".demand").parent().find(".error").hide(); + var selectOtherFlag = false; + var content = [] + box.each(function () { + var s = $(this).parent().find("span").text(); + if (s.indexOf("璇峰~鍐�") > 0) { + selectOtherFlag = true; + } else { + content.push(s); + } + }) + $("#business").val(content.join("_")); + } + + /*if(!checkEmail()){ scrollBy(0,-300); return false; } - - if(!checkNature()){ - scrollBy(0,-200); - return false; - } if(!checkWork()){ scrollBy(0,-100); return false; } - + + if(!checkNature()){ + scrollBy(0,-200); + return false; + } if(mid=="1018"||mid=="1016"){ if(!checkOtherInfo()){ scrollBy(0,-100); return false; } - } */ + } + */ var cls = $(".agreement i").attr("class"); if(cls.indexOf("checked")>-1){ diff --git a/projects/fangjia-coresite/src/main/webapp/saleLeads/didataApply.ftl b/projects/fangjia-coresite/src/main/webapp/saleLeads/didataApply.ftl new file mode 100644 index 000000000..07a832de8 --- /dev/null +++ b/projects/fangjia-coresite/src/main/webapp/saleLeads/didataApply.ftl @@ -0,0 +1,471 @@ + + + + + +鏃剁┖浠峰�兼寲鎺樺钩鍙� -鍦版暟 didata.ai + + +<#include "../jsconstants.ftl" /> + + + + + + + + + + + + + + + + +<#escape x as x?html> +
    +
    + +
    +
    + +
    +
    +
    +
    +

    +
    鎴戜滑寰堜箰鎰忎负鎮ㄦ彁渚涘府鍔�
    +
    √ 璇﹀敖鐨勪骇鍝佸姛鑳藉強鏈嶅姟浠嬬粛
    +
    √ 閽堝鎮ㄦ墍澶勭殑琛屼笟涓庝笟鍔″満鏅紝鎻愪緵涓撲笟鐨勬湇鍔℃柟妗�
    +
    √ 鏍规嵁鎮ㄧ殑涓氬姟瑙勬ā鍜岄渶姹傦紝鎻愪緵浼樻儬鐨勪环鏍兼柟妗�
    +
    绱ф�ュ挩璇㈠彲鎷ㄦ墦400-188-3787
    +
    棰勭害鐢宠璇风暀涓嬫偍鐨勮仈绯绘柟寮忥紝鎴戜滑浼氬敖蹇笌鎮ㄨ仈绯�
    +
    +
    +
    + + + + +
    +
    +
    + + +
    +
    +
    + + +
    +
    +
    + + +
    +
    +
    + + +
    +
    +
    +
    +
    +
    + 鎻愪氦 +
    +
    +
    + + +<#include "../citys.ftl"/> + + + + + + + + + + + + + + + + + + + diff --git a/projects/fangjia-coresite/src/main/webapp/saleLeads/didataResult.ftl b/projects/fangjia-coresite/src/main/webapp/saleLeads/didataResult.ftl new file mode 100644 index 000000000..d81912435 --- /dev/null +++ b/projects/fangjia-coresite/src/main/webapp/saleLeads/didataResult.ftl @@ -0,0 +1,86 @@ + + + + + +鍜ㄨ淇℃伅鎻愪氦鎴愬姛 - 鍦版暟 didata.ai + + + + + + + + + + + + + + + + +
    +
    + +
    +
    + +
    +
    +
    +
    +

    鎮ㄧ殑淇℃伅宸叉彁浜ゆ垚鍔�

    + +

    鑻ユ偍鏈変换浣曢棶棰橈紝娆㈣繋鎷ㄦ墦瀹㈡湇鐢佃瘽锛�400-188-3787

    +
    +
    +
    +
    + + + + + + + + + diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/SeawaterQuery.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/SeawaterQuery.java new file mode 100644 index 000000000..66cc39114 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/SeawaterQuery.java @@ -0,0 +1,242 @@ +package com.fangjia.dal; +import org.sothis.core.util.Pager; + +import java.util.List; + +public class SeawaterQuery { + + private String id; + private String cat; + private String region; + private String districtName; + private String cityDisplay; + private Integer room; + private String direction; + private String fitment; + private String sort; + private Boolean order; + private Integer StartDate; + private Integer endDate; + private String estateType; + private Double startArea; + private Double endArea; + private Integer startPrice; + private Integer endPrice; + private Integer status; + private String rootName; + private String sortField; + private List regionAlias; + private List districtAlias; + private String[] keys; + private String table; + private Pager pager; + private String editEstateType; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCat() { + return cat; + } + + public void setCat(String cat) { + this.cat = cat; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getDistrictName() { + return districtName; + } + + public void setDistrictName(String districtName) { + this.districtName = districtName; + } + + public Integer getRoom() { + return room; + } + + public void setRoom(Integer room) { + this.room = room; + } + + public String getDirection() { + return direction; + } + + public void setDirection(String direction) { + this.direction = direction; + } + + public String getFitment() { + return fitment; + } + + public void setFitment(String fitment) { + this.fitment = fitment; + } + + public Boolean getOrder() { + return order; + } + + public void setOrder(Boolean order) { + this.order = order; + } + + public Integer getStartDate() { + return StartDate; + } + + public void setStartDate(Integer startDate) { + StartDate = startDate; + } + + public Integer getEndDate() { + return endDate; + } + + public void setEndDate(Integer endDate) { + this.endDate = endDate; + } + + public String getEstateType() { + return estateType; + } + + public void setEstateType(String estateType) { + this.estateType = estateType; + } + + public Integer getStartPrice() { + return startPrice; + } + + public void setStartPrice(Integer startPrice) { + this.startPrice = startPrice; + } + + public Integer getEndPrice() { + return endPrice; + } + + public void setEndPrice(Integer endPrice) { + this.endPrice = endPrice; + } + + public Integer getStatus() { + return status; + } + + public void setStatus(Integer status) { + this.status = status; + } + + public String getSortField() { + return sortField; + } + + public void setSortField(String sortField) { + this.sortField = sortField; + } + + public List getRegionAlias() { + return regionAlias; + } + + public void setRegionAlias(List regionAlias) { + this.regionAlias = regionAlias; + } + + public List getDistrictAlias() { + return districtAlias; + } + + public void setDistrictAlias(List districtAlias) { + this.districtAlias = districtAlias; + } + + public String[] getKeys() { + return keys; + } + + public void setKeys(String[] keys) { + this.keys = keys; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String getRootName() { + return rootName; + } + + public void setRootName(String rootName) { + this.rootName = rootName; + } + + public Double getStartArea() { + return startArea; + } + + public void setStartArea(Double startArea) { + this.startArea = startArea; + } + + public Double getEndArea() { + return endArea; + } + + public void setEndArea(Double endArea) { + this.endArea = endArea; + } + + public Pager getPager() { + return pager; + } + + public void setPager(Pager pager) { + this.pager = pager; + } + + public String getSort() { + return sort; + } + + public void setSort(String sort) { + this.sort = sort; + } + + public String getCityDisplay() { + return cityDisplay; + } + + public void setCityDisplay(String cityDisplay) { + this.cityDisplay = cityDisplay; + } + + public String getEditEstateType() { + return editEstateType; + } + + public void setEditEstateType(String editEstateType) { + this.editEstateType = editEstateType; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java index 8daca61ed..6cb65c498 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/enums/SalesLeadEnum.java @@ -22,7 +22,8 @@ public enum SalesLeadEnum { DOWN_APPLY("1016", "下载申请"), FDANG_DIAO_BAO_GAO("1017", "房产专业调查评估报告"), BUSINESS_ASK("1018", "业务咨询"), - HOUSE_EVAL("1019", "房产评估"); + HOUSE_EVAL("1019", "房产评估"), + DIDATA("1020", "地数平台"); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java index e7a406dc8..2058cd9aa 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java @@ -102,6 +102,7 @@ public class PaiMai implements MongoEntity { private String houseNum; private String unitNum; private String roomNum; + private String estateType; @Id @GeneratedValue @@ -492,6 +493,7 @@ public class PaiMai implements MongoEntity { this.matchEstateType = matchEstateType; } + @Column(name = "houseNum") public String getHouseNum() { return houseNum; } @@ -500,6 +502,7 @@ public class PaiMai implements MongoEntity { this.houseNum = houseNum; } + @Column(name = "unitNum") public String getUnitNum() { return unitNum; } @@ -508,6 +511,7 @@ public class PaiMai implements MongoEntity { this.unitNum = unitNum; } + @Column(name = "roomNum") public String getRoomNum() { return roomNum; } @@ -515,4 +519,13 @@ public class PaiMai implements MongoEntity { public void setRoomNum(String roomNum) { this.roomNum = roomNum; } + + @Column(name = "estateType") + public String getEstateType() { + return estateType; + } + + public void setEstateType(String estateType) { + this.estateType = estateType; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java index fd659f31b..7b2405df1 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/AverageStatistics.java @@ -95,10 +95,10 @@ public class AverageStatistics implements MongoEntity { private Double tuboshiEsfDiff; /**中原地产二手房均价*/ - private Integer centanetNewPrice; - private Double centanetNewDiff; - private Integer centanetEsfPrice; - private Double centanetEsfDiff; + private Integer zhongyuanNewPrice; + private Double zhongyuanNewDiff; + private Integer zhongyuanEsfPrice; + private Double zhongyuanEsfDiff; @@ -155,9 +155,18 @@ public class AverageStatistics implements MongoEntity { private Integer zhugezhaofangNewPrice; private Double zhugezhaofangNewDiff; - + /**吉屋*/ + private Integer jiwuEsfPrice; + private Double jiwuEsfDiff; + private Integer jiwuNewPrice; + private Double jiwuNewDiff; + + /**365*/ + private Integer tsfhouseEsfPrice; + private Double tsfhouseEsfDiff; + private Integer tsfhouseNewPrice; + private Double tsfhouseNewDiff; - private String level;//价格评级 private Integer lastAvgPrice;//最终价 private String maxDeviate;//偏离度最大的网站,房价网-云估价-房估估 @@ -578,22 +587,22 @@ public class AverageStatistics implements MongoEntity { this.fangguguNewPrice = fangguguNewPrice; } - @Column(name = "centanet_new_price") - public Integer getCentanetNewPrice() { - return centanetNewPrice; + @Column(name = "zhongyuan_new_price") + public Integer getZhongyuanNewPrice() { + return zhongyuanNewPrice; } - public void setCentanetNewPrice(Integer centanetNewPrice) { - this.centanetNewPrice = centanetNewPrice; + public void setZhongyuanNewPrice(Integer zhongyuanNewPrice) { + this.zhongyuanNewPrice = zhongyuanNewPrice; } - @Column(name = "centanet_esf_price") - public Integer getCentanetEsfPrice() { - return centanetEsfPrice; + @Column(name = "zhongyuan_esf_price") + public Integer getZhongyuanEsfPrice() { + return zhongyuanEsfPrice; } - public void setCentanetEsfPrice(Integer centanetEsfPrice) { - this.centanetEsfPrice = centanetEsfPrice; + public void setZhongyuanEsfPrice(Integer zhongyuanEsfPrice) { + this.zhongyuanEsfPrice = zhongyuanEsfPrice; } @Column(name = "lianjia_esf_price") @@ -659,22 +668,22 @@ public class AverageStatistics implements MongoEntity { this.tuboshiEsfDiff = tuboshiEsfDiff; } - @Column(name = "centanet_new_diff") - public Double getCentanetNewDiff() { - return centanetNewDiff; + @Column(name = "zhongyuan_new_diff") + public Double getZhongyuanNewDiff() { + return zhongyuanNewDiff; } - public void setCentanetNewDiff(Double centanetNewDiff) { - this.centanetNewDiff = centanetNewDiff; + public void setZhongyuanNewDiff(Double zhongyuanNewDiff) { + this.zhongyuanNewDiff = zhongyuanNewDiff; } - @Column(name = "centanet_esf_diff") - public Double getCentanetEsfDiff() { - return centanetEsfDiff; + @Column(name = "zhongyuan_esf_diff") + public Double getZhongyuanEsfDiff() { + return zhongyuanEsfDiff; } - public void setCentanetEsfDiff(Double centanetEsfDiff) { - this.centanetEsfDiff = centanetEsfDiff; + public void setZhongyuanEsfDiff(Double zhongyuanEsfDiff) { + this.zhongyuanEsfDiff = zhongyuanEsfDiff; } @Column(name = "master_district_name") @@ -1074,19 +1083,76 @@ public class AverageStatistics implements MongoEntity { public void setFangtanEsfDiff(Double fangtanEsfDiff) { this.fangtanEsfDiff = fangtanEsfDiff; } - - - - - - - - - - - - - - - + + @Column(name = "jiwu_esf_price") + public Integer getJiwuEsfPrice() { + return jiwuEsfPrice; + } + + public void setJiwuEsfPrice(Integer jiwuEsfPrice) { + this.jiwuEsfPrice = jiwuEsfPrice; + } + + @Column(name = "jiwu_esf_diff") + public Double getJiwuEsfDiff() { + return jiwuEsfDiff; + } + + public void setJiwuEsfDiff(Double jiwuEsfDiff) { + this.jiwuEsfDiff = jiwuEsfDiff; + } + + @Column(name = "jiwu_new_price") + public Integer getJiwuNewPrice() { + return jiwuNewPrice; + } + + public void setJiwuNewPrice(Integer jiwuNewPrice) { + this.jiwuNewPrice = jiwuNewPrice; + } + + @Column(name = "jiwu_new_diff") + public Double getJiwuNewDiff() { + return jiwuNewDiff; + } + + public void setJiwuNewDiff(Double jiwuNewDiff) { + this.jiwuNewDiff = jiwuNewDiff; + } + + @Column(name = "365house_esf_price") + public Integer getTsfhouseEsfPrice() { + return tsfhouseEsfPrice; + } + + public void setTsfhouseEsfPrice(Integer tsfhouseEsfPrice) { + this.tsfhouseEsfPrice = tsfhouseEsfPrice; + } + + @Column(name = "365house_esf_diff") + public Double getTsfhouseEsfDiff() { + return tsfhouseEsfDiff; + } + + public void setTsfhouseEsfDiff(Double tsfhouseEsfDiff) { + this.tsfhouseEsfDiff = tsfhouseEsfDiff; + } + + @Column(name = "365house_new_price") + public Integer getTsfhouseNewPrice() { + return tsfhouseNewPrice; + } + + public void setTsfhouseNewPrice(Integer tsfhouseNewPrice) { + this.tsfhouseNewPrice = tsfhouseNewPrice; + } + + @Column(name = "365house_new_diff") + public Double getTsfhouseNewDiff() { + return tsfhouseNewDiff; + } + + public void setTsfhouseNewDiff(Double tsfhouseNewDiff) { + this.tsfhouseNewDiff = tsfhouseNewDiff; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/SqmetreRent.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/SqmetreRent.java index a5bb8b9a4..839d829bc 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/SqmetreRent.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/SqmetreRent.java @@ -22,18 +22,16 @@ public class SqmetreRent implements MongoEntity { private String city; private String region; private String districtName; - private Double avgPrice; - private int markup; + private Integer avgPrice; + private Integer calcAvgPrice; + private Double markup; private Date weekend; private String cat; private Date creationDate; private Date modificationDate; - private String sampleEnum; - private double credit; - private String cluster; - - private Double oldValue; - private String oldSampleEnum; + private Double credit; + private String valueType; + private Integer priceToRent; @Id @GeneratedValue @@ -76,22 +74,47 @@ public class SqmetreRent implements MongoEntity { } @Column(name = "avg_price") - public Double getAvgPrice() { + public Integer getAvgPrice() { return avgPrice; } - public void setAvgPrice(Double avgPrice) { + public void setAvgPrice(Integer avgPrice) { this.avgPrice = avgPrice; } - + @Column(name = "calc_avg_price") + public Integer getCalcAvgPrice() { + return calcAvgPrice; + } + + public void setCalcAvgPrice(Integer calcAvgPrice) { + this.calcAvgPrice = calcAvgPrice; + } + + @Column(name = "value_type") + public String getValueType() { + return valueType; + } + + public void setValueType(String valueType) { + this.valueType = valueType; + } + + @Column(name = "price_to_rent") + public Integer getPriceToRent() { + return priceToRent; + } + + public void setPriceToRent(Integer priceToRent) { + this.priceToRent = priceToRent; + } @Column(name = "markup") - public int getMarkup() { + public Double getMarkup() { return markup; } - public void setMarkup(int markup) { + public void setMarkup(Double markup) { this.markup = markup; } @@ -123,52 +146,15 @@ public class SqmetreRent implements MongoEntity { } - @Column(name = "sample_enum") - public String getSampleEnum() { - return sampleEnum; - } - - public void setSampleEnum(String sampleEnum) { - this.sampleEnum = sampleEnum; - } - - @Column(name = "credit") - public double getCredit() { + public Double getCredit() { return credit; } - public void setCredit(double credit) { + public void setCredit(Double credit) { this.credit = credit; } - @Column(name = "cluster") - public String getCluster() { - return cluster; - } - - public void setCluster(String cluster) { - this.cluster = cluster; - } - - @Column(name = "old_value") - public Double getOldValue() { - return oldValue; - } - - public void setOldValue(Double oldValue) { - this.oldValue = oldValue; - } - - @Column(name = "old_sample_enum") - public String getOldSampleEnum() { - return oldSampleEnum; - } - - public void setOldSampleEnum(String oldSampleEnum) { - this.oldSampleEnum = oldSampleEnum; - } - @Column(name = "cat") public String getCat() { return cat; diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java index 12cb8ca9b..ffb9f957f 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java @@ -83,6 +83,8 @@ public class Loudong implements MongoEntity { private Date modifyDate; /**户数*/ private Integer households; + /**一梯几户的户*/ + private Integer roomCount; /** 总建筑面积 */ private Double totalAreas; /** 开盘时间 */ diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/EvaluateRatioDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/EvaluateRatioDao.java new file mode 100644 index 000000000..eaef95a4b --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/EvaluateRatioDao.java @@ -0,0 +1,11 @@ +package com.fangjia.dal.mongo.open.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.open.model.EvaluateRatio; + +/** + * @author shenjian + * @since 2022/7/27 + */ +public interface EvaluateRatioDao extends MongoDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/EvaluateRatioDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/EvaluateRatioDaoImpl.java new file mode 100644 index 000000000..f1c34d2af --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/EvaluateRatioDaoImpl.java @@ -0,0 +1,13 @@ +package com.fangjia.dal.mongo.open.dao.impl; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.open.dao.EvaluateRatioDao; +import com.fangjia.dal.mongo.open.model.EvaluateRatio; + +/** + * @author shenjian + * @since 2022/7/27 + */ +public class EvaluateRatioDaoImpl extends MongoDaoImpl implements EvaluateRatioDao { + +} \ No newline at end of file diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/EvaluateRatio.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/EvaluateRatio.java new file mode 100644 index 000000000..b91857b30 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/EvaluateRatio.java @@ -0,0 +1,72 @@ +package com.fangjia.dal.mongo.open.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; + +/** + * 估价系数 + * + * @author shenjian + * @since 2022/7/27 + */ +@Entity +@Table(name = "business.evaluate_ratio") +public class EvaluateRatio implements MongoEntity { + + private static final long serialVersionUID = 5299813614465690722L; + private String id; + private String category; + private String city; + private String region; + private Double ratio; + + @Id + @GeneratedValue + @Column(name = "_id", unique = true, nullable = false) + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "category") + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "ratio") + public Double getRatio() { + return ratio; + } + + public void setRatio(Double ratio) { + this.ratio = ratio; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDao.java index edbd0546e..ecb61ff6f 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDao.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDao.java @@ -2,6 +2,7 @@ package com.fangjia.dal.seawater.dao; import java.util.List; +import com.fangjia.dal.SeawaterQuery; import com.fangjia.dal.seawater.model.Seawater; /** @@ -18,4 +19,12 @@ public interface SeawaterDao extends com.fangjia.dal.SeawaterDao { Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root, String sort, boolean order, int start, int limit , String table); List findHouses(String city, String region, List alias, Integer minPrice, Integer maxPrice, Integer limit); + + List findSeawatersByQuery(SeawaterQuery query); + + long findSeawatersCountByQuery(SeawaterQuery query); + + void batchDeleteByParams(SeawaterQuery query); + + void batchUpdateEstateType(SeawaterQuery query); } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 69108673a..b0fc86c99 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -2,6 +2,7 @@ package com.fangjia.dal.seawater.dao; import java.util.*; +import com.fangjia.dal.SeawaterQuery; import org.apache.commons.codec.binary.Base64; import org.apache.commons.lang.StringUtils; @@ -11,6 +12,7 @@ import com.fangjia.dal.seawater.model.Seawater; import net.sf.json.JSONArray; import net.sf.json.JSONObject; +import org.sothis.core.util.Pager; public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl implements SeawaterDao { @@ -29,7 +31,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i @Override public Seawater findByIdFromEs(String id,String table) { Seawater seawater = null; - List jsonList = queryList(id,null,null,null,null,null,null,null,null, null, null, null, null, null,null,null,false,-1,-1,table); + SeawaterQuery query = new SeawaterQuery(); + query.setId(id); + query.setTable(table); + Pager pager = new Pager(); + pager.setStartRow(-1); + pager.setPageSize(-1); + query.setPager(pager); + List jsonList = queryList(query); if(jsonList == null){ return null; }else{ @@ -44,7 +53,28 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i @Override public List findSeawatersFromEs(String category, String city, String region, String name, List alias, List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root, String sort, boolean order, int start, int limit, String table) { List seawaterList= new ArrayList(); - List jsonList = queryList(null,category,city,region,name,alias,regionAlias,startDate,endDate, startArea, endArea, startPrice, endPrice, room,root,sort,order,start,limit,table); + SeawaterQuery query = new SeawaterQuery(); + query.setCat(category); + query.setCityDisplay(city); + query.setRegion(region); + query.setDistrictName(name); + query.setDistrictAlias(alias); + query.setRegionAlias(regionAlias); + query.setStartDate(startDate); + query.setEndDate(endDate); + query.setStartArea(startArea); + query.setEndArea(endArea); + query.setStartPrice(startPrice); + query.setEndPrice(endPrice); + query.setRoom(room); + query.setRootName(root); + query.setOrder(order); + Pager pager = new Pager(); + pager.setStartRow(start); + pager.setPageSize(limit); + query.setPager(pager); + query.setTable(table); + List jsonList = queryList(query); if(jsonList == null){ return null; }else{ @@ -53,6 +83,23 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i return seawaterList; } + @Override + public List findSeawatersByQuery(SeawaterQuery query){ + List seawaterList; + List jsonList = queryList(query); + if(jsonList == null){ + return null; + }else{ + seawaterList = wrapSeawater(jsonList); + } + return seawaterList; + } + + @Override + public long findSeawatersCountByQuery(SeawaterQuery query) { + int count = queryCount(query); + return count; + } @Override public List findHouses(String city, String region, List alias, Integer minPrice, Integer maxPrice, Integer limit) { @@ -136,6 +183,42 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i } } + @Override + public void batchDeleteByParams(SeawaterQuery query) { + HttpRequest request = new HttpRequest(); + + JSONObject reqBody = wrapQueryParams(query); + + JSONObject script = new JSONObject(); + script.put("source", "ctx._source.status=1"); + script.put("lang", "painless"); + reqBody.put("script", script); + + try { + request.postWithBody("http://es.fangjia.com:9200/seawater/_update_by_query?conflicts=proceed&wait_for_completion=false", reqBody, this.buildHeader(), "UTF-8"); + } catch (HttpRequestException e) { + e.printStackTrace(); + } + } + + @Override + public void batchUpdateEstateType(SeawaterQuery query) { + HttpRequest request = new HttpRequest(); + + JSONObject reqBody = wrapQueryParams(query); + + JSONObject script = new JSONObject(); + script.put("source", "ctx._source.estate_type="+"'"+query.getEditEstateType()+"'"); + script.put("lang", "painless"); + reqBody.put("script", script); + + try { + request.postWithBody("http://es.fangjia.com:9200/seawater/_update_by_query?conflicts=proceed&wait_for_completion=false", reqBody, this.buildHeader(), "UTF-8"); + } catch (HttpRequestException e) { + e.printStackTrace(); + } + } + private JSONObject addMatch(String name, String value){ JSONObject term = new JSONObject(); JSONObject match = new JSONObject(); @@ -234,6 +317,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i }catch(Exception e){ seawater.setHall(0); } + try{ + seawater.setToilet(source.getInt("toilet")); + }catch(Exception e){ + seawater.setToilet(0); + } try{ seawater.setpDate(source.getInt("p_date")); }catch(Exception e){ @@ -256,6 +344,9 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(source.containsKey("build_type")){ seawater.setBuildType(source.getString("build_type")); } + if(source.containsKey("status")){ + seawater.setStatus(source.getInt("status")); + } seawaterList.add(seawater); } return seawaterList; @@ -431,29 +522,381 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i } return null; } - - private List queryList(String id,String category, String city, String region, String name, List alias, - List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){ - JSONObject json = query(id,category,city,region,name,alias,regionAlias,startDate,endDate, startArea, endArea, startPrice, endPrice, room,root,sort,order,start,limit,table); + + private JSONObject query(SeawaterQuery query){ + long time1 = System.currentTimeMillis(); + JSONObject reqBody = new JSONObject(); + JSONObject q = new JSONObject(); + JSONObject queryJson = new JSONObject(); + List jsonList = new ArrayList(); + if(StringUtils.isNotBlank(query.getId())) { + JSONObject param = new JSONObject(); + param.put("_id", query.getId()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + if(StringUtils.isNotBlank(query.getCat())) { + JSONObject param = new JSONObject(); + param.put("cat", query.getCat()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + + Set namesList = new LinkedHashSet(); + if(StringUtils.isNotBlank(query.getDistrictName())) { + namesList.add(query.getDistrictName()); + } + if (query.getDistrictAlias() != null) { + namesList.addAll(query.getDistrictAlias()); + } + if(namesList != null && namesList.size() > 0){ + JSONObject param = new JSONObject(); + param.put("district_name", namesList); + JSONObject temp = new JSONObject(); + temp.put("terms", param); + jsonList.add(temp); + } + + Set regionsList = new LinkedHashSet(); + if(StringUtils.isNotBlank(query.getRegion())) { + regionsList.add(query.getRegion()); + } + if (query.getRegionAlias() != null) { + regionsList.addAll(query.getRegionAlias()); + } + if(regionsList != null && regionsList.size() > 0){ + JSONObject param = new JSONObject(); + param.put("region", regionsList); + JSONObject temp = new JSONObject(); + temp.put("terms", param); + jsonList.add(temp); + } + + if(StringUtils.isNotBlank(query.getCityDisplay())){ + JSONObject param = new JSONObject(); + param.put("city", query.getCityDisplay()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + if(StringUtils.isNotBlank(query.getEstateType())) { + JSONObject param = new JSONObject(); + param.put("estate_type", query.getEstateType()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + if(query.getRoom() != null) { + JSONObject param = new JSONObject(); + param.put("room", query.getRoom()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + if(StringUtils.isNotBlank(query.getRootName())) { + JSONObject param = new JSONObject(); + param.put("root", query.getRootName()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + + if(query.getStartDate() != null || query.getEndDate() != null){ + JSONObject param = new JSONObject(); + JSONObject temp = new JSONObject(); + JSONObject temp2 = new JSONObject(); + if(query.getStartDate() != null){ + temp2.put("gte", query.getStartDate()); + } + if(query.getEndDate() != null){ + temp2.put("lte", query.getEndDate()); + } + param.put("p_date", temp2); + temp.put("range", param); + jsonList.add(temp); + } + + + if(query.getStartArea() != null || query.getEndArea() != null){ + JSONObject param = new JSONObject(); + JSONObject temp = new JSONObject(); + JSONObject temp2 = new JSONObject(); + if(query.getStartArea() != null){ + temp2.put("gte", query.getStartArea()); + } + if(query.getEndArea() != null){ + temp2.put("lte", query.getEndArea()); + } + param.put("area", temp2); + temp.put("range", param); + jsonList.add(temp); + } + + if(query.getStartPrice() != null || query.getEndPrice() != null){ + JSONObject param = new JSONObject(); + JSONObject temp = new JSONObject(); + JSONObject temp2 = new JSONObject(); + if(query.getStartPrice() != null){ + temp2.put("gte", query.getStartPrice()); + } + if(query.getEndPrice() != null){ + temp2.put("lte", query.getEndPrice()); + } + param.put("avg_price", temp2); + temp.put("range", param); + jsonList.add(temp); + } + + if(StringUtils.isNotBlank(query.getSort())){ + JSONObject jsonOrder = new JSONObject(); + if(query.getOrder()){ + jsonOrder.put("order", "asc"); + }else{ + jsonOrder.put("order", "desc"); + } + JSONObject jsonSort = new JSONObject(); + jsonSort.put(query.getSortField(), jsonOrder); + List sortJsonList = new ArrayList(); + sortJsonList.add(jsonSort); + reqBody.put("sort", sortJsonList); + } + + if(query.getPager().getStartRow() >= 0){ + reqBody.put("from", query.getPager().getStartRow()); + } + if(query.getPager().getPageSize() >= 0){ + reqBody.put("size", query.getPager().getPageSize()); + } + + if(query.getStatus() != null){ + JSONObject param = new JSONObject(); + param.put("status", query.getStatus()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + + queryJson.put("must", jsonList); + q.put("bool", queryJson); + reqBody.put("query", q); + //用户名:密码 + Base64 token = new Base64(); + String authenticationEncoding = token.encodeAsString(new String(""+ES_USER+":"+ES_PASSWORD+"").getBytes()); + HttpRequest request = new HttpRequest(); + Map headers = new HashMap(); + headers.put("Content-Type", "application/json"); + headers.put("Authorization", "Basic "+authenticationEncoding+""); + try { + String response = request.postWithBody("http://es.fangjia.com:9200/"+query.getTable()+"/_search?rest_total_hits_as_int=true", reqBody, headers, "UTF-8"); + JSONObject json = JSONObject.fromObject(response); + long time2 = System.currentTimeMillis(); + System.out.println(time2-time1); + return json; + } catch (HttpRequestException e) { + e.printStackTrace(); + } + return null; + } + +// private List queryList(String id,String category, String city, String region, String name, List alias, +// List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){ +// JSONObject json = query(id,category,city,region,name,alias,regionAlias,startDate,endDate, startArea, endArea, startPrice, endPrice, room,root,sort,order,start,limit,table); +// if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("hits")){ +// return null; +// } +// JSONArray hits = json.getJSONObject("hits").getJSONArray("hits"); +// List results = new ArrayList(); +// for(int i=0 ;i < hits.size(); i++){ +// JSONObject jsonObject = hits.getJSONObject(i); +// results.add(jsonObject); +// } +// return results; +// } + + private List queryList(SeawaterQuery query){ + JSONObject json = query(query); if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("hits")){ - return null; - } - JSONArray hits = json.getJSONObject("hits").getJSONArray("hits"); - List results = new ArrayList(); - for(int i=0 ;i < hits.size(); i++){ - JSONObject jsonObject = hits.getJSONObject(i); - results.add(jsonObject); - } - return results; + return null; + } + JSONArray hits = json.getJSONObject("hits").getJSONArray("hits"); + List results = new ArrayList(); + for(int i=0 ;i < hits.size(); i++){ + JSONObject jsonObject = hits.getJSONObject(i); + results.add(jsonObject); + } + return results; } private int queryCount(String id,String category, String city, String region, String name, List alias, List regionAlias, Integer startDate, Integer endDate, Integer room, String root,String sort, boolean order, int start, int limit,String table){ - JSONObject json = query(id,category,city,region,name,alias,regionAlias,startDate,endDate,null, null, null, null, room,root,sort,order,start,limit,table); + SeawaterQuery query = new SeawaterQuery(); + query.setCat(category); + query.setCityDisplay(city); + query.setRegion(region); + query.setDistrictName(name); + query.setDistrictAlias(alias); + query.setRegionAlias(regionAlias); + query.setStartDate(startDate); + query.setEndDate(endDate); + query.setRoom(room); + query.setRootName(root); + query.setOrder(order); + Pager pager = new Pager(); + pager.setStartRow(start); + pager.setPageSize(limit); + query.setPager(pager); + query.setTable(table); + JSONObject json = query(query); if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("total")){ return 0; } return json.getJSONObject("hits").getInt("total"); } + private int queryCount(SeawaterQuery query){ + JSONObject json = query(query); + if(!json.containsKey("hits") || !json.getJSONObject("hits").containsKey("total")){ + return 0; + } + return json.getJSONObject("hits").getInt("total"); + } + + private JSONObject wrapQueryParams(SeawaterQuery query){ + JSONObject reqBody = new JSONObject(); + JSONObject q = new JSONObject(); + JSONObject queryJson = new JSONObject(); + List jsonList = new ArrayList(); + if(StringUtils.isNotBlank(query.getId())) { + JSONObject param = new JSONObject(); + param.put("_id", query.getId()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + if(StringUtils.isNotBlank(query.getCat())) { + JSONObject param = new JSONObject(); + param.put("cat", query.getCat()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + + Set namesList = new LinkedHashSet(); + if(StringUtils.isNotBlank(query.getDistrictName())) { + namesList.add(query.getDistrictName()); + } + if(namesList != null && namesList.size() > 0){ + JSONObject param = new JSONObject(); + param.put("district_name", namesList); + JSONObject temp = new JSONObject(); + temp.put("terms", param); + jsonList.add(temp); + } + + Set regionsList = new LinkedHashSet(); + if(StringUtils.isNotBlank(query.getRegion())) { + regionsList.add(query.getRegion()); + } + if (query.getRegionAlias() != null) { + regionsList.addAll(query.getRegionAlias()); + } + if(regionsList != null && regionsList.size() > 0){ + JSONObject param = new JSONObject(); + param.put("region", regionsList); + JSONObject temp = new JSONObject(); + temp.put("terms", param); + jsonList.add(temp); + } + + if(StringUtils.isNotBlank(query.getCityDisplay())){ + JSONObject param = new JSONObject(); + param.put("city", query.getCityDisplay()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + if(StringUtils.isNotBlank(query.getEstateType())) { + JSONObject param = new JSONObject(); + param.put("estate_type", query.getEstateType()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + if(query.getRoom() != null) { + JSONObject param = new JSONObject(); + param.put("room", query.getRoom()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + if(StringUtils.isNotBlank(query.getRootName())) { + JSONObject param = new JSONObject(); + param.put("root", query.getRootName()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + + if(query.getStartDate() != null || query.getEndDate() != null){ + JSONObject param = new JSONObject(); + JSONObject temp = new JSONObject(); + JSONObject temp2 = new JSONObject(); + if(query.getStartDate() != null){ + temp2.put("gte", query.getStartDate()); + } + if(query.getEndDate() != null){ + temp2.put("lte", query.getEndDate()); + } + param.put("p_date", temp2); + temp.put("range", param); + jsonList.add(temp); + } + + + if(query.getStartArea() != null || query.getEndArea() != null){ + JSONObject param = new JSONObject(); + JSONObject temp = new JSONObject(); + JSONObject temp2 = new JSONObject(); + if(query.getStartArea() != null){ + temp2.put("gte", query.getStartArea()); + } + if(query.getEndArea() != null){ + temp2.put("lte", query.getEndArea()); + } + param.put("area", temp2); + temp.put("range", param); + jsonList.add(temp); + } + + if(query.getStartPrice() != null || query.getEndPrice() != null){ + JSONObject param = new JSONObject(); + JSONObject temp = new JSONObject(); + JSONObject temp2 = new JSONObject(); + if(query.getStartPrice() != null){ + temp2.put("gte", query.getStartPrice()); + } + if(query.getEndPrice() != null){ + temp2.put("lte", query.getEndPrice()); + } + param.put("avg_price", temp2); + temp.put("range", param); + jsonList.add(temp); + } + + if(query.getStatus() != null){ + JSONObject param = new JSONObject(); + param.put("status", query.getStatus()); + JSONObject temp = new JSONObject(); + temp.put("term", param); + jsonList.add(temp); + } + + queryJson.put("must", jsonList); + q.put("bool", queryJson); + reqBody.put("query", q); + return reqBody; + } + } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java index 8283df958..4b72f411c 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java @@ -102,8 +102,10 @@ public class FjtEvaluation implements SqlEntity{ private String finishDateStr;//竣工时间 private String useType;//房屋用途 + private String certificateAddress; //产证地址 private Date downPdfDate;//下载报告时间 + private Integer loan ;//是否续贷 1:是 0:否 @GeneratedValue(strategy = GenerationType.AUTO) @Id @@ -753,6 +755,16 @@ public class FjtEvaluation implements SqlEntity{ } + @Column(name = "certificate_address") + public String getCertificateAddress() { + return certificateAddress; + } + + public void setCertificateAddress(String certificateAddress) { + this.certificateAddress = certificateAddress; + } + + @Column(name = "down_pdf_date") public Date getDownPdfDate() { return downPdfDate; @@ -762,6 +774,15 @@ public class FjtEvaluation implements SqlEntity{ this.downPdfDate = downPdfDate; } + @Column(name = "loan") + public Integer getLoan() { + return loan; + } + + public void setLoan(Integer loan) { + this.loan = loan; + } + @Override public String toString() { return "FjtEvaluation{" + diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java index a22ef8332..acc9fcd36 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ReportFileAttach.java @@ -11,7 +11,7 @@ public class ReportFileAttach implements SqlEntity { private Integer userId; private Integer evaluationId; private String queryId; - private Integer fileClass; + private Integer fileClass;//1:询价单 2:产证 3:房屋照片 4:其它附件 5:产调报告或权籍 private String fileName; private String fileId; diff --git a/projects/fangjia-datastore/pom.xml b/projects/fangjia-datastore/pom.xml index e86529d96..5c5fb279c 100644 --- a/projects/fangjia-datastore/pom.xml +++ b/projects/fangjia-datastore/pom.xml @@ -33,7 +33,7 @@ true lib/ com.fangjia.datastore.DataStoreServer - + @@ -94,8 +94,8 @@ org.mongodb - mongo-java-driver - 2.6.5 + mongodb-driver-legacy + 3.9.1 log4j diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java index 1f740a992..9631409cf 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/ConfigurationException.java @@ -1,22 +1,21 @@ package com.fangjia.datastore; -public class ConfigurationException extends Exception { - - private static final long serialVersionUID = 6570405263317311822L; - - public ConfigurationException() { - } - - public ConfigurationException(String message) { - super(message); - } - - public ConfigurationException(Throwable cause) { - super(cause); - } - - public ConfigurationException(String message, Throwable cause) { - super(message, cause); - } - +public class ConfigurationException extends Exception +{ + private static final long serialVersionUID = 6570405263317311822L; + + public ConfigurationException() { + } + + public ConfigurationException(final String message) { + super(message); + } + + public ConfigurationException(final Throwable cause) { + super(cause); + } + + public ConfigurationException(final String message, final Throwable cause) { + super(message, cause); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java index ab27690ef..1303b2372 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Cursor.java @@ -1,80 +1,83 @@ package com.fangjia.datastore; +import com.fangjia.datastore.mongo.protocol.QueryMessage; import java.util.concurrent.atomic.AtomicLong; -import com.fangjia.datastore.mongo.protocol.QueryMessage; - -public class Cursor { - private static final AtomicLong ID_GENERATOR = new AtomicLong(1); - private long id; - private int limit; - private int skip; - private long totalCount; - private QueryMessage queryMessage; - - public Cursor(int skip, int limit, QueryMessage queryMessage, long totalCount) { - id = ID_GENERATOR.getAndIncrement(); - this.skip = skip; - this.limit = limit; - this.totalCount = totalCount; - this.queryMessage = queryMessage; - } - - public long getId() { - return id; - } - - public int getLimit() { - return limit; - } - - public void setLimit(int limit) { - this.limit = limit; - } - - public int nextSkip() { - skip += limit; - return skip; - } - - public void setSkip(int skip) { - this.skip = skip; - } - - public boolean hasMore() { - return this.totalCount > this.skip; - } - - public long getTotalCount() { - return totalCount; - } - - public void setTotalCount(long totalCount) { - this.totalCount = totalCount; - } - - public QueryMessage getQueryMessage() { - return queryMessage; - } - - public void setQueryMessage(QueryMessage queryMessage) { - this.queryMessage = queryMessage; - } - - public int getSkip() { - return skip; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("id:").append(id); - builder.append(", limit:").append(limit); - builder.append(", skip:").append(skip); - builder.append(", totalCount:").append(totalCount); - builder.append(", queryMessage:").append(queryMessage); - builder.append("}"); - return builder.toString(); - } - +public class Cursor +{ + private static final AtomicLong ID_GENERATOR; + private long id; + private int limit; + private int skip; + private long totalCount; + private QueryMessage queryMessage; + + public Cursor(final int skip, final int limit, final QueryMessage queryMessage, final long totalCount) { + this.id = Cursor.ID_GENERATOR.getAndIncrement(); + this.skip = skip; + this.limit = limit; + this.totalCount = totalCount; + this.queryMessage = queryMessage; + } + + public long getId() { + return this.id; + } + + public int getLimit() { + return this.limit; + } + + public void setLimit(final int limit) { + this.limit = limit; + } + + public int nextSkip() { + return this.skip += this.limit; + } + + public void setSkip(final int skip) { + this.skip = skip; + } + + public boolean hasMore() { + return this.totalCount > this.skip; + } + + public long getTotalCount() { + return this.totalCount; + } + + public void setTotalCount(final long totalCount) { + this.totalCount = totalCount; + } + + public QueryMessage getQueryMessage() { + return this.queryMessage; + } + + public void setQueryMessage(final QueryMessage queryMessage) { + this.queryMessage = queryMessage; + } + + public int getSkip() { + return this.skip; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("id:").append(this.id); + builder.append(", limit:").append(this.limit); + builder.append(", skip:").append(this.skip); + builder.append(", totalCount:").append(this.totalCount); + builder.append(", queryMessage:").append(this.queryMessage); + builder.append("}"); + return builder.toString(); + } + + static { + ID_GENERATOR = new AtomicLong(1L); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java index 52f2196ed..619d7f038 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStore.java @@ -1,44 +1,35 @@ package com.fangjia.datastore; -import java.util.Collection; import java.util.List; - import org.bson.BSONObject; - -import com.fangjia.datastore.config.Reloadable; -import com.fangjia.datastore.config.StoreConfig; import com.fangjia.datastore.solr.Index; +import java.util.Collection; +import com.fangjia.datastore.config.StoreConfig; +import com.fangjia.datastore.config.Reloadable; -public interface DataStore extends Reloadable { - - StoreConfig getStoreConfig(); - - Mongo getMongo(); - - Object getAttribute(String key); - - Object setAttribute(String key, Object attr); - - Collection getIndexes() throws DataStoreException; - - /** - * 按查询条件重建索引 - * - * @param query - * @throws DataStoreException - */ - BSONObject reIndex(BSONObject query) throws DataStoreException; - - void ensureIndex(Collection indexes) throws DataStoreException; - - QueryResponse find(BSONObject query, BSONObject fields, BSONObject sort, BSONObject facet, int skip, int limit, boolean bulk, - boolean fromMaster) throws DataStoreException; - - int count(BSONObject query, boolean fromMaster) throws DataStoreException; - - void remove(BSONObject query) throws DataStoreException; - - void update(BSONObject query, BSONObject data,String ip) throws DataStoreException; - - void save(List docList) throws DataStoreException; +public interface DataStore extends Reloadable +{ + StoreConfig getStoreConfig(); + + Mongo getMongo(); + + Object getAttribute(final String p0); + + Object setAttribute(final String p0, final Object p1); + + Collection getIndexes() throws DataStoreException; + + BSONObject reIndex(final BSONObject p0) throws DataStoreException; + + void ensureIndex(final Collection p0) throws DataStoreException; + + QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final BSONObject p3, final int p4, final int p5, final boolean p6, final boolean p7) throws DataStoreException; + + int count(final BSONObject p0, final boolean p1) throws DataStoreException; + + void remove(final BSONObject p0) throws DataStoreException; + + int update(final BSONObject p0, final BSONObject p1) throws DataStoreException; + + void save(final List p0) throws DataStoreException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java index 7777ebef4..bb2f19617 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContext.java @@ -3,15 +3,15 @@ package com.fangjia.datastore; import com.fangjia.datastore.config.DataStoreConfig; import com.fangjia.datastore.config.Reloadable; -public interface DataStoreContext extends Reloadable { - - DataStoreConfig getDataStoreConfig(); - - DataStore getDataStore(String storeName) throws DataStoreException; - - Cursor getCursor(long cursorId); - - void addCursor(Cursor cursor); - - Cursor removeCursor(long cursorId); +public interface DataStoreContext extends Reloadable +{ + DataStoreConfig getDataStoreConfig(); + + DataStore getDataStore(final String p0) throws DataStoreException; + + Cursor getCursor(final long p0); + + void addCursor(final Cursor p0); + + Cursor removeCursor(final long p0); } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java index 2e11d6ca7..54fde188a 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreContextImpl.java @@ -1,56 +1,61 @@ package com.fangjia.datastore; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import com.fangjia.datastore.config.DataStoreConfig; import com.fangjia.datastore.config.StoreConfig; +import java.util.Iterator; +import java.util.Set; +import java.util.HashMap; +import java.util.concurrent.TimeUnit; import com.google.common.collect.MapMaker; +import java.util.Map; +import com.fangjia.datastore.config.DataStoreConfig; -public class DataStoreContextImpl implements DataStoreContext { - - private final DataStoreConfig config; - private final Map cursorMap; - private final Map stores; - - public DataStoreContextImpl(DataStoreConfig config) throws ConfigurationException { - this.config = config; - cursorMap = new MapMaker().concurrencyLevel(50).expiration(config.getServerConfig().getCursorExpirationMinutes(), - TimeUnit.MINUTES).makeMap(); - Set storeNames = config.getStoreNames(); - stores = new HashMap(storeNames.size()); - for (String storeName : storeNames) { - StoreConfig storeConfig = this.config.getStoreConfig(storeName); - stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig()))); - } - } - - public DataStoreConfig getDataStoreConfig() { - return config; - } - - public DataStore getDataStore(String storeName) throws DataStoreException { - if (!stores.containsKey(storeName)) { - throw new DataStoreException("unknown store name: \"" + storeName + "\""); - } - return stores.get(storeName); - } - - public void reload() throws ConfigurationException { - config.reload(); - } - - public Cursor getCursor(long cursorId) { - return this.cursorMap.get(cursorId); - } - - public void addCursor(Cursor cursor) { - this.cursorMap.put(cursor.getId(), cursor); - } - - public Cursor removeCursor(long cursorId) { - return this.cursorMap.remove(cursorId); - } +public class DataStoreContextImpl implements DataStoreContext +{ + private final DataStoreConfig config; + private final Map cursorMap; + private final Map stores; + + public DataStoreContextImpl(final DataStoreConfig config) throws ConfigurationException { + this.config = config; + this.cursorMap = new MapMaker().concurrencyLevel(50).expiration((long)config.getServerConfig().getCursorExpirationMinutes(), TimeUnit.MINUTES).makeMap(); + final Set storeNames = config.getStoreNames(); + this.stores = new HashMap(storeNames.size()); + for (final String storeName : storeNames) { + final StoreConfig storeConfig = this.config.getStoreConfig(storeName); + this.stores.put(storeName, new DataStoreImpl(storeConfig, new Mongo(storeConfig.getMongoConfig()))); + } + } + + @Override + public DataStoreConfig getDataStoreConfig() { + return this.config; + } + + @Override + public DataStore getDataStore(final String storeName) throws DataStoreException { + if (!this.stores.containsKey(storeName)) { + throw new DataStoreException("unknown store name: \"" + storeName + "\""); + } + return this.stores.get(storeName); + } + + @Override + public void reload() throws ConfigurationException { + this.config.reload(); + } + + @Override + public Cursor getCursor(final long cursorId) { + return this.cursorMap.get(cursorId); + } + + @Override + public void addCursor(final Cursor cursor) { + this.cursorMap.put(cursor.getId(), cursor); + } + + @Override + public Cursor removeCursor(final long cursorId) { + return this.cursorMap.remove(cursorId); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java index 034460ce5..23e8b15cb 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreException.java @@ -1,23 +1,21 @@ package com.fangjia.datastore; -public class DataStoreException extends Exception { - - private static final long serialVersionUID = -5492973985910379215L; - - public DataStoreException() { - super(); - } - - public DataStoreException(String message, Throwable cause) { - super(message, cause); - } - - public DataStoreException(String message) { - super(message); - } - - public DataStoreException(Throwable cause) { - super(cause); - } - +public class DataStoreException extends Exception +{ + private static final long serialVersionUID = -5492973985910379215L; + + public DataStoreException() { + } + + public DataStoreException(final String message, final Throwable cause) { + super(message, cause); + } + + public DataStoreException(final String message) { + super(message); + } + + public DataStoreException(final Throwable cause) { + super(cause); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java index 9f10d1e4a..215b15b64 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreImpl.java @@ -1,702 +1,637 @@ package com.fangjia.datastore; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; +import com.fangjia.datastore.utils.LoggerFactory; +import com.mongodb.WriteResult; +import com.mongodb.WriteConcern; import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.commons.collections.CollectionUtils; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.Collections; import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.FacetField.Count; -import org.bson.BSONObject; +import java.util.Arrays; +import java.util.ArrayList; +import com.fangjia.datastore.solr.SolrException; +import java.util.Iterator; import org.bson.BasicBSONObject; import org.bson.types.ObjectId; +import java.util.List; +import com.mongodb.BasicDBObject; +import java.util.Date; +import org.apache.commons.collections.CollectionUtils; +import org.bson.BSONObject; +import com.fangjia.datastore.solr.Index; +import java.util.Collection; +import com.mongodb.DBCollection; +import com.mongodb.gridfs.GridFS; +import java.util.HashMap; +import com.fangjia.datastore.solr.CoreImpl; +import com.fangjia.datastore.utils.MongoUtils; +import com.fangjia.datastore.config.StoreConfig; +import java.util.Map; +import com.fangjia.datastore.solr.Core; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.Set; +import com.mongodb.DBObject; +import java.util.Comparator; import org.slf4j.Logger; -import com.fangjia.datastore.config.StoreConfig; -import com.fangjia.datastore.solr.Core; -import com.fangjia.datastore.solr.CoreImpl; -import com.fangjia.datastore.solr.Index; -import com.fangjia.datastore.solr.SolrException; -import com.fangjia.datastore.solr.SolrSchema; -import com.fangjia.datastore.utils.LoggerFactory; -import com.fangjia.datastore.utils.MongoUtils; -import com.mongodb.BasicDBObject; -import com.mongodb.Bytes; -import com.mongodb.DBCollection; -import com.mongodb.DBObject; -import com.mongodb.WriteConcern; -import com.mongodb.gridfs.GridFS; - -public class DataStoreImpl implements DataStore { - - private final static Logger LOGGER = LoggerFactory.getLogger(DataStoreImpl.class); - private final static Logger PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(DataStoreImpl.class); - private final static String SEAWEED_UPDATE_LOG = "seaweed_update_log"; - - private final static Comparator OBJECT_ID_COMPARATOR = new Comparator() { - - public int compare(DBObject o1, DBObject o2) { - ObjectId id1 = (ObjectId) o1.get("_id"); - ObjectId id2 = (ObjectId) o2.get("_id"); - return id1.compareTo(id2); - } - }; - private final static Set SPECIAL_KEYS = new HashSet(Arrays.asList("_id", "$bulk", SolrSchema.VERTION)); - private final static int MAX_REMOVE_ROWS = 1000; - - private final AtomicBoolean indexing = new AtomicBoolean(false); - - private final Core core; - private final Map attributes; - private final StoreConfig storeConfig; - private Mongo mongo; - private String mongoDbName; - private String mongoCollectionName; - private final FileStore fileStore; - - public DataStoreImpl(StoreConfig storeConfig, Mongo mongo) throws ConfigurationException { - String[] split = MongoUtils.parseFullCollectionName(storeConfig.getName()); - if (null == split) { - throw new IllegalArgumentException("invalid store name: \"" + storeConfig.getName() + "\""); - } - this.storeConfig = storeConfig; - this.mongo = mongo; - mongoDbName = split[0]; - mongoCollectionName = split[1]; - core = new CoreImpl(storeConfig); - attributes = new HashMap(); - - if (storeConfig.isTextFieldToGridFS()) { - fileStore = new FileStoreImpl(new GridFS(mongo.getDB(mongoDbName), mongoCollectionName), storeConfig.getFields()); - } else { - fileStore = null; - } - } - - public void reload() throws ConfigurationException { - this.core.reload(); - } - - public DBCollection getMongoCollection() { - return mongo.getDB(mongoDbName).getCollection(mongoCollectionName); - } - - public DBCollection getUpdateLogMongoCollection() { - return mongo.getDB(mongoDbName).getCollection(SEAWEED_UPDATE_LOG); - } - - public StoreConfig getStoreConfig() { - return storeConfig; - } - - public Mongo getMongo() { - return mongo; - } - - public Object getAttribute(String key) { - return attributes.get(key); - } - - public Object setAttribute(String key, Object attr) { - return attributes.put(key, attr); - } - - public Collection getIndexes() throws DataStoreException { - try { - return core.getIndexes(); - } catch (Exception e) { - throw new DataStoreException("error get indexes", e); - } - } - - public BSONObject reIndex(BSONObject query) throws DataStoreException { - try { - // 如没有查询条件,则全部重建 - if (null == query || CollectionUtils.isEmpty(query.keySet())) { - if (indexing.compareAndSet(false, true)) { - String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); // 操作发起的时间戳 - Iterator i = this.getMongoCollection().find().addOption(Bytes.QUERYOPTION_NOTIMEOUT) - .sort(new BasicDBObject("_id", 1)); - core.reIndex(null, i); - // 删除重建之前的索引 - query = new BasicBSONObject(); - BSONObject versionObj = new BasicBSONObject(); - versionObj.put("$lte", currentVersion); - query.put(SolrSchema.VERTION, versionObj); - core.deleteByQuery(query); - return query; - } else { - throw new DataStoreException(storeConfig.getName() + ": the re-index is in progress."); - } - } else { - doReindexByQuery(query); - return null; - } - } catch (Exception e) { - throw new DataStoreException("error re-index", e); - } finally { - indexing.compareAndSet(true, false); - } - } - - private interface Closure { - /** - * 返回false则跳出遍历 - * - * @param input - * @return - * @throws SolrException - * @throws DataStoreException - */ - public boolean execute(List input) throws SolrException, DataStoreException; - } - - /** - * 根据查询条件重建索引 - * - * @param query - * @throws DataStoreException - */ - private void doReindexByQuery(BSONObject query) throws DataStoreException { - String operationName = this.storeConfig.getName() + "_re-index_" + System.currentTimeMillis(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(operationName + ": query=" + query); - } - - try { - iterateSolrQuery(operationName, query, this.storeConfig.getDataStoreConfig().getServerConfig().getReIndexBatchSize(), - new Closure() { - - public boolean execute(List objIdList) throws SolrException, DataStoreException { - DBObject seletor = new BasicDBObject(); - seletor.put("_id", new BasicDBObject("$in", objIdList)); - Iterator i = getMongoCollection().find(seletor); - core.reIndex(objIdList, i); - return true; - } - - }); - core.deleteByQuery(query); - } catch (SolrException e) { - throw new DataStoreException(operationName + ": ", e); - } - - } - - /** - * 根据query遍历。
    - * 这个方法只适用于reIndex, update, remove等会重建索引的操作, 不能简单的用来遍历 - * - * @param operationName - * @param query - * @param batchSize - * @param closure - * @throws SolrException - * @throws DataStoreException - */ - private void iterateSolrQuery(String operationName, BSONObject query, int batchSize, Closure closure) throws SolrException, - DataStoreException { - if (null == query.get(SolrSchema.VERTION)) { - String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); // 操作发起的时间戳 - BSONObject versionObj = new BasicBSONObject(); - versionObj.put("$lte", currentVersion); - query.put(SolrSchema.VERTION, versionObj); // 只遍历时间戳之前的数据 - } - - long startTime = System.currentTimeMillis(); - int currentPage = 0; // 当前遍历页面 - long c = 0; - long lastC = c; - int round = 0; - long roundIndexCount = 0; - QueryResponse response = null; - int lastNumFound = -1; - while (true) { - response = findInSolr(query, null, null, currentPage * batchSize, batchSize, true, true); - if (currentPage == 0) {// 每轮第一次迭代 - if (lastNumFound == -1) { - // 第一轮 - lastNumFound = response.getNumFound(); - } else if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) { - long sleep = 20000; - if (LOGGER.isInfoEnabled()) { - LOGGER.info("{}[{}]: wait solr commit, sleep {}s.", new Object[] { operationName, round, sleep / 1000 }); - } - try { - Thread.sleep(sleep); - } catch (InterruptedException e) { - throw new DataStoreException(operationName + ": ", e); - } - response = findInSolr(query, null, null, 0, 1, true, true); - if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) { - // TODO: 可能需要删除solr中有但mongodb中没有的索引数据 - // core.deleteByQuery(query); - break; - } - lastNumFound = response.getNumFound(); - continue; - } else { - lastNumFound = response.getNumFound(); - } - } - - int returnedSize = response.getNumReturned(); - List idList = response.getIdList(); - // 有可能为空 - if (null != idList && idList.size() > 0) { - // 执行重建 - if (!closure.execute(idList)) { - // 返回false则不再遍历 - break; - } - } - c += returnedSize; - roundIndexCount += returnedSize; - if (c - lastC >= batchSize && LOGGER.isInfoEnabled()) { - LOGGER.info("{}[{}]: {}/{}, {}/s", new Object[] { operationName, round, c, response.getNumFound(), - (int) (c / ((System.currentTimeMillis() - startTime) / 1000.0)) }); - lastC = c; - } - - // 因为重建后的索引已不在query条件内,所以query条件的整个集合大小随着遍历的往前进行,一直在变动 - // 而分页是顺序向前的,所以一次执行,不能保证全部遍历到,所以要多次执行 - if (currentPage == 0 && returnedSize >= lastNumFound) { - // 第一页数量就小于BATCH_SIZE,则不用往下遍历 - break; - } else if (returnedSize == 0 || roundIndexCount >= lastNumFound) { - // 本轮重建数量大于总数量,从头再来过 - currentPage = 0; - round++; - roundIndexCount = 0; - } else { - // 当前页数量跟BATCH_SIZE一致,向前翻页 - currentPage++; - } - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info("{}[{}] finished, total time: {}s, count: {}/{}", new Object[] { operationName, round, - (System.currentTimeMillis() - startTime) / 1000, roundIndexCount, c }); - } - } - - public QueryResponse find(BSONObject query, BSONObject fields, BSONObject sort, BSONObject facet, int skip, int limit, - boolean bulk, boolean fromMaster) throws DataStoreException { - checkFields(fields); - try { - QueryResponse response = findInSolr(query, sort, facet, skip, limit, fromMaster, bulk); - List docList = find(response, fields); - int numReturned = docList.size() - 1; - if (numReturned > response.getNumFound()) { - response.setNumFound(numReturned); - docList.get(0).put("$numFound", numReturned); - } - if (numReturned != response.getNumReturned()) { - response.setNumReturned(numReturned); - } - response.setDocList(docList); - return response; - } catch (SolrException e) { - throw new DataStoreException("error execute find{query=" + query + ", fields=" + fields + ", sort=" + sort - + ", facet=" + facet + ", skip=" + skip + ", limit=" + limit + "}: ", e); - } - } - - private QueryResponse findInSolr(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, - boolean fromMaster, boolean bulk) throws SolrException, DataStoreException { - QueryResponse response = null; - List idList = null; - if (null == facet || facet.keySet().size() == 0) { - idList = getObjectIdsInQuery(query); - } - if (null != idList && idList.size() > 0) { - response = new QueryResponse(); - response.setIdList(idList); - response.setNumReturned(idList.size()); - } else { - response = core.find(query, sort, facet, skip, limit, fromMaster, bulk); - } - return response; - } - - @SuppressWarnings("unchecked") - private List getObjectIdsInQuery(BSONObject query) throws DataStoreException { - List ret = null; - Object value = query.get("_id"); - if (null == value) { - ret = null; - } else if (value instanceof BSONObject) { - List ids = (List) ((BSONObject) value).get("$in"); - if (null != ids && ids.size() > 0) { - if (ids.get(0) instanceof ObjectId) { - ret = (List) ids; - } else { - ret = new ArrayList(ids.size()); - for (Object id : ids) { - ret.add(new ObjectId(String.valueOf(id))); - } - } - } - } else if (value instanceof ObjectId) { - ret = Arrays.asList((ObjectId) value); - } else if (value instanceof String) { - ret = Arrays.asList(new ObjectId((String) value)); - } - if (ret != null && containsNoneIdQuery(query)) { - throw new DataStoreException("id query with other query:" + query); - } - return ret; - } - - private boolean containsNoneIdQuery(BSONObject query) { - if (null == query) { - return false; - } - for (String key : query.keySet()) { - if (SPECIAL_KEYS.contains(key)) { - continue; - } - return true; - } - return false; - } - - public int count(BSONObject query, boolean fromMaster) throws DataStoreException { - try { - QueryResponse response = core.find(query, null, null, 0, 0, fromMaster, false); - List docList = find(response, null); - response.setDocList(docList); - return response.getNumFound(); - } catch (SolrException e) { - throw new DataStoreException("error execute count{query=" + query + "}: ", e); - } - } - - private List find(QueryResponse response, BSONObject fields) throws DataStoreException { - List idList = response.getIdList(); - List newDocList = new ArrayList(null == idList ? 1 : (idList.size() + 1)); - - DBObject meta = new BasicDBObject("$numFound", response.getNumFound()); - List facetFieldList = response.getFacetFieldList(); - if (null != facetFieldList && facetFieldList.size() > 0) { - List facetList = new ArrayList(facetFieldList.size()); - for (FacetField facetField : facetFieldList) { - DBObject facet = new BasicDBObject(); - facet.put("name", facetField.getName()); - facet.put("end", facetField.getEnd()); - facet.put("gap", facetField.getGap()); - facet.put("valueCount", facetField.getValueCount()); - List values = facetField.getValues(); - if (null != values && values.size() > 0) { - List vs = new ArrayList(values.size()); - for (Count c : values) { - DBObject v = new BasicDBObject(); - v.put("name", c.getName()); - v.put("count", c.getCount()); - vs.add(v); - } - facet.put("values", vs); - } - facetList.add(facet); - } - meta.put("$facet", facetList); - } - newDocList.add(meta); - - if (null != idList && idList.size() > 0) { - /** - * 此处是为了性能考虑 - * 当查询只需要_id的时候,直接返回从solr中检索出的idList即可 - * 但是由于存在solr中存在的_id在mongodb中不存在,所以把这段代码注释了,从而一定程度降低了只查询_id请求的性能,保证了数据一致性 - */ -// if (null != fields && fields.containsField("_id") && fields.keySet().size() == 1) { -// for (ObjectId id : idList) { -// newDocList.add(new BasicDBObject("_id", id)); -// } -// } else { - DBObject query = new BasicDBObject(); - if (idList.size() == 1) { - query.put("_id", idList.get(0)); - } else { - query.put("_id", new BasicDBObject("$in", idList)); - } - DBObject fieldsQuery = null; - if (null != fields) { - fieldsQuery = new BasicDBObject(); - fieldsQuery.putAll(fields); - } - - long start = System.currentTimeMillis(); - List docList = this.getMongoCollection().find(query, fieldsQuery).toArray(); - Collections.sort(docList, OBJECT_ID_COMPARATOR); - long time = System.currentTimeMillis() - start; - if (time > 100) { - if (PERFORMANCE_LOGGER.isWarnEnabled()) { - PERFORMANCE_LOGGER.warn(this.storeConfig.getName() + ": query mongodb, id size: " + idList.size() - + ", time: " + time); - } - } - if (null != docList && !docList.isEmpty()) { - for (Iterator i = idList.iterator(); i.hasNext();) { - int index = binarySearch(docList, i.next()); - if (index >= 0) { - newDocList.add(docList.get(index)); - } - } - } - - if (storeConfig.isTextFieldToGridFS()) { - newDocList = fileStore.find(newDocList, fields); - } -// } - } - return newDocList; - } - - private static int binarySearch(List list, ObjectId id) { - int low = 0; - int high = list.size() - 1; - - while (low <= high) { - int mid = (low + high) >>> 1; - ObjectId midVal = (ObjectId) list.get(mid).get("_id"); - int cmp = midVal.compareTo(id); - - if (cmp < 0) - low = mid + 1; - else if (cmp > 0) - high = mid - 1; - else - return mid; // key found - } - return -(low + 1); // key not found - } - - private boolean isOpSupported(String op) { - return "$inc".equals(op) || "$set".equals(op) || "$unset".equals(op); - } - - private void checkUpdateFields(BSONObject data) throws DataStoreException { - for (String op : data.keySet()) { - if (!isOpSupported(op)) { - throw new DataStoreException("modifier operation " + op + " is not supported!"); - } - BSONObject update = (BSONObject) data.get(op); - if (null != update) { - for (String field : update.keySet()) { - if (!storeConfig.containsField(field)) { - throw new DataStoreException("unknown field: " + field + " !"); - } - if (!storeConfig.isFieldValueLegal(field, update.get(field))) { - throw new DataStoreException("invalid value " + update.get(field) + " of field '" + field + "'!"); - } - } - } - } - } - - private void checkFields(BSONObject fields) throws DataStoreException { - if (null != fields) { - for (String field : fields.keySet()) { - if (!storeConfig.containsField(field)) { - throw new DataStoreException("unknown field: " + field + " !"); - } - } - } - } - - public void update(final BSONObject query, final BSONObject data,final String ip) throws DataStoreException { - checkUpdateFields(data); - String operationName = this.storeConfig.getName() + "_update_" + System.currentTimeMillis(); - - if (LOGGER.isInfoEnabled()) { - LOGGER.info(operationName + ": query=" + query); - } - - try { - iterateSolrQuery(operationName, query, 1000, new Closure() { - - public boolean execute(List objIdList) throws SolrException, DataStoreException { - final DBObject update = new BasicDBObject(); - final String ipAddress = ip; - update.putAll(data); - - // 更新大字段关联文件 - if (storeConfig.isTextFieldToGridFS()) { - fileStore.update(objIdList, update); - - // 把要更新的大字段数据,置为null,不需要把数据插入主表 - BSONObject realData = (BSONObject) update.get("$set"); - if (null != realData) { - Set removeField = new HashSet(); - for (String field : realData.keySet()) { - if (storeConfig.isBigField(field)) { - // realData.removeField(field); - removeField.add(field); - } - } - realData.keySet().removeAll(removeField); - } - } - - DBObject seletor = new BasicDBObject(); - if (objIdList.size() == 1) { - seletor.put("_id", objIdList.get(0)); - } else { - seletor.put("_id", new BasicDBObject("$in", objIdList)); - } - getMongoCollection().update(seletor, update, false, true, WriteConcern.NORMAL); - //修改小区状态为1时记录状态 - if(update.get("$set")!=null){ - BasicBSONObject updateData = new BasicBSONObject(); - updateData = (BasicBSONObject) update.get("$set"); - String status = updateData.getString("status"); - if(status!=null && "1".equals(status)){ - DBObject saveObj = new BasicDBObject(); - saveObj.put("ip", ipAddress); - saveObj.put("districtIp", objIdList.get(0).toString()); - - saveObj.put("type", "remove"); - saveObj.put("c_date", new Date()); - getUpdateLogMongoCollection().save(saveObj); - } +public class DataStoreImpl implements DataStore +{ + private static final Logger LOGGER; + private static final Logger PERFORMANCE_LOGGER; + private static final String SEAWEED_UPDATE_LOG = "seaweed_update_log"; + private static final Comparator OBJECT_ID_COMPARATOR; + private static final Set SPECIAL_KEYS; + private static final int MAX_REMOVE_ROWS = 10; + private final AtomicBoolean indexing; + private final Core core; + private final Map attributes; + private final StoreConfig storeConfig; + private Mongo mongo; + private String mongoDbName; + private String mongoCollectionName; + private final FileStore fileStore; + + public DataStoreImpl(final StoreConfig storeConfig, final Mongo mongo) throws ConfigurationException { + this.indexing = new AtomicBoolean(false); + final String[] split = MongoUtils.parseFullCollectionName(storeConfig.getName()); + if (null == split) { + throw new IllegalArgumentException("invalid store name: \"" + storeConfig.getName() + "\""); + } + this.storeConfig = storeConfig; + this.mongo = mongo; + this.mongoDbName = split[0]; + this.mongoCollectionName = split[1]; + this.core = new CoreImpl(storeConfig); + this.attributes = new HashMap(); + if (storeConfig.isTextFieldToGridFS()) { + this.fileStore = new FileStoreImpl(new GridFS(mongo.getDB(this.mongoDbName), this.mongoCollectionName), storeConfig.getFields()); + } + else { + this.fileStore = null; + } + } + + @Override + public void reload() throws ConfigurationException { + this.core.reload(); + } + + public DBCollection getMongoCollection() { + return this.mongo.getDB(this.mongoDbName).getCollection(this.mongoCollectionName); + } + + public DBCollection getUpdateLogMongoCollection() { + return this.mongo.getDB(this.mongoDbName).getCollection("seaweed_update_log"); + } + + @Override + public StoreConfig getStoreConfig() { + return this.storeConfig; + } + + @Override + public Mongo getMongo() { + return this.mongo; + } + + @Override + public Object getAttribute(final String key) { + return this.attributes.get(key); + } + + @Override + public Object setAttribute(final String key, final Object attr) { + return this.attributes.put(key, attr); + } + + @Override + public Collection getIndexes() throws DataStoreException { + try { + return this.core.getIndexes(); + } + catch (Exception e) { + throw new DataStoreException("error get indexes", e); + } + } + + @Override + public BSONObject reIndex(BSONObject query) throws DataStoreException { + try { + if (null != query && !CollectionUtils.isEmpty((Collection)query.keySet())) { + this.doReindexByQuery(query); + return null; + } + if (this.indexing.compareAndSet(false, true)) { + final String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); + final Iterator i = (Iterator)this.getMongoCollection().find().addOption(16).sort((DBObject)new BasicDBObject("_id", (Object)1)); + this.core.reIndex(null, i); + query = (BSONObject)new BasicBSONObject(); + final BSONObject versionObj = (BSONObject)new BasicBSONObject(); + versionObj.put("$lte", (Object)currentVersion); + query.put("_version", (Object)versionObj); + this.core.deleteByQuery(query); + return query; + } + throw new DataStoreException(this.storeConfig.getName() + ": the re-index is in progress."); + } + catch (Exception e) { + throw new DataStoreException("error re-index", e); + } + finally { + this.indexing.compareAndSet(true, false); + } + } + + private void doReindexByQuery(final BSONObject query) throws DataStoreException { + final String operationName = this.storeConfig.getName() + "_re-index_" + System.currentTimeMillis(); + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info(operationName + ": query=" + query); + } + try { + this.iterateSolrQuery(operationName, query, this.storeConfig.getDataStoreConfig().getServerConfig().getReIndexBatchSize(), new Closure() { + @Override + public boolean execute(final List objIdList) throws SolrException, DataStoreException { + final DBObject seletor = (DBObject)new BasicDBObject(); + seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList)); + final Iterator i = (Iterator)DataStoreImpl.this.getMongoCollection().find(seletor); + DataStoreImpl.this.core.reIndex(objIdList, i); + return true; + } + }); + this.core.deleteByQuery(query); + } + catch (SolrException e) { + throw new DataStoreException(operationName + ": ", e); + } + } + + private void iterateSolrQuery(final String operationName, final BSONObject query, final int batchSize, final Closure closure) throws SolrException, DataStoreException { + if (null == query.get("_version")) { + final String currentVersion = Core.VERSION_DATE_FORMAT.format(new Date()); + final BSONObject versionObj = new BasicBSONObject(); + versionObj.put("$lte", currentVersion); + query.put("_version", versionObj); + } + final long startTime = System.currentTimeMillis(); + int currentPage = 0; + long lastC; + long c = lastC = 0L; + int round = 0; + long roundIndexCount = 0L; + QueryResponse response = null; + int lastNumFound = -1; + while (true) { + response = this.findInSolr(query, null, null, currentPage * batchSize, batchSize, true, true); + if (currentPage == 0) { + if (lastNumFound == -1) { + lastNumFound = response.getNumFound(); + } + else if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) { + final long sleep = 20000L; + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info("{}[{}]: wait solr commit, sleep {}s.", new Object[] { operationName, round, sleep / 1000L }); } - // 重建索引 - core.save(getMongoCollection().find(seletor).addOption(Bytes.QUERYOPTION_NOTIMEOUT)); - return true; - } - - }); - } catch (SolrException e) { - throw new DataStoreException(operationName + ": ", e); - } - } - - public void remove(final BSONObject query) throws DataStoreException { - - String operationName = this.storeConfig.getName() + "_remove_" + System.currentTimeMillis(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(operationName + ": query=" + query); - } - int count = this.count(query, true); - if (count > MAX_REMOVE_ROWS && !query.containsField("$remove")) { - throw new DataStoreException("can not remove more than 1000 rows at single time, actual rows: " + count); - } - try { - List removeIdList = getObjectIdsInQuery(query); - if(removeIdList != null) { - doExecuteDelete(removeIdList); - }else { - iterateSolrQuery(operationName, query, 1000, new Closure() { - public boolean execute(List objIdList) throws SolrException, DataStoreException { - return doExecuteDelete(objIdList); - } - }); - } - } catch (SolrException e) { - throw new DataStoreException(operationName + ": ", e); - } - } - - private boolean doExecuteDelete(List objIdList) throws SolrException, DataStoreException { - DBObject seletor = new BasicDBObject(); - if (objIdList.size() == 1) { - seletor.put("_id", objIdList.get(0)); - } else { - seletor.put("_id", new BasicDBObject("$in", objIdList)); - } - getMongoCollection().remove(seletor, WriteConcern.NORMAL); - - // 删除大字段的关联文件 - if (storeConfig.isTextFieldToGridFS()) { - fileStore.remove(objIdList); - } - - core.deleteByQuery(seletor); - return true; - } - - public void save(List docList) throws DataStoreException { - if (null == docList || docList.isEmpty()) { - return; - } - if (storeConfig.isTextFieldToGridFS()) { - List insertList = new ArrayList(docList.size()); - List fileList = new ArrayList(docList.size()); - boolean hasBigField = false; - for (BSONObject bson : docList) { - DBObject fileObj = new BasicDBObject(); - for (String field : bson.keySet()) { - if (!storeConfig.containsField(field)) { - throw new DataStoreException("unknown field: " + field + " !"); - } - if (!storeConfig.isFieldValueLegal(field, bson.get(field))) { - throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!"); - } - - // 若是大字段,将内容保存至fileObj后存入fileList - if (storeConfig.isBigField(field)) { - hasBigField = true; - - fileObj.put(field, bson.get(field)); - - // 主表中不保存该字段内容 - bson.put(field, null); - } - } - fileList.add(fileObj); - - DBObject dbObj = new BasicDBObject(); - dbObj.putAll(bson); - insertList.add(dbObj); - this.getMongoCollection().save(dbObj, WriteConcern.NORMAL); - } - - // 保存大字段 - if (hasBigField) - fileStore.save(fileList, insertList); - } else { - for (BSONObject bson : docList) { - for (String field : bson.keySet()) { - if (!storeConfig.containsField(field)) { - throw new DataStoreException("unknown field: " + field + " !"); - } - if (!storeConfig.isFieldValueLegal(field, bson.get(field))) { - throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!"); - } - } - DBObject dbObj = new BasicDBObject(); - dbObj.putAll(bson); - this.getMongoCollection().save(dbObj, WriteConcern.NORMAL); - } - } - - try { - core.save(docList.iterator()); - } catch (SolrException e) { - throw new DataStoreException("error execute save{docList=" + docList + "}: ", e); - } - } - - public void ensureIndex(Collection indexes) throws DataStoreException { - if (null == indexes || indexes.isEmpty()) { - return; - } - try { - Iterator i = this.getMongoCollection().find().addOption(Bytes.QUERYOPTION_NOTIMEOUT); - core.ensureIndex(indexes, i); - } catch (SolrException e) { - throw new DataStoreException("error execute ensureIndex{indexes=" + indexes + "}:", e); - } - } - + try { + Thread.sleep(sleep); + } + catch (InterruptedException e) { + throw new DataStoreException(operationName + ": ", e); + } + response = this.findInSolr(query, null, null, 0, 1, true, true); + if (response.getNumFound() > 0 && lastNumFound == response.getNumFound()) { + break; + } + lastNumFound = response.getNumFound(); + continue; + } + else { + lastNumFound = response.getNumFound(); + } + } + final int returnedSize = response.getNumReturned(); + final List idList = response.getIdList(); + if (null != idList && idList.size() > 0 && !closure.execute(idList)) { + break; + } + c += returnedSize; + roundIndexCount += returnedSize; + if (c - lastC >= batchSize && DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info("{}[{}]: {}/{}, {}/s", new Object[] { operationName, round, c, response.getNumFound(), (int)(c / ((System.currentTimeMillis() - startTime) / 1000.0)) }); + lastC = c; + } + if (currentPage == 0 && returnedSize >= lastNumFound) { + break; + } + if (returnedSize == 0 || roundIndexCount >= lastNumFound) { + currentPage = 0; + ++round; + roundIndexCount = 0L; + } + else { + ++currentPage; + } + } + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info("{}[{}] finished, total time: {}s, count: {}/{}", new Object[] { operationName, round, (System.currentTimeMillis() - startTime) / 1000L, roundIndexCount, c }); + } + } + + @Override + public QueryResponse find(final BSONObject query, final BSONObject fields, final BSONObject sort, final BSONObject facet, final int skip, final int limit, final boolean bulk, final boolean fromMaster) throws DataStoreException { + this.checkFields(fields); + try { + final QueryResponse response = this.findInSolr(query, sort, facet, skip, limit, fromMaster, bulk); + final List docList = this.find(response, fields); + final int numReturned = docList.size() - 1; + if (numReturned > response.getNumFound()) { + response.setNumFound(numReturned); + docList.get(0).put("$numFound", (Object)numReturned); + } + if (numReturned != response.getNumReturned()) { + response.setNumReturned(numReturned); + } + response.setDocList(docList); + return response; + } + catch (SolrException e) { + throw new DataStoreException("error execute find{query=" + query + ", fields=" + fields + ", sort=" + sort + ", facet=" + facet + ", skip=" + skip + ", limit=" + limit + "}: ", e); + } + } + + private QueryResponse findInSolr(final BSONObject query, final BSONObject sort, final BSONObject facet, final int skip, final int limit, final boolean fromMaster, final boolean bulk) throws SolrException, DataStoreException { + QueryResponse response = null; + List idList = null; + if (null == facet || facet.keySet().size() == 0) { + idList = this.getObjectIdsInQuery(query); + } + if (null != idList && idList.size() > 0) { + response = new QueryResponse(); + response.setIdList(idList); + response.setNumReturned(idList.size()); + } + else { + response = this.core.find(query, sort, facet, skip, limit, fromMaster, bulk); + } + return response; + } + + private List getObjectIdsInQuery(final BSONObject query) throws DataStoreException { + List ret = null; + final Object value = query.get("_id"); + if (null == value) { + ret = null; + } + else if (value instanceof BSONObject) { + final List ids = (List)((BSONObject)value).get("$in"); + if (null != ids && ids.size() > 0) { + if (ids.get(0) instanceof ObjectId) { + ret = (List)ids; + } + else { + ret = new ArrayList(ids.size()); + for (final Object id : ids) { + ret.add(new ObjectId(String.valueOf(id))); + } + } + } + } + else if (value instanceof ObjectId) { + ret = Arrays.asList((ObjectId)value); + } + else if (value instanceof String) { + ret = Arrays.asList(new ObjectId((String)value)); + } + if (ret != null && this.containsNoneIdQuery(query)) { + throw new DataStoreException("id query with other query:" + query); + } + return ret; + } + + private boolean containsNoneIdQuery(final BSONObject query) { + if (null == query) { + return false; + } + for (final String key : query.keySet()) { + if (DataStoreImpl.SPECIAL_KEYS.contains(key)) { + continue; + } + return true; + } + return false; + } + + @Override + public int count(final BSONObject query, final boolean fromMaster) throws DataStoreException { + try { + final QueryResponse response = this.core.find(query, null, null, 0, 0, fromMaster, false); + final List docList = this.find(response, null); + response.setDocList(docList); + return response.getNumFound(); + } + catch (SolrException e) { + throw new DataStoreException("error execute count{query=" + query + "}: ", e); + } + } + + private List find(final QueryResponse response, final BSONObject fields) throws DataStoreException { + final List idList = response.getIdList(); + List newDocList = new ArrayList((null == idList) ? 1 : (idList.size() + 1)); + final DBObject meta = new BasicDBObject("$numFound", response.getNumFound()); + final List facetFieldList = response.getFacetFieldList(); + if (null != facetFieldList && facetFieldList.size() > 0) { + final List facetList = new ArrayList(facetFieldList.size()); + for (final FacetField facetField : facetFieldList) { + final DBObject facet = new BasicDBObject(); + facet.put("name", facetField.getName()); + facet.put("end", facetField.getEnd()); + facet.put("gap", facetField.getGap()); + facet.put("valueCount", facetField.getValueCount()); + final List values = facetField.getValues(); + if (null != values && values.size() > 0) { + final List vs = new ArrayList(values.size()); + for (final FacetField.Count c : values) { + final DBObject v = new BasicDBObject(); + v.put("name", c.getName()); + v.put("count", c.getCount()); + vs.add(v); + } + facet.put("values", vs); + } + facetList.add(facet); + } + meta.put("$facet", facetList); + } + newDocList.add(meta); + if (null != idList && idList.size() > 0) { + final DBObject query = new BasicDBObject(); + if (idList.size() == 1) { + query.put("_id", idList.get(0)); + } + else { + query.put("_id", new BasicDBObject("$in", idList)); + } + DBObject fieldsQuery = null; + if (null != fields) { + fieldsQuery = new BasicDBObject(); + fieldsQuery.putAll(fields); + } + final long start = System.currentTimeMillis(); + final List docList = this.getMongoCollection().find(query, fieldsQuery).toArray(); + Collections.sort(docList, DataStoreImpl.OBJECT_ID_COMPARATOR); + final long time = System.currentTimeMillis() - start; + if (time > 100L && DataStoreImpl.PERFORMANCE_LOGGER.isWarnEnabled()) { + DataStoreImpl.PERFORMANCE_LOGGER.warn(this.storeConfig.getName() + ": query mongodb, id size: " + idList.size() + ", time: " + time); + } + if (null != docList && !docList.isEmpty()) { + final Iterator i = idList.iterator(); + while (i.hasNext()) { + final int index = binarySearch(docList, i.next()); + if (index >= 0) { + newDocList.add(docList.get(index)); + } + } + } + if (this.storeConfig.isTextFieldToGridFS()) { + newDocList = this.fileStore.find(newDocList, fields); + } + } + return newDocList; + } + + private static int binarySearch(final List list, final ObjectId id) { + int low = 0; + int high = list.size() - 1; + while (low <= high) { + final int mid = low + high >>> 1; + final ObjectId midVal = (ObjectId)list.get(mid).get("_id"); + final int cmp = midVal.compareTo(id); + if (cmp < 0) { + low = mid + 1; + } + else { + if (cmp <= 0) { + return mid; + } + high = mid - 1; + } + } + return -(low + 1); + } + + private boolean isOpSupported(final String op) { + return "$inc".equals(op) || "$set".equals(op) || "$unset".equals(op); + } + + private void checkUpdateFields(final BSONObject data) throws DataStoreException { + for (final String op : data.keySet()) { + if (!this.isOpSupported(op)) { + throw new DataStoreException("modifier operation " + op + " is not supported!"); + } + final BSONObject update = (BSONObject)data.get(op); + if (null == update) { + continue; + } + for (final String field : update.keySet()) { + if (!this.storeConfig.containsField(field)) { + throw new DataStoreException("unknown field: " + field + " !"); + } + if (!this.storeConfig.isFieldValueLegal(field, update.get(field))) { + throw new DataStoreException("invalid value " + update.get(field) + " of field '" + field + "'!"); + } + } + } + } + + private void checkFields(final BSONObject fields) throws DataStoreException { + if (null != fields) { + for (final String field : fields.keySet()) { + if (!this.storeConfig.containsField(field)) { + throw new DataStoreException("unknown field: " + field + " !"); + } + } + } + } + + @Override + public int update(final BSONObject query, final BSONObject data) throws DataStoreException { + final AtomicInteger count = new AtomicInteger(0); + this.checkUpdateFields(data); + final String operationName = this.storeConfig.getName() + "_update_" + System.currentTimeMillis(); + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info(operationName + ": query=" + query); + } + try { + this.iterateSolrQuery(operationName, query, 1000, new Closure() { + @Override + public boolean execute(final List objIdList) throws SolrException, DataStoreException { + final DBObject update = (DBObject)new BasicDBObject(); + update.putAll(data); + if (DataStoreImpl.this.storeConfig.isTextFieldToGridFS()) { + DataStoreImpl.this.fileStore.update(objIdList, (BSONObject)update); + final BSONObject realData = (BSONObject)update.get("$set"); + if (null != realData) { + final Set removeField = new HashSet(); + for (final String field : realData.keySet()) { + if (DataStoreImpl.this.storeConfig.isBigField(field)) { + removeField.add(field); + } + } + realData.keySet().removeAll(removeField); + } + } + final DBObject seletor = (DBObject)new BasicDBObject(); + if (objIdList.size() == 1) { + seletor.put("_id", (Object)objIdList.get(0)); + } + else { + seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList)); + } + final WriteResult writeResult = DataStoreImpl.this.getMongoCollection().update(seletor, update, false, true, WriteConcern.ACKNOWLEDGED); + count.set(writeResult.getN()); + DataStoreImpl.this.core.save((Iterator)DataStoreImpl.this.getMongoCollection().find(seletor).addOption(16)); + return true; + } + }); + return count.get(); + } + catch (SolrException e) { + throw new DataStoreException(operationName + ": ", e); + } + } + + @Override + public void remove(final BSONObject query) throws DataStoreException { + final String operationName = this.storeConfig.getName() + "_remove_" + System.currentTimeMillis(); + if (DataStoreImpl.LOGGER.isInfoEnabled()) { + DataStoreImpl.LOGGER.info(operationName + ": query=" + query); + } + final int count = this.count(query, true); + if (count > 10 && !query.containsField("$remove")) { + throw new DataStoreException("can not remove more than 10 rows at single time, actual rows: " + count); + } + try { + final List removeIdList = this.getObjectIdsInQuery(query); + if (removeIdList != null) { + this.doExecuteDelete(removeIdList); + } + else { + this.iterateSolrQuery(operationName, query, 1000, new Closure() { + @Override + public boolean execute(final List objIdList) throws SolrException, DataStoreException { + return DataStoreImpl.this.doExecuteDelete(objIdList); + } + }); + } + } + catch (SolrException e) { + throw new DataStoreException(operationName + ": ", e); + } + } + + private boolean doExecuteDelete(final List objIdList) throws SolrException, DataStoreException { + final DBObject seletor = (DBObject)new BasicDBObject(); + if (objIdList.size() == 1) { + seletor.put("_id", (Object)objIdList.get(0)); + } + else { + seletor.put("_id", (Object)new BasicDBObject("$in", (Object)objIdList)); + } + this.getMongoCollection().remove(seletor, WriteConcern.NORMAL); + if (this.storeConfig.isTextFieldToGridFS()) { + this.fileStore.remove(objIdList); + } + this.core.deleteByQuery((BSONObject)seletor); + return true; + } + + @Override + public void save(final List docList) throws DataStoreException { + if (null == docList || docList.isEmpty()) { + return; + } + if (this.storeConfig.isTextFieldToGridFS()) { + final List insertList = new ArrayList(docList.size()); + final List fileList = new ArrayList(docList.size()); + boolean hasBigField = false; + for (final BSONObject bson : docList) { + final DBObject fileObj = (DBObject)new BasicDBObject(); + for (final String field : bson.keySet()) { + if (!this.storeConfig.containsField(field)) { + throw new DataStoreException("unknown field: " + field + " !"); + } + if (!this.storeConfig.isFieldValueLegal(field, bson.get(field))) { + throw new DataStoreException("invalid value: " + bson.get(field) + " of field '" + field + "'!"); + } + if (!this.storeConfig.isBigField(field)) { + continue; + } + hasBigField = true; + fileObj.put(field, bson.get(field)); + bson.put(field, (Object)null); + } + fileList.add(fileObj); + final DBObject dbObj = (DBObject)new BasicDBObject(); + dbObj.putAll(bson); + insertList.add(dbObj); + this.getMongoCollection().save(dbObj, WriteConcern.NORMAL); + } + if (hasBigField) { + this.fileStore.save(fileList, insertList); + } + } + else { + for (final BSONObject bson2 : docList) { + for (final String field2 : bson2.keySet()) { + if (!this.storeConfig.containsField(field2)) { + throw new DataStoreException("unknown field: " + field2 + " !"); + } + if (!this.storeConfig.isFieldValueLegal(field2, bson2.get(field2))) { + throw new DataStoreException("invalid value: " + bson2.get(field2) + " of field '" + field2 + "'!"); + } + } + final DBObject dbObj2 = (DBObject)new BasicDBObject(); + dbObj2.putAll(bson2); + this.getMongoCollection().save(dbObj2, WriteConcern.NORMAL); + } + } + try { + this.core.save(docList.iterator()); + } + catch (SolrException e) { + throw new DataStoreException("error execute save{docList=" + docList + "}: ", e); + } + } + + @Override + public void ensureIndex(final Collection indexes) throws DataStoreException { + if (null == indexes || indexes.isEmpty()) { + return; + } + try { + final Iterator i = (Iterator)this.getMongoCollection().find().addOption(16); + this.core.ensureIndex(indexes, i); + } + catch (SolrException e) { + throw new DataStoreException("error execute ensureIndex{indexes=" + indexes + "}:", e); + } + } + + static { + LOGGER = LoggerFactory.getLogger(DataStoreImpl.class); + PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(DataStoreImpl.class); + OBJECT_ID_COMPARATOR = new Comparator() { + @Override + public int compare(final DBObject o1, final DBObject o2) { + final ObjectId id1 = (ObjectId)o1.get("_id"); + final ObjectId id2 = (ObjectId)o2.get("_id"); + return id1.compareTo(id2); + } + }; + SPECIAL_KEYS = new HashSet(Arrays.asList("_id", "$bulk", "_version")); + } + + private interface Closure + { + boolean execute(final List p0) throws SolrException, DataStoreException; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java index ccab8e57f..959b16cfc 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/DataStoreServer.java @@ -1,193 +1,178 @@ package com.fangjia.datastore; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import org.apache.mina.core.buffer.IoBuffer; -import org.apache.mina.core.service.IoHandlerAdapter; +import org.slf4j.LoggerFactory; import org.apache.mina.core.service.IoService; -import org.apache.mina.core.service.IoServiceListener; -import org.apache.mina.core.service.SimpleIoProcessorPool; -import org.apache.mina.core.session.IdleStatus; -import org.apache.mina.core.session.IoSession; -import org.apache.mina.filter.codec.ProtocolCodecException; -import org.apache.mina.filter.codec.ProtocolCodecFilter; -import org.apache.mina.filter.logging.LogLevel; -import org.apache.mina.filter.logging.LoggingFilter; -import org.apache.mina.transport.socket.SocketSessionConfig; -import org.apache.mina.transport.socket.nio.NioProcessor; -import org.apache.mina.transport.socket.nio.NioSession; -import org.apache.mina.transport.socket.nio.NioSocketAcceptor; +import java.util.Arrays; import org.bson.BSONObject; import org.bson.BasicBSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fangjia.datastore.config.DataStoreConfig; -import com.fangjia.datastore.handler.DeleteMessageHandler; -import com.fangjia.datastore.handler.GetMoreMessageHandler; -import com.fangjia.datastore.handler.InsertMessageHandler; -import com.fangjia.datastore.handler.KillCursorsMessageHandler; -import com.fangjia.datastore.handler.MessageHandler; -import com.fangjia.datastore.handler.QueryMessageHandler; +import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import org.apache.mina.filter.codec.ProtocolCodecException; +import java.io.IOException; +import org.apache.mina.core.session.IoSession; +import org.apache.mina.transport.socket.SocketSessionConfig; +import java.net.SocketAddress; +import java.net.InetSocketAddress; +import org.apache.mina.core.session.IdleStatus; +import org.apache.mina.core.service.IoHandler; import com.fangjia.datastore.handler.UpdateMessageHandler; -import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator; +import com.fangjia.datastore.mongo.protocol.UpdateMessage; +import com.fangjia.datastore.handler.DeleteMessageHandler; +import com.fangjia.datastore.mongo.protocol.DeleteMessage; +import com.fangjia.datastore.handler.KillCursorsMessageHandler; +import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; +import com.fangjia.datastore.handler.GetMoreMessageHandler; +import com.fangjia.datastore.mongo.protocol.GetMoreMessage; +import com.fangjia.datastore.handler.QueryMessageHandler; +import com.fangjia.datastore.mongo.protocol.QueryMessage; +import com.fangjia.datastore.handler.InsertMessageHandler; +import com.fangjia.datastore.mongo.protocol.InsertMessage; +import java.util.concurrent.ExecutorService; +import org.apache.mina.filter.logging.LogLevel; +import org.apache.mina.filter.logging.LoggingFilter; +import org.apache.mina.core.filterchain.IoFilter; +import org.apache.mina.filter.codec.ProtocolDecoder; +import org.apache.mina.filter.codec.ProtocolEncoder; +import org.apache.mina.filter.codec.ProtocolCodecFilter; import com.fangjia.datastore.mongo.codec.MongoProtocolDecoder; import com.fangjia.datastore.mongo.codec.MongoProtocolEncoder; -import com.fangjia.datastore.mongo.protocol.DeleteMessage; -import com.fangjia.datastore.mongo.protocol.GetMoreMessage; -import com.fangjia.datastore.mongo.protocol.InsertMessage; -import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; +import org.apache.mina.core.service.IoProcessor; +import java.util.concurrent.Executor; +import org.apache.mina.core.service.SimpleIoProcessorPool; +import org.apache.mina.transport.socket.nio.NioProcessor; +import java.util.concurrent.Executors; +import org.apache.mina.core.buffer.IoBufferAllocator; +import org.apache.mina.core.buffer.IoBuffer; +import com.fangjia.datastore.mongo.codec.BSONIoBufferAllocator; +import com.fangjia.datastore.config.DataStoreConfig; +import java.util.HashMap; +import com.fangjia.datastore.handler.MessageHandler; +import java.util.Map; import com.fangjia.datastore.mongo.protocol.MongoMessage; -import com.fangjia.datastore.mongo.protocol.QueryMessage; -import com.fangjia.datastore.mongo.protocol.ReplyMessage; -import com.fangjia.datastore.mongo.protocol.UpdateMessage; - -public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener { - - private final NioSocketAcceptor acceptor; - private final DataStoreContext context; - private static final Logger LOGGER = LoggerFactory.getLogger(DataStoreServer.class); - private static ThreadLocal CURRENT_MESSAGE = new ThreadLocal(); - private final Map, MessageHandler> handlerMap = new HashMap, MessageHandler>(); - - public DataStoreServer(String[] args) throws Exception { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("starting datastore..."); - } - - DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "d:\\goojiaconfig\\datastore")); - config.setReadOnly(Boolean.getBoolean("datastore.readOnly")); - if (LOGGER.isInfoEnabled()) { - LOGGER.info(config.toString()); - } - context = new DataStoreContextImpl(config); - - IoBuffer.setAllocator(new BSONIoBufferAllocator()); - - final ExecutorService executorService = Executors.newCachedThreadPool(); - acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool(NioProcessor.class, config - .getServerConfig().getMaxConnections())); - - acceptor.getFilterChain().addLast("codec", - new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder())); - acceptor.getFilterChain().addLast("performance", - new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime())); - if (!config.getServerConfig().isDisableLogging()) { - LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class); - loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE); - acceptor.getFilterChain().addLast("logger", loggingFilter); - } - - Runtime.getRuntime().addShutdownHook(new Thread() { - - @Override - public void run() { - LOGGER.info("shutting down..."); - acceptor.unbind(); - executorService.shutdown(); - LOGGER.info("shut down"); - } - - }); - - // register message handlers - register(InsertMessage.class, new InsertMessageHandler(context)); - register(QueryMessage.class, new QueryMessageHandler(context)); - register(GetMoreMessage.class, new GetMoreMessageHandler(context)); - register(KillCursorsMessage.class, new KillCursorsMessageHandler(context)); - register(DeleteMessage.class, new DeleteMessageHandler(context)); - register(UpdateMessage.class, new UpdateMessageHandler(context)); - - acceptor.setHandler(this); - acceptor.addListener(this); - acceptor.setReuseAddress(true); - SocketSessionConfig sessionConfig = acceptor.getSessionConfig(); - sessionConfig.setTcpNoDelay(true); - sessionConfig.setKeepAlive(true); - sessionConfig.setReuseAddress(true); - sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize()); - sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300); - acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort())); - } - - private void register(Class messageClass, MessageHandler handler) { - this.handlerMap.put(messageClass, handler); - } - - /** - * @param args - * @throws Exception - * @throws - */ - public static void main(String[] args) throws Exception { - new DataStoreServer(args); - } - - @Override - public void exceptionCaught(IoSession session, Throwable cause) throws Exception { - if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("connection closed on session: {}", session); - } - } else if (cause instanceof ProtocolCodecException) { - // ProtocolDecoderException or ProtocolEncoderException - MongoMessage message = CURRENT_MESSAGE.get(); - if (null != message) { - if (message instanceof QueryMessage) { - QueryMessage queryMessage = (QueryMessage) message; - ReplyMessage reply = new ReplyMessage(); - reply.setNumberReturned(1); - reply.setResponseFlag(8); - reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID()); - BSONObject msg = new BasicBSONObject("$err", cause.getMessage()); - reply.setDocuments(Arrays.asList(msg)); - session.write(reply); - } - } - } else if (LOGGER.isErrorEnabled()) { - LOGGER.error("exception caught on session:{" + session + "}", cause); - } - } - - @Override - public void messageReceived(IoSession session, Object message) throws Exception { - if (message instanceof MongoMessage) { - MongoMessage mongoMessage = (MongoMessage) message; - CURRENT_MESSAGE.set(mongoMessage); - MessageHandler handler = this.handlerMap.get(mongoMessage.getClass()); - if (null != handler) { - handler.messageReceived(session, mongoMessage); - } else { - throw new RuntimeException("unknown message received:" + message); - } - } else { - throw new RuntimeException("unknown message received:" + message); - } - } - - public void serviceActivated(IoService service) throws Exception { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("datastore started"); - LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort()); - } - } - - public void serviceIdle(IoService service, IdleStatus idleStatus) throws Exception { - // empty - } - - public void serviceDeactivated(IoService service) throws Exception { - // empty - } - - public void sessionDestroyed(IoSession session) throws Exception { - // empty - } +import org.slf4j.Logger; +import org.apache.mina.transport.socket.nio.NioSocketAcceptor; +import org.apache.mina.core.service.IoServiceListener; +import org.apache.mina.core.service.IoHandlerAdapter; +public class DataStoreServer extends IoHandlerAdapter implements IoServiceListener +{ + private final NioSocketAcceptor acceptor; + private final DataStoreContext context; + private static final Logger LOGGER; + private static ThreadLocal CURRENT_MESSAGE; + private final Map, MessageHandler> handlerMap; + + public DataStoreServer(final String[] args) throws Exception { + this.handlerMap = new HashMap, MessageHandler>(); + if (DataStoreServer.LOGGER.isInfoEnabled()) { + DataStoreServer.LOGGER.info("starting datastore..."); + } + final DataStoreConfig config = new DataStoreConfig(System.getProperty("datastore.home", "D:\\goojiaconfig\\seaweedsolr6")); + config.setReadOnly(Boolean.getBoolean("datastore.readOnly")); + if (DataStoreServer.LOGGER.isInfoEnabled()) { + DataStoreServer.LOGGER.info(config.toString()); + } + this.context = new DataStoreContextImpl(config); + IoBuffer.setAllocator(new BSONIoBufferAllocator()); + final ExecutorService executorService = Executors.newCachedThreadPool(); + this.acceptor = new NioSocketAcceptor(executorService, new SimpleIoProcessorPool(NioProcessor.class, config.getServerConfig().getMaxConnections())); + this.acceptor.getFilterChain().addLast("codec", new ProtocolCodecFilter(new MongoProtocolEncoder(), new MongoProtocolDecoder())); + this.acceptor.getFilterChain().addLast("performance", new PerformanceLogFilter(config.getServerConfig().getPerformanceLogFilterMaxTime())); + if (!config.getServerConfig().isDisableLogging()) { + final LoggingFilter loggingFilter = new LoggingFilter(DataStoreServer.class); + loggingFilter.setExceptionCaughtLogLevel(LogLevel.NONE); + this.acceptor.getFilterChain().addLast("logger", loggingFilter); + } + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + DataStoreServer.LOGGER.info("shutting down..."); + DataStoreServer.this.acceptor.unbind(); + executorService.shutdown(); + DataStoreServer.LOGGER.info("shut down"); + } + }); + this.register(InsertMessage.class, new InsertMessageHandler(this.context)); + this.register(QueryMessage.class, new QueryMessageHandler(this.context)); + this.register(GetMoreMessage.class, new GetMoreMessageHandler(this.context)); + this.register(KillCursorsMessage.class, new KillCursorsMessageHandler(this.context)); + this.register(DeleteMessage.class, new DeleteMessageHandler(this.context)); + this.register(UpdateMessage.class, new UpdateMessageHandler(this.context)); + this.acceptor.setHandler(this); + this.acceptor.addListener(this); + this.acceptor.setReuseAddress(true); + final SocketSessionConfig sessionConfig = this.acceptor.getSessionConfig(); + sessionConfig.setTcpNoDelay(true); + sessionConfig.setKeepAlive(true); + sessionConfig.setReuseAddress(true); + sessionConfig.setReadBufferSize(config.getServerConfig().getMinaReadBufferSize()); + sessionConfig.setIdleTime(IdleStatus.BOTH_IDLE, 300); + this.acceptor.bind(new InetSocketAddress(config.getServerConfig().getPort())); + } + + private void register(final Class messageClass, final MessageHandler handler) { + this.handlerMap.put(messageClass, handler); + } + + public static void main(final String[] args) throws Exception { + new DataStoreServer(args); + } + + public void exceptionCaught(final IoSession session, final Throwable cause) throws Exception { + if (cause instanceof IOException && null != cause.getMessage() && cause.getMessage().contains("Connection reset by peer")) { + if (DataStoreServer.LOGGER.isInfoEnabled()) { + DataStoreServer.LOGGER.info("connection closed on session: {}", (Object)session); + } + } + else if (cause instanceof ProtocolCodecException) { + final MongoMessage message = DataStoreServer.CURRENT_MESSAGE.get(); + if (null != message && message instanceof QueryMessage) { + final QueryMessage queryMessage = (QueryMessage)message; + final ReplyMessage reply = new ReplyMessage(); + reply.setNumberReturned(1); + reply.setResponseFlag(8); + reply.getMessageHeader().setResponseTo(queryMessage.getMessageHeader().getRequestID()); + final BSONObject msg = (BSONObject)new BasicBSONObject("$err", cause.getMessage()); + reply.setDocuments(Arrays.asList(msg)); + session.write((Object)reply); + } + } + else if (DataStoreServer.LOGGER.isErrorEnabled()) { + DataStoreServer.LOGGER.error("exception caught on session:{" + session + "}", cause); + } + } + + public void messageReceived(final IoSession session, final Object message) throws Exception { + if (!(message instanceof MongoMessage)) { + throw new RuntimeException("unknown message received:" + message); + } + final MongoMessage mongoMessage = (MongoMessage)message; + DataStoreServer.CURRENT_MESSAGE.set(mongoMessage); + final MessageHandler handler = this.handlerMap.get(mongoMessage.getClass()); + if (null != handler) { + handler.messageReceived(session, mongoMessage); + return; + } + throw new RuntimeException("unknown message received:" + message); + } + + public void serviceActivated(final IoService service) throws Exception { + if (DataStoreServer.LOGGER.isInfoEnabled()) { + DataStoreServer.LOGGER.info("datastore started"); + DataStoreServer.LOGGER.info("listening at " + this.context.getDataStoreConfig().getServerConfig().getPort()); + } + } + + public void serviceIdle(final IoService service, final IdleStatus idleStatus) throws Exception { + } + + public void serviceDeactivated(final IoService service) throws Exception { + } + + public void sessionDestroyed(final IoSession session) throws Exception { + } + + static { + LOGGER = LoggerFactory.getLogger((Class)DataStoreServer.class); + DataStoreServer.CURRENT_MESSAGE = new ThreadLocal(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java index 0ec78a5c7..daf576ffa 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStore.java @@ -1,64 +1,22 @@ package com.fangjia.datastore; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Set; - import org.bson.BSONObject; +import java.util.Set; import org.bson.types.ObjectId; - import com.mongodb.DBObject; +import java.util.List; -public interface FileStore { - public final static String DEFAULT_CHARSET = "UTF-8"; - - /** - * - * @param fileList - * 仅包含大字段的文档列表 - * @param dbObjList - * 已经保存到数据库的主对象列表 - * @throws IllegalArgumentException - * 大字段数据不支持默认编码时抛出 - */ - void save(List fileList, List dbObjList) throws DataStoreException; - - /** - * 删除指定的大字段内容 - * - * @param objIdList - * 删除的主对象的id列表 - * @param removeFields - * 需要删除的大字段名集合 - */ - public void remove(List objIdList, Set removeFields); - - /** - * 删除所有大字段内容 - * - * @param objIdList - */ - public void remove(List objIdList); - - /** - * @param objIdList - * 待更新的主对象id列表 - * @param data - * 更新文档 - * @throws UnsupportedEncodingException - * 大字段数据不支持默认编码时抛出 - */ - public void update(List objIdList, BSONObject data) throws DataStoreException; - - /** - * @param docList - * 查询的结果文档 - * @param fields - * 查询字段对象 - * @return - * @throws IOException - * 读取大字段数据异常时、大字段数据不支持默认编码时抛出 - */ - public List find(List docList, BSONObject fields) throws DataStoreException; +public interface FileStore +{ + public static final String DEFAULT_CHARSET = "UTF-8"; + + void save(final List p0, final List p1) throws DataStoreException; + + void remove(final List p0, final Set p1); + + void remove(final List p0); + + void update(final List p0, final BSONObject p1) throws DataStoreException; + + List find(final List p0, final BSONObject p1) throws DataStoreException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java index 4285d99b7..9cef6152f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/FileStoreImpl.java @@ -1,249 +1,229 @@ package com.fangjia.datastore; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.zip.GZIPOutputStream; - import org.apache.commons.io.IOUtils; +import com.mongodb.gridfs.GridFSDBFile; +import java.util.HashSet; +import com.mongodb.BasicDBObject; +import java.util.ArrayList; +import java.util.Set; +import java.io.OutputStream; +import java.util.zip.GZIPOutputStream; +import java.io.ByteArrayOutputStream; +import com.mongodb.gridfs.GridFSFile; +import java.util.Date; +import java.io.IOException; import org.apache.commons.lang.StringUtils; +import com.fangjia.datastore.config.FieldTypeEnum; import org.bson.BSONObject; import org.bson.types.ObjectId; - -import com.fangjia.datastore.config.Field; -import com.fangjia.datastore.config.FieldTypeEnum; -import com.mongodb.BasicDBObject; import com.mongodb.DBObject; +import java.util.List; +import java.util.Iterator; +import java.util.HashMap; +import java.util.Collection; +import com.fangjia.datastore.config.Field; +import java.util.Map; import com.mongodb.gridfs.GridFS; -import com.mongodb.gridfs.GridFSDBFile; -import com.mongodb.gridfs.GridFSFile; -public class FileStoreImpl implements FileStore { - - private GridFS gridFS; - private Map bigFieldSet; - - public FileStoreImpl(GridFS gridFS, Collection fields) { - this.gridFS = gridFS; - - // 把大字段的字段名放入set - bigFieldSet = new HashMap(); - for (Field field : fields) { - if (field.isBigField()) { - bigFieldSet.put(field.getName(), field); - } - } - } - - public void save(List fileList, List dbObjList) { - if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size()) - throw new IllegalArgumentException("empty list or unequal length list is not allowed!"); - - Iterator dbObjIt = dbObjList.iterator(); - for (Iterator fileIt = fileList.iterator(); fileIt.hasNext();) { - // 取得已保存主对象的id - String id = ((ObjectId) dbObjIt.next().get("_id")).toString(); - - BSONObject file = fileIt.next(); - for (String fieldName : file.keySet()) { - byte[] bytes = null; - - Field field = bigFieldSet.get(fieldName); - if (null == field) { - continue; - } - FieldTypeEnum fieldType = field.getType(); - if (FieldTypeEnum.TEXT == fieldType) { - String value = (String) file.get(fieldName); - if (!StringUtils.isBlank(value)) { - try { - bytes = gzip(value); - } catch (IOException e) { - throw new IllegalArgumentException("unsupported encoding value!"); - } - } - } - - if (null != bytes && bytes.length > 0) { - GridFSFile mongofile = gridFS.createFile(bytes); - mongofile.put("uploadDate", new Date()); - mongofile.put("filename", id); - mongofile.put("contentType", fieldName); - mongofile.save(); - } - } - } - } - - private byte[] gzip(String string) throws IOException { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream); - gzipOutputStream.write(string.getBytes(DEFAULT_CHARSET)); - return outputStream.toByteArray(); - } - - public void remove(List objIdList) { - remove(objIdList, null); - } - - public void remove(List objIdList, Set removeFields) { - // 删除表的所有大字段 - if (null == removeFields) - removeFields = bigFieldSet.keySet(); - - // 若到删除的字段中没有大字段,直接返回 - if (removeFields.size() == 0) - return; - - List idList = new ArrayList(objIdList.size()); - for (ObjectId objId : objIdList) { - idList.add(objId.toString()); - } - - // 删除大字段关联的GridFS - DBObject fileSeletor = new BasicDBObject(); - fileSeletor.put("filename", new BasicDBObject("$in", idList)); - fileSeletor.put("contentType", new BasicDBObject("$in", removeFields)); - gridFS.remove(fileSeletor); - } - - public void update(List objIdList, BSONObject data) throws DataStoreException { - // 待删除的字段 - BSONObject unset = (BSONObject) data.get("$unset"); - if (null != unset) { - // 待删除的大字段set - Set removeSet = new HashSet(); - for (String field : unset.keySet()) { - if (bigFieldSet.containsKey(field)) - removeSet.add(field); - } - - if (removeSet.size() > 0) - remove(objIdList, removeSet); - } - - // 更新数据 - BSONObject set = (BSONObject) data.get("$set"); - if (null == set) - return; - - Map fileMap = new HashMap(); - for (String field : set.keySet()) { - if (bigFieldSet.containsKey(field)) { - fileMap.put(field, set.get(field)); - } - } - - // 更新数据中没有大字段 - if (fileMap.size() == 0) - return; - - // 删除大字段关联的GridFS - remove(objIdList, fileMap.keySet()); - - // 将新的内容存入GridFS - for (ObjectId objId : objIdList) { - String id = objId.toString(); - for (String fieldName : fileMap.keySet()) { - byte[] bytes = null; - - Field field = bigFieldSet.get(fieldName); - if (null == field) { - continue; - } - FieldTypeEnum fieldType = field.getType(); - if (FieldTypeEnum.TEXT == fieldType) { - String value = (String) fileMap.get(fieldName); - if (!StringUtils.isBlank(value)) { - try { - bytes = gzip(value); - } catch (IOException e) { - throw new IllegalArgumentException("unsupported encoding value!"); - } - } - } - - if (null != bytes && bytes.length > 0) { - GridFSFile mongofile = gridFS.createFile(bytes); - mongofile.put("uploadDate", new Date()); - mongofile.put("filename", id); - mongofile.put("contentType", fieldName); - mongofile.save(); - } - } - } - - } - - public List find(List docList, BSONObject fields) throws DataStoreException { - // 判断查询的字段中有没有大字段 - Set queryBigField = new HashSet(); - if (null == fields) { - queryBigField = bigFieldSet.keySet(); - } else { - for (String field : fields.keySet()) { - if (bigFieldSet.containsKey(field)) { - queryBigField.add(field); - } - } - } - - // 没有大字段,则返回原结果文档 - if (queryBigField.size() == 0) - return docList; - - Map docMap = new HashMap(docList.size()); - List idList = new ArrayList(docList.size()); - for (DBObject dbObj : docList) { - ObjectId objId = (ObjectId) dbObj.get("_id"); - if (null != objId) { - String id = objId.toString(); - idList.add(id); - docMap.put(id, dbObj); - } - } - - DBObject fileSeletor = new BasicDBObject(); - fileSeletor.put("filename", new BasicDBObject("$in", idList)); - fileSeletor.put("contentType", new BasicDBObject("$in", queryBigField)); - List fileList = gridFS.find(fileSeletor); - - for (GridFSDBFile file : fileList) { - // 对应主记录id - String id = file.getFilename(); - // 对应主记录大字段名 - String fieldName = file.getContentType(); - - DBObject obj = docMap.get(id); - if (obj.keySet().contains(fieldName)) { - - try { - ByteArrayOutputStream data = new ByteArrayOutputStream(); - GZIPOutputStream out = new GZIPOutputStream(data); - IOUtils.copy(file.getInputStream(), out); - - Field field = bigFieldSet.get(fieldName); - if (null == field) { - continue; - } - FieldTypeEnum fieldType = field.getType(); - if (FieldTypeEnum.TEXT == fieldType) { - obj.put(fieldName, new String(data.toByteArray(), DEFAULT_CHARSET)); - } - } catch (IOException e) { - throw new DataStoreException("error extract text field[" + fieldName + "]:", e); - } - } - } - - return docList; - } +public class FileStoreImpl implements FileStore +{ + private GridFS gridFS; + private Map bigFieldSet; + + public FileStoreImpl(final GridFS gridFS, final Collection fields) { + this.gridFS = gridFS; + this.bigFieldSet = new HashMap(); + for (final Field field : fields) { + if (field.isBigField()) { + this.bigFieldSet.put(field.getName(), field); + } + } + } + + @Override + public void save(final List fileList, final List dbObjList) { + if (null == fileList || null == dbObjList || fileList.size() != dbObjList.size()) { + throw new IllegalArgumentException("empty list or unequal length list is not allowed!"); + } + final Iterator dbObjIt = dbObjList.iterator(); + final Iterator fileIt = fileList.iterator(); + while (fileIt.hasNext()) { + final String id = ((ObjectId)dbObjIt.next().get("_id")).toString(); + final BSONObject file = (BSONObject)fileIt.next(); + for (final String fieldName : file.keySet()) { + byte[] bytes = null; + final Field field = this.bigFieldSet.get(fieldName); + if (null == field) { + continue; + } + final FieldTypeEnum fieldType = field.getType(); + if (FieldTypeEnum.TEXT == fieldType) { + final String value = (String)file.get(fieldName); + if (!StringUtils.isBlank(value)) { + try { + bytes = this.gzip(value); + } + catch (IOException e) { + throw new IllegalArgumentException("unsupported encoding value!"); + } + } + } + if (null == bytes || bytes.length <= 0) { + continue; + } + final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes); + mongofile.put("uploadDate", (Object)new Date()); + mongofile.put("filename", (Object)id); + mongofile.put("contentType", (Object)fieldName); + mongofile.save(); + } + } + } + + private byte[] gzip(final String string) throws IOException { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(outputStream); + gzipOutputStream.write(string.getBytes("UTF-8")); + return outputStream.toByteArray(); + } + + @Override + public void remove(final List objIdList) { + this.remove(objIdList, null); + } + + @Override + public void remove(final List objIdList, Set removeFields) { + if (null == removeFields) { + removeFields = this.bigFieldSet.keySet(); + } + if (removeFields.size() == 0) { + return; + } + final List idList = new ArrayList(objIdList.size()); + for (final ObjectId objId : objIdList) { + idList.add(objId.toString()); + } + final DBObject fileSeletor = (DBObject)new BasicDBObject(); + fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList)); + fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)removeFields)); + this.gridFS.remove(fileSeletor); + } + + @Override + public void update(final List objIdList, final BSONObject data) throws DataStoreException { + final BSONObject unset = (BSONObject)data.get("$unset"); + if (null != unset) { + final Set removeSet = new HashSet(); + for (final String field : unset.keySet()) { + if (this.bigFieldSet.containsKey(field)) { + removeSet.add(field); + } + } + if (removeSet.size() > 0) { + this.remove(objIdList, removeSet); + } + } + final BSONObject set = (BSONObject)data.get("$set"); + if (null == set) { + return; + } + final Map fileMap = new HashMap(); + for (final String field2 : set.keySet()) { + if (this.bigFieldSet.containsKey(field2)) { + fileMap.put(field2, set.get(field2)); + } + } + if (fileMap.size() == 0) { + return; + } + this.remove(objIdList, fileMap.keySet()); + for (final ObjectId objId : objIdList) { + final String id = objId.toString(); + for (final String fieldName : fileMap.keySet()) { + byte[] bytes = null; + final Field field3 = this.bigFieldSet.get(fieldName); + if (null == field3) { + continue; + } + final FieldTypeEnum fieldType = field3.getType(); + if (FieldTypeEnum.TEXT == fieldType) { + final String value = (String)fileMap.get(fieldName); + if (!StringUtils.isBlank(value)) { + try { + bytes = this.gzip(value); + } + catch (IOException e) { + throw new IllegalArgumentException("unsupported encoding value!"); + } + } + } + if (null == bytes || bytes.length <= 0) { + continue; + } + final GridFSFile mongofile = (GridFSFile)this.gridFS.createFile(bytes); + mongofile.put("uploadDate", (Object)new Date()); + mongofile.put("filename", (Object)id); + mongofile.put("contentType", (Object)fieldName); + mongofile.save(); + } + } + } + + @Override + public List find(final List docList, final BSONObject fields) throws DataStoreException { + Set queryBigField = new HashSet(); + if (null == fields) { + queryBigField = this.bigFieldSet.keySet(); + } + else { + for (final String field : fields.keySet()) { + if (this.bigFieldSet.containsKey(field)) { + queryBigField.add(field); + } + } + } + if (queryBigField.size() == 0) { + return docList; + } + final Map docMap = new HashMap(docList.size()); + final List idList = new ArrayList(docList.size()); + for (final DBObject dbObj : docList) { + final ObjectId objId = (ObjectId)dbObj.get("_id"); + if (null != objId) { + final String id = objId.toString(); + idList.add(id); + docMap.put(id, dbObj); + } + } + final DBObject fileSeletor = (DBObject)new BasicDBObject(); + fileSeletor.put("filename", (Object)new BasicDBObject("$in", (Object)idList)); + fileSeletor.put("contentType", (Object)new BasicDBObject("$in", (Object)queryBigField)); + final List fileList = (List)this.gridFS.find(fileSeletor); + for (final GridFSDBFile file : fileList) { + final String id2 = file.getFilename(); + final String fieldName = file.getContentType(); + final DBObject obj = docMap.get(id2); + if (obj.keySet().contains(fieldName)) { + try { + final ByteArrayOutputStream data = new ByteArrayOutputStream(); + final GZIPOutputStream out = new GZIPOutputStream(data); + IOUtils.copy(file.getInputStream(), (OutputStream)out); + final Field field2 = this.bigFieldSet.get(fieldName); + if (null == field2) { + continue; + } + final FieldTypeEnum fieldType = field2.getType(); + if (FieldTypeEnum.TEXT != fieldType) { + continue; + } + obj.put(fieldName, (Object)new String(data.toByteArray(), "UTF-8")); + } + catch (IOException e) { + throw new DataStoreException("error extract text field[" + fieldName + "]:", e); + } + } + } + return docList; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java index 1955bee2c..bb424447a 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/Mongo.java @@ -1,46 +1,46 @@ package com.fangjia.datastore; -import java.util.List; - -import com.fangjia.datastore.config.MongoConfig; -import com.mongodb.DB; -import com.mongodb.MongoException; import com.mongodb.WriteConcern; +import com.mongodb.MongoException; +import java.util.List; +import com.mongodb.DB; +import com.fangjia.datastore.config.MongoConfig; -public class Mongo { - - private final MongoConfig mongoConfig; - - public Mongo(MongoConfig mongoConfig) { - this.mongoConfig = mongoConfig; - } - - private com.mongodb.Mongo getMongo() { - try { - return mongoConfig.getMongo(); - } catch (ConfigurationException e) { - throw new RuntimeException(e); - } - } - - public DB getDB(String dbname) { - return getMongo().getDB(dbname); - } - - public List getDatabaseNames() throws MongoException { - return getMongo().getDatabaseNames(); - } - - public void setWriteConcern(WriteConcern concern) { - getMongo().setWriteConcern(concern); - } - - public WriteConcern getWriteConcern() { - return getMongo().getWriteConcern(); - } - - public String toString() { - return getMongo().toString(); - } - +public class Mongo +{ + private final MongoConfig mongoConfig; + + public Mongo(final MongoConfig mongoConfig) { + this.mongoConfig = mongoConfig; + } + + private com.mongodb.Mongo getMongo() { + try { + return this.mongoConfig.getMongo(); + } + catch (ConfigurationException e) { + throw new RuntimeException(e); + } + } + + public DB getDB(final String dbname) { + return this.getMongo().getDB(dbname); + } + + public List getDatabaseNames() throws MongoException { + return (List)this.getMongo().getDatabaseNames(); + } + + public void setWriteConcern(final WriteConcern concern) { + this.getMongo().setWriteConcern(concern); + } + + public WriteConcern getWriteConcern() { + return this.getMongo().getWriteConcern(); + } + + @Override + public String toString() { + return this.getMongo().toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java index 26d62b527..7ae1f812f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/PerformanceLogFilter.java @@ -1,114 +1,119 @@ package com.fangjia.datastore; -import org.apache.mina.core.filterchain.IoFilterAdapter; -import org.apache.mina.core.session.IdleStatus; -import org.apache.mina.core.session.IoSession; -import org.apache.mina.core.write.WriteRequest; -import org.slf4j.Logger; - -import com.fangjia.datastore.mongo.protocol.MongoMessage; import com.fangjia.datastore.utils.LoggerFactory; +import org.apache.mina.core.session.IdleStatus; +import org.apache.mina.core.write.WriteRequest; +import org.apache.mina.core.filterchain.IoFilter; +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import org.apache.mina.core.session.IoSession; +import org.slf4j.Logger; +import org.apache.mina.core.filterchain.IoFilterAdapter; -public class PerformanceLogFilter extends IoFilterAdapter { - private final long maxTime; - - private static final Logger LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class); - - public PerformanceLogFilter(long maxTime) { - this.maxTime = maxTime; - } - - private void logPerformace(long time, String methodName, IoSession session, Object message) { - Object[] params; - if (message instanceof MongoMessage) { - MongoMessage mongoMessage = (MongoMessage) message; - params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() }; - } else { - params = new Object[] { time, methodName, session, message }; - } - if (time < maxTime * 3) { - LOGGER.warn("time={}, methodName={}, session={}, message={}", params); - } else { - LOGGER.error("time={}, methodName={}, session={}, message={}", params); - } - } - - @Override - public void messageReceived(NextFilter nextFilter, IoSession session, Object message) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.messageReceived(session, message); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "messageReceived", session, message); - } - } - } - - @Override - public void messageSent(NextFilter nextFilter, IoSession session, WriteRequest writeRequest) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.messageSent(session, writeRequest); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "messageSent", session, writeRequest); - } - } - } - - @Override - public void sessionCreated(NextFilter nextFilter, IoSession session) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.sessionCreated(session); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "sessionCreated", session, null); - } - } - } - - @Override - public void sessionOpened(NextFilter nextFilter, IoSession session) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.sessionOpened(session); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "sessionOpened", session, null); - } - } - } - - @Override - public void sessionIdle(NextFilter nextFilter, IoSession session, IdleStatus status) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.sessionIdle(session, status); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "sessionIdle", session, status); - } - } - } - - @Override - public void sessionClosed(NextFilter nextFilter, IoSession session) throws Exception { - long time = System.currentTimeMillis(); - try { - nextFilter.sessionClosed(session); - } finally { - time = System.currentTimeMillis() - time; - if (time >= maxTime) { - logPerformace(time, "sessionClosed", session, null); - } - } - } - +public class PerformanceLogFilter extends IoFilterAdapter +{ + private final long maxTime; + private static final Logger LOGGER; + + public PerformanceLogFilter(final long maxTime) { + this.maxTime = maxTime; + } + + private void logPerformace(final long time, final String methodName, final IoSession session, final Object message) { + Object[] params; + if (message instanceof MongoMessage) { + final MongoMessage mongoMessage = (MongoMessage)message; + params = new Object[] { time, methodName, session, mongoMessage.toPrettyString() }; + } + else { + params = new Object[] { time, methodName, session, message }; + } + if (time < this.maxTime * 3L) { + PerformanceLogFilter.LOGGER.warn("time={}, methodName={}, session={}, message={}", params); + } + else { + PerformanceLogFilter.LOGGER.error("time={}, methodName={}, session={}, message={}", params); + } + } + + public void messageReceived(final NextFilter nextFilter, final IoSession session, final Object message) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.messageReceived(session, message); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "messageReceived", session, message); + } + } + } + + public void messageSent(final NextFilter nextFilter, final IoSession session, final WriteRequest writeRequest) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.messageSent(session, writeRequest); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "messageSent", session, writeRequest); + } + } + } + + public void sessionCreated(final NextFilter nextFilter, final IoSession session) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.sessionCreated(session); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "sessionCreated", session, null); + } + } + } + + public void sessionOpened(final NextFilter nextFilter, final IoSession session) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.sessionOpened(session); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "sessionOpened", session, null); + } + } + } + + public void sessionIdle(final NextFilter nextFilter, final IoSession session, final IdleStatus status) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.sessionIdle(session, status); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "sessionIdle", session, status); + } + } + } + + public void sessionClosed(final NextFilter nextFilter, final IoSession session) throws Exception { + long time = System.currentTimeMillis(); + try { + nextFilter.sessionClosed(session); + } + finally { + time = System.currentTimeMillis() - time; + if (time >= this.maxTime) { + this.logPerformace(time, "sessionClosed", session, null); + } + } + } + + static { + LOGGER = LoggerFactory.getPerformanceLogger(PerformanceLogFilter.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java index 224844bef..f5392909b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/QueryResponse.java @@ -1,72 +1,80 @@ package com.fangjia.datastore; import java.util.Collections; +import org.apache.solr.client.solrj.response.FacetField; +import com.mongodb.DBObject; +import org.bson.types.ObjectId; import java.util.List; -import org.apache.solr.client.solrj.response.FacetField; -import org.bson.types.ObjectId; - -import com.mongodb.DBObject; - -public class QueryResponse { - private List idList = Collections.emptyList(); - private List docList = Collections.emptyList(); - private List facetFieldList = null; - private int numFound; - private int numReturned; - private int numToSkip; - public final static QueryResponse EMPTY_RESPONSE = new QueryResponse(); - - public List getIdList() { - return idList; - } - - public void setIdList(List idList) { - this.idList = idList; - } - - public boolean hasMore() { - return this.numFound > numReturned + numToSkip; - } - - public int getNumFound() { - return numFound; - } - - public void setNumFound(int numFound) { - this.numFound = numFound; - } - - public int getNumReturned() { - return numReturned; - } - - public void setNumReturned(int numReturned) { - this.numReturned = numReturned; - } - - public int getNumToSkip() { - return numToSkip; - } - - public void setNumToSkip(int numToSkip) { - this.numToSkip = numToSkip; - } - - public List getFacetFieldList() { - return facetFieldList; - } - - public void setFacetFieldList(List facetFieldList) { - this.facetFieldList = facetFieldList; - } - - public List getDocList() { - return docList; - } - - public void setDocList(List docList) { - this.docList = docList; - } - +public class QueryResponse +{ + private List idList; + private List docList; + private List facetFieldList; + private int numFound; + private int numReturned; + private int numToSkip; + public static final QueryResponse EMPTY_RESPONSE; + + public QueryResponse() { + this.idList = Collections.emptyList(); + this.docList = Collections.emptyList(); + this.facetFieldList = null; + } + + public List getIdList() { + return this.idList; + } + + public void setIdList(final List idList) { + this.idList = idList; + } + + public boolean hasMore() { + return this.numFound > this.numReturned + this.numToSkip; + } + + public int getNumFound() { + return this.numFound; + } + + public void setNumFound(final int numFound) { + this.numFound = numFound; + } + + public int getNumReturned() { + return this.numReturned; + } + + public void setNumReturned(final int numReturned) { + this.numReturned = numReturned; + } + + public int getNumToSkip() { + return this.numToSkip; + } + + public void setNumToSkip(final int numToSkip) { + this.numToSkip = numToSkip; + } + + public List getFacetFieldList() { + return this.facetFieldList; + } + + public void setFacetFieldList(final List facetFieldList) { + this.facetFieldList = facetFieldList; + } + + public List getDocList() { + return this.docList; + } + + public void setDocList(final List docList) { + this.docList = docList; + } + + static { + EMPTY_RESPONSE = new QueryResponse(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java index f2e1546b4..590bb13d9 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecuteException.java @@ -1,22 +1,21 @@ package com.fangjia.datastore; -public class TriggerExecuteException extends Exception { - - private static final long serialVersionUID = -1165411196743370433L; - - public TriggerExecuteException() { - } - - public TriggerExecuteException(String message) { - super(message); - } - - public TriggerExecuteException(Throwable cause) { - super(cause); - } - - public TriggerExecuteException(String message, Throwable cause) { - super(message, cause); - } - +public class TriggerExecuteException extends Exception +{ + private static final long serialVersionUID = -1165411196743370433L; + + public TriggerExecuteException() { + } + + public TriggerExecuteException(final String message) { + super(message); + } + + public TriggerExecuteException(final Throwable cause) { + super(cause); + } + + public TriggerExecuteException(final String message, final Throwable cause) { + super(message, cause); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java index b998e420e..ce37f7653 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/TriggerExecutor.java @@ -1,165 +1,161 @@ package com.fangjia.datastore; -import java.io.IOException; +import org.slf4j.LoggerFactory; +import org.apache.http.HttpRequest; import java.io.InputStream; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.List; - -import org.apache.http.HttpHost; import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.params.AllClientPNames; -import org.apache.http.conn.scheme.PlainSocketFactory; -import org.apache.http.conn.scheme.Scheme; -import org.apache.http.conn.scheme.SchemeRegistry; +import com.mongodb.DBObject; +import org.bson.BasicBSONDecoder; +import org.apache.http.protocol.HttpContext; +import org.apache.http.HttpHost; +import org.apache.http.HttpEntity; import org.apache.http.entity.ByteArrayEntity; +import org.sothis.core.util.GZIPUtils; +import org.bson.BasicBSONEncoder; +import com.mongodb.BasicDBObject; +import org.apache.http.client.methods.HttpPost; +import java.net.URL; +import java.net.URISyntaxException; +import java.io.IOException; +import org.bson.BSONObject; +import java.util.List; +import com.fangjia.datastore.config.Trigger; +import org.apache.http.params.HttpParams; +import org.apache.http.conn.ClientConnectionManager; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager; import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.CoreProtocolPNames; -import org.apache.http.params.HttpParams; -import org.apache.http.protocol.HttpContext; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; +import org.apache.http.conn.scheme.SchemeSocketFactory; +import org.apache.http.conn.scheme.Scheme; +import org.apache.http.conn.scheme.PlainSocketFactory; +import org.apache.http.conn.scheme.SchemeRegistry; +import org.apache.http.client.HttpClient; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sothis.core.util.GZIPUtils; -import com.fangjia.datastore.config.Trigger; -import com.mongodb.BasicDBObject; -import com.mongodb.DBObject; - -public class TriggerExecutor { - - private final static int MAX_TOTAL = 200; - private final static int MAX_FOR_ROUTE = 200; - private final static String DEFAULT_CHARSET = "UTF-8"; - private final static int DEFAULT_PORT = 80; - private final static int MAX_RETRY_COUNT = 10; - - private final static Logger LOGGER = LoggerFactory.getLogger(TriggerExecutor.class); - private final static TriggerExecutor TRIGGER_EXECUTOR = new TriggerExecutor(); - - private final HttpClient httpClient; - - private TriggerExecutor() { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("init trigger executor"); - } - SchemeRegistry schemeRegistry = new SchemeRegistry(); - schemeRegistry.register(new Scheme("http", DEFAULT_PORT, new PlainSocketFactory())); - - HttpParams httpParams = new BasicHttpParams(); - httpParams.setParameter(CoreProtocolPNames.HTTP_CONTENT_CHARSET, DEFAULT_CHARSET); - httpParams.setParameter(AllClientPNames.SO_TIMEOUT, 300000); - httpParams.setParameter(AllClientPNames.CONNECTION_TIMEOUT, 60000); - - ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry); - connManager.setMaxTotal(MAX_TOTAL); - connManager.setDefaultMaxPerRoute(MAX_FOR_ROUTE); - - httpClient = new DefaultHttpClient(connManager, httpParams); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("trigger executor initialized[{}]", toString()); - } - } - - public static TriggerExecutor getInstance() { - return TRIGGER_EXECUTOR; - } - - public List execute(Trigger trigger, List docList) throws TriggerExecuteException { - try { - List ret = doExecute(trigger, docList); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size()); - } - return ret; - } catch (IOException e) { - throw new TriggerExecuteException("error execute trigger: " + trigger, e); - } catch (URISyntaxException e) { - throw new TriggerExecuteException("error execute trigger: " + trigger, e); - } - } - - private List doExecute(Trigger trigger, List docList) throws IOException, URISyntaxException, - TriggerExecuteException { - URL url = new URL(trigger.getInterface()); - HttpPost httpPost = new HttpPost(url.toURI()); - DBObject object = new BasicDBObject(); - object.put("doc_list_size", docList.size()); - object.put("doc_list", docList); - httpPost.setEntity(new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BSONEncoder().encode(object)))); - HttpHost host = new HttpHost(url.getHost(), url.getPort() == -1 ? DEFAULT_PORT : url.getPort()); - HttpResponse httpResponse = executeWithRetry(host, httpPost, null, trigger); - InputStream inputStream = httpResponse.getEntity().getContent(); - try { - if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream)); - if (null != returnObject) { - List ret = (List) returnObject.get("doc_list"); - if (ret == null) { - throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]"); - } - return ret; - } else { - throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !"); - } - } else { - throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:" - + httpResponse.getStatusLine().getStatusCode()); - } - } finally { - if (null != inputStream) { - inputStream.close(); - } - } - } - - private HttpResponse executeWithRetry(HttpHost host, HttpPost httpPost, HttpContext httpContext, Trigger trigger) - throws IOException, TriggerExecuteException { - int retryCounter = 0; - HttpResponse httpResponse = null; - while (true) { - try { - httpResponse = httpClient.execute(host, httpPost, (HttpContext) null); - if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { - return httpResponse; - } else { - retryCounter++; - sleepAndRetry(new IllegalStateException("invalid http status code: " - + httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger); - } - } catch (Exception e) { - retryCounter++; - sleepAndRetry(e, retryCounter, trigger); - } finally { - if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - httpResponse.getEntity().getContent().close(); - } - } - } - } - - private void sleepAndRetry(Exception e, int retryCounter, Trigger trigger) throws TriggerExecuteException { - if (retryCounter > MAX_RETRY_COUNT) { - throw new TriggerExecuteException( - "error execute trigger[" + trigger + "] after " + MAX_RETRY_COUNT + " times retry!", e); - } - if (LOGGER.isErrorEnabled()) { - LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter - + "th time after " + (retryCounter * 30) + " seconds..."); - } - try { - Thread.sleep(retryCounter * 30000); - } catch (InterruptedException e1) { - if (LOGGER.isWarnEnabled()) { - LOGGER.warn("", e); - } - } - } +public class TriggerExecutor +{ + private static final int MAX_TOTAL = 200; + private static final int MAX_FOR_ROUTE = 200; + private static final String DEFAULT_CHARSET = "UTF-8"; + private static final int DEFAULT_PORT = 80; + private static final int MAX_RETRY_COUNT = 10; + private static final Logger LOGGER; + private static final TriggerExecutor TRIGGER_EXECUTOR; + private final HttpClient httpClient; + + private TriggerExecutor() { + if (TriggerExecutor.LOGGER.isDebugEnabled()) { + TriggerExecutor.LOGGER.debug("init trigger executor"); + } + final SchemeRegistry schemeRegistry = new SchemeRegistry(); + schemeRegistry.register(new Scheme("http", 80, (SchemeSocketFactory)new PlainSocketFactory())); + final HttpParams httpParams = (HttpParams)new BasicHttpParams(); + httpParams.setParameter("http.protocol.content-charset", (Object)"UTF-8"); + httpParams.setParameter("http.socket.timeout", (Object)300000); + httpParams.setParameter("http.connection.timeout", (Object)60000); + final ThreadSafeClientConnManager connManager = new ThreadSafeClientConnManager(schemeRegistry); + connManager.setMaxTotal(200); + connManager.setDefaultMaxPerRoute(200); + this.httpClient = (HttpClient)new DefaultHttpClient((ClientConnectionManager)connManager, httpParams); + if (TriggerExecutor.LOGGER.isDebugEnabled()) { + TriggerExecutor.LOGGER.debug("trigger executor initialized[{}]", (Object)this.toString()); + } + } + + public static TriggerExecutor getInstance() { + return TriggerExecutor.TRIGGER_EXECUTOR; + } + + public List execute(final Trigger trigger, final List docList) throws TriggerExecuteException { + try { + final List ret = this.doExecute(trigger, docList); + if (TriggerExecutor.LOGGER.isDebugEnabled()) { + TriggerExecutor.LOGGER.debug("execute trigger: " + trigger + ", doc size:" + docList.size() + ", return size:" + ret.size()); + } + return ret; + } + catch (IOException e) { + throw new TriggerExecuteException("error execute trigger: " + trigger, e); + } + catch (URISyntaxException e2) { + throw new TriggerExecuteException("error execute trigger: " + trigger, e2); + } + } + + private List doExecute(final Trigger trigger, final List docList) throws IOException, URISyntaxException, TriggerExecuteException { + final URL url = new URL(trigger.getInterface()); + final HttpPost httpPost = new HttpPost(url.toURI()); + final DBObject object = (DBObject)new BasicDBObject(); + object.put("doc_list_size", (Object)docList.size()); + object.put("doc_list", (Object)docList); + httpPost.setEntity((HttpEntity)new ByteArrayEntity(GZIPUtils.makeAsByteArray(new BasicBSONEncoder().encode((BSONObject)object)))); + final HttpHost host = new HttpHost(url.getHost(), (url.getPort() == -1) ? 80 : url.getPort()); + final HttpResponse httpResponse = this.executeWithRetry(host, httpPost, null, trigger); + final InputStream inputStream = httpResponse.getEntity().getContent(); + try { + if (httpResponse.getStatusLine().getStatusCode() != 200) { + throw new TriggerExecuteException("error execute trigger:" + trigger + ", get http code:" + httpResponse.getStatusLine().getStatusCode()); + } + final BSONObject returnObject = new BasicBSONDecoder().readObject(GZIPUtils.wrap(inputStream)); + if (null == returnObject) { + throw new TriggerExecuteException("trigger" + trigger + " returns nothing at all !"); + } + final List ret = (List)returnObject.get("doc_list"); + if (ret == null) { + throw new TriggerExecuteException("trigger" + trigger + " return an null doc_list[" + ret + "]"); + } + return ret; + } + finally { + if (null != inputStream) { + inputStream.close(); + } + } + } + + private HttpResponse executeWithRetry(final HttpHost host, final HttpPost httpPost, final HttpContext httpContext, final Trigger trigger) throws IOException, TriggerExecuteException { + int retryCounter = 0; + HttpResponse httpResponse = null; + while (true) { + try { + httpResponse = this.httpClient.execute(host, (HttpRequest)httpPost, (HttpContext)null); + if (httpResponse.getStatusLine().getStatusCode() == 200) { + return httpResponse; + } + ++retryCounter; + this.sleepAndRetry(new IllegalStateException("invalid http status code: " + httpResponse.getStatusLine().getStatusCode()), retryCounter, trigger); + } + catch (Exception e) { + ++retryCounter; + this.sleepAndRetry(e, retryCounter, trigger); + } + finally { + if (null != httpResponse && httpResponse.getStatusLine().getStatusCode() != 200) { + httpResponse.getEntity().getContent().close(); + } + } + } + } + + private void sleepAndRetry(final Exception e, final int retryCounter, final Trigger trigger) throws TriggerExecuteException { + if (retryCounter > 10) { + throw new TriggerExecuteException("error execute trigger[" + trigger + "] after " + 10 + " times retry!", e); + } + if (TriggerExecutor.LOGGER.isErrorEnabled()) { + TriggerExecutor.LOGGER.error("error execute trigger[" + trigger + "]:(" + e.getMessage() + "), retry for the " + retryCounter + "th time after " + retryCounter * 30 + " seconds..."); + } + try { + Thread.sleep(retryCounter * 30000); + } + catch (InterruptedException e2) { + if (TriggerExecutor.LOGGER.isWarnEnabled()) { + TriggerExecutor.LOGGER.warn("", (Throwable)e); + } + } + } + + static { + LOGGER = LoggerFactory.getLogger((Class)TriggerExecutor.class); + TRIGGER_EXECUTOR = new TriggerExecutor(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java index b70a6f73a..f532a2655 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/DataStoreConfig.java @@ -1,170 +1,172 @@ package com.fangjia.datastore.config; -import java.io.File; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.dom4j.Document; import org.dom4j.DocumentException; +import java.io.File; +import java.util.Iterator; +import java.util.List; import org.dom4j.Element; import org.dom4j.Node; +import org.dom4j.Document; +import java.util.Collections; +import java.util.Set; +import com.fangjia.datastore.ConfigurationException; +import java.util.HashMap; +import java.util.Map; import org.dom4j.io.SAXReader; -import com.fangjia.datastore.ConfigurationException; - -public class DataStoreConfig implements Reloadable { - private final SAXReader reader = new SAXReader(); - private final String serverConfigFile; - private final String storeConfigFile; - private final ServerConfig serverConfig; - private final Map storeConfigs; - private final Map solrConfigs; - private final Map mongoConfigs; - private boolean readOnly = false; - - public DataStoreConfig(String datastoreHome) throws ConfigurationException { - this.serverConfigFile = datastoreHome + "/conf/server.xml"; - this.storeConfigFile = datastoreHome + "/conf/store.xml"; - this.serverConfig = new ServerConfig(); - this.storeConfigs = new HashMap(); - this.solrConfigs = new HashMap(); - this.mongoConfigs = new HashMap(); - load(); - } - - public ServerConfig getServerConfig() { - return serverConfig; - } - - public SolrConfig getSolrConfigById(String id) { - return solrConfigs.get(id); - } - - public MongoConfig getMongoConfigById(String id) { - return this.mongoConfigs.get(id); - } - - public StoreConfig getStoreConfig(String storeName) { - return this.storeConfigs.get(storeName); - } - - public Set getStoreNames() { - return Collections.unmodifiableSet(this.storeConfigs.keySet()); - } - - public void reload() throws ConfigurationException { - load(); - } - - private synchronized void load() throws ConfigurationException { - Document document = read(this.serverConfigFile); - serverConfig.reload(document.selectSingleNode("//datastore//server")); - initSolrConfigs(document); - initMongoConfigs(document); - document = read(this.storeConfigFile); - initStoreConfigs(document, solrConfigs, mongoConfigs); - } - - synchronized StoreConfig getOrCreateStoreConfig(String storeName) { - StoreConfig ret = this.storeConfigs.get(storeName); - if (null == ret) { - ret = new StoreConfig(this, storeName); - storeConfigs.put(storeName, ret); - } - return ret; - } - - private synchronized SolrConfig getOrCreateSolrConfig(String solrId) { - SolrConfig ret = this.solrConfigs.get(solrId); - if (null == ret) { - ret = new SolrConfig(this, solrId); - solrConfigs.put(solrId, ret); - } - return ret; - } - - private synchronized MongoConfig getOrCreateMongoConfig(String mongoId) throws ConfigurationException { - MongoConfig ret = this.mongoConfigs.get(mongoId); - if (null == ret) { - ret = new MongoConfig(mongoId); - mongoConfigs.put(mongoId, ret); - } - return ret; - } - - private void initSolrConfigs(Document document) throws ConfigurationException { - List solrNodes = document.selectNodes("//datastore//solr"); - if (null == solrNodes || solrNodes.isEmpty()) { - throw new ConfigurationException("no solr is configured in " + serverConfigFile + " !"); - } - Map newSolrConfigs = new HashMap(); - for (Node solrNode : solrNodes) { - String id = ((Element) solrNode).attributeValue("id"); - SolrConfig config = getOrCreateSolrConfig(id); - config.reload(solrNode); - newSolrConfigs.put(id, config); - } - this.solrConfigs.clear(); - this.solrConfigs.putAll(newSolrConfigs); - } - - private void initMongoConfigs(Document document) throws ConfigurationException { - List mongoNodes = document.selectNodes("//datastore//mongo"); - if (null == mongoNodes || mongoNodes.isEmpty()) { - throw new ConfigurationException("no mongo is configured in " + serverConfigFile + " !"); - } - Map newMongoConfigs = new HashMap(); - for (Node mongoNode : mongoNodes) { - String id = ((Element) mongoNode).attributeValue("id"); - MongoConfig mongoConfig = getOrCreateMongoConfig(id); - mongoConfig.reload(mongoNode); - newMongoConfigs.put(id, mongoConfig); - } - this.mongoConfigs.clear(); - this.mongoConfigs.putAll(newMongoConfigs); - } - - private void initStoreConfigs(Document document, Map solrConfigMap, - Map mongoConfigMap) throws ConfigurationException { - List storeNodes = document.selectNodes("//datastore//store"); - if (null == storeNodes || storeNodes.isEmpty()) { - throw new ConfigurationException("no store is configured in " + storeConfigFile + " !"); - } - Map newStoreConfigs = new HashMap(); - for (Node storeNode : storeNodes) { - String name = ((Element) storeNode).attributeValue("name"); - final StoreConfig storeConfig = getOrCreateStoreConfig(name); - storeConfig.reload(storeNode); - newStoreConfigs.put(name, storeConfig); - } - this.storeConfigs.clear(); - this.storeConfigs.putAll(newStoreConfigs); - } - - private Document read(String configFile) throws ConfigurationException { - try { - return reader.read(new File(configFile)); - } catch (DocumentException e) { - throw new ConfigurationException("error read datastore config file: " + configFile, e); - } - } - - @Override - public String toString() { - StringBuilder ret = new StringBuilder(); - ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append( - this.storeConfigFile).append(", readOnly:").append(this.readOnly); - return ret.toString(); - } - - public boolean isReadOnly() { - return readOnly; - } - - public void setReadOnly(boolean readOnly) { - this.readOnly = readOnly; - } +public class DataStoreConfig implements Reloadable +{ + private final SAXReader reader; + private final String serverConfigFile; + private final String storeConfigFile; + private final ServerConfig serverConfig; + private final Map storeConfigs; + private final Map solrConfigs; + private final Map mongoConfigs; + private boolean readOnly; + + public DataStoreConfig(final String datastoreHome) throws ConfigurationException { + this.reader = new SAXReader(); + this.readOnly = false; + this.serverConfigFile = datastoreHome + "/conf/server.xml"; + this.storeConfigFile = datastoreHome + "/conf/store.xml"; + this.serverConfig = new ServerConfig(); + this.storeConfigs = new HashMap(); + this.solrConfigs = new HashMap(); + this.mongoConfigs = new HashMap(); + this.load(); + } + + public ServerConfig getServerConfig() { + return this.serverConfig; + } + + public SolrConfig getSolrConfigById(final String id) { + return this.solrConfigs.get(id); + } + + public MongoConfig getMongoConfigById(final String id) { + return this.mongoConfigs.get(id); + } + + public StoreConfig getStoreConfig(final String storeName) { + return this.storeConfigs.get(storeName); + } + + public Set getStoreNames() { + return Collections.unmodifiableSet((Set)this.storeConfigs.keySet()); + } + + @Override + public void reload() throws ConfigurationException { + this.load(); + } + + private synchronized void load() throws ConfigurationException { + Document document = this.read(this.serverConfigFile); + this.serverConfig.reload(document.selectSingleNode("//datastore//server")); + this.initSolrConfigs(document); + this.initMongoConfigs(document); + document = this.read(this.storeConfigFile); + this.initStoreConfigs(document, this.solrConfigs, this.mongoConfigs); + } + + synchronized StoreConfig getOrCreateStoreConfig(final String storeName) { + StoreConfig ret = this.storeConfigs.get(storeName); + if (null == ret) { + ret = new StoreConfig(this, storeName); + this.storeConfigs.put(storeName, ret); + } + return ret; + } + + private synchronized SolrConfig getOrCreateSolrConfig(final String solrId) { + SolrConfig ret = this.solrConfigs.get(solrId); + if (null == ret) { + ret = new SolrConfig(this, solrId); + this.solrConfigs.put(solrId, ret); + } + return ret; + } + + private synchronized MongoConfig getOrCreateMongoConfig(final String mongoId) throws ConfigurationException { + MongoConfig ret = this.mongoConfigs.get(mongoId); + if (null == ret) { + ret = new MongoConfig(mongoId); + this.mongoConfigs.put(mongoId, ret); + } + return ret; + } + + private void initSolrConfigs(final Document document) throws ConfigurationException { + final List solrNodes = (List)document.selectNodes("//datastore//solr"); + if (null == solrNodes || solrNodes.isEmpty()) { + throw new ConfigurationException("no solr is configured in " + this.serverConfigFile + " !"); + } + final Map newSolrConfigs = new HashMap(); + for (final Node solrNode : solrNodes) { + final String id = ((Element)solrNode).attributeValue("id"); + final SolrConfig config = this.getOrCreateSolrConfig(id); + config.reload(solrNode); + newSolrConfigs.put(id, config); + } + this.solrConfigs.clear(); + this.solrConfigs.putAll(newSolrConfigs); + } + + private void initMongoConfigs(final Document document) throws ConfigurationException { + final List mongoNodes = (List)document.selectNodes("//datastore//mongo"); + if (null == mongoNodes || mongoNodes.isEmpty()) { + throw new ConfigurationException("no mongo is configured in " + this.serverConfigFile + " !"); + } + final Map newMongoConfigs = new HashMap(); + for (final Node mongoNode : mongoNodes) { + final String id = ((Element)mongoNode).attributeValue("id"); + final MongoConfig mongoConfig = this.getOrCreateMongoConfig(id); + mongoConfig.reload(mongoNode); + newMongoConfigs.put(id, mongoConfig); + } + this.mongoConfigs.clear(); + this.mongoConfigs.putAll(newMongoConfigs); + } + + private void initStoreConfigs(final Document document, final Map solrConfigMap, final Map mongoConfigMap) throws ConfigurationException { + final List storeNodes = (List)document.selectNodes("//datastore//store"); + if (null == storeNodes || storeNodes.isEmpty()) { + throw new ConfigurationException("no store is configured in " + this.storeConfigFile + " !"); + } + final Map newStoreConfigs = new HashMap(); + for (final Node storeNode : storeNodes) { + final String name = ((Element)storeNode).attributeValue("name"); + final StoreConfig storeConfig = this.getOrCreateStoreConfig(name); + storeConfig.reload(storeNode); + newStoreConfigs.put(name, storeConfig); + } + this.storeConfigs.clear(); + this.storeConfigs.putAll(newStoreConfigs); + } + + private Document read(final String configFile) throws ConfigurationException { + try { + return this.reader.read(new File(configFile)); + } + catch (DocumentException e) { + throw new ConfigurationException("error read datastore config file: " + configFile, (Throwable)e); + } + } + + @Override + public String toString() { + final StringBuilder ret = new StringBuilder(); + ret.append("server config file:").append(this.serverConfigFile).append(", store config file: ").append(this.storeConfigFile).append(", readOnly:").append(this.readOnly); + return ret.toString(); + } + + public boolean isReadOnly() { + return this.readOnly; + } + + public void setReadOnly(final boolean readOnly) { + this.readOnly = readOnly; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java index 00c58cb2a..c1fad3456 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Field.java @@ -1,127 +1,118 @@ package com.fangjia.datastore.config; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; import org.bson.types.ObjectId; +import java.util.regex.Pattern; +import java.util.Date; +import java.util.Map; +import java.util.List; +import org.apache.commons.lang.StringUtils; -public class Field { - private String name; - private FieldTypeEnum type; - private boolean required; - - Field() { - } - - public String getName() { - return name; - } - - public FieldTypeEnum getType() { - return type; - } - - void setName(String name) { - if (StringUtils.isEmpty(name)) { - throw new IllegalArgumentException("invalid field name:" + name); - } - this.name = name; - } - - void setType(FieldTypeEnum type) { - if (null == type) { - throw new IllegalArgumentException("invalid type of field[" + name + "]:" + type); - } - this.type = type; - } - - public boolean isRequired() { - return required; - } - - void setRequired(boolean required) { - this.required = required; - } - - public boolean isBigField() { - return this.type == FieldTypeEnum.TEXT; - } - - /** - * 验证一个值是否匹配其对应的类型 - * - * @param value - * @return 匹配则返回true,不匹配返回false - */ - public boolean validate(Object value) { - if (this.required && value == null) { - return false; - } - if (value == null) { - return true; - } - Class valueClass = value.getClass(); - switch (this.type) { - case BOOLEAN: - return valueClass == boolean.class || valueClass == Boolean.class; - case BYTE: - return valueClass == byte.class || valueClass == Byte.class; - case INTEGER: - return valueClass == int.class || valueClass == Integer.class; - case LONG: - return valueClass == long.class || valueClass == Long.class; - case DOUBLE: - return valueClass == double.class || valueClass == Double.class; - case STRING: - case TEXT: - return valueClass == String.class; - case LIST: - return value instanceof List; - case MAP: - return value instanceof Map; - case DATETIME: - return value instanceof Date; - case BINARY: - return valueClass == byte[].class; - case REGEXP: - return valueClass == Pattern.class; - case OBJECTID: - return valueClass == ObjectId.class; - default: - throw new RuntimeException("un cased type:" + this); - } - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(this.name).append(this.type).append(this.required).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (null == obj) { - return false; - } - if (!(obj instanceof Field)) { - return false; - } - Field other = (Field) obj; - return new EqualsBuilder().append(this.name, other.name).append(this.type, other.type) - .append(this.required, other.required).isEquals(); - } - - @Override - public String toString() { - return new StringBuilder().append("{field:").append(name).append(", type:").append(type).append(", required:") - .append(required).append("}").toString(); - } - +public class Field +{ + private String name; + private FieldTypeEnum type; + private boolean required; + + Field() { + } + + public String getName() { + return this.name; + } + + public FieldTypeEnum getType() { + return this.type; + } + + void setName(final String name) { + if (StringUtils.isEmpty(name)) { + throw new IllegalArgumentException("invalid field name:" + name); + } + this.name = name; + } + + void setType(final FieldTypeEnum type) { + if (null == type) { + throw new IllegalArgumentException("invalid type of field[" + this.name + "]:" + type); + } + this.type = type; + } + + public boolean isRequired() { + return this.required; + } + + void setRequired(final boolean required) { + this.required = required; + } + + public boolean isBigField() { + return this.type == FieldTypeEnum.TEXT; + } + + public boolean validate(final Object value) { + if (this.required && value == null) { + return false; + } + if (value == null) { + return true; + } + final Class valueClass = value.getClass(); + switch (this.type) { + case BOOLEAN: + return valueClass == Boolean.TYPE || valueClass == Boolean.class; + case BYTE: + return valueClass == Byte.TYPE || valueClass == Byte.class; + case INTEGER: + return valueClass == Integer.TYPE || valueClass == Integer.class; + case LONG: + return valueClass == Long.TYPE || valueClass == Long.class; + case DOUBLE: + return valueClass == Double.TYPE || valueClass == Double.class; + case STRING: + case TEXT: + return valueClass == String.class; + case LIST: + return value instanceof List; + case MAP: + return value instanceof Map; + case DATETIME: + return value instanceof Date; + case BINARY: + return valueClass == byte[].class; + case REGEXP: + return valueClass == Pattern.class; + case OBJECTID: + return valueClass == ObjectId.class; + default: + throw new RuntimeException("un cased type:" + this); + } + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append((Object)this.name).append((Object)this.type).append(this.required).toHashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (null == obj) { + return false; + } + if (!(obj instanceof Field)) { + return false; + } + final Field other = (Field)obj; + return new EqualsBuilder().append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).append(this.required, other.required).isEquals(); + } + + @Override + public String toString() { + return "{field:" + this.name + ", type:" + this.type + ", required:" + this.required + "}"; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java index a75f63b43..ffd4b0bb2 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/FieldTypeEnum.java @@ -1,22 +1,31 @@ package com.fangjia.datastore.config; -public enum FieldTypeEnum { - BOOLEAN, BYTE, INTEGER, LONG, DOUBLE, STRING, TEXT, LIST, MAP, DATETIME, BINARY, REGEXP, OBJECTID; - - private FieldTypeEnum() { - } - - public String getType() { - return this.name().toLowerCase(); - } - - public static FieldTypeEnum parse(String type) { - for (FieldTypeEnum typeEnum : FieldTypeEnum.values()) { - if (typeEnum.name().toLowerCase().equals(type)) { - return typeEnum; - } - } - return null; - } - +public enum FieldTypeEnum +{ + BOOLEAN, + BYTE, + INTEGER, + LONG, + DOUBLE, + STRING, + TEXT, + LIST, + MAP, + DATETIME, + BINARY, + REGEXP, + OBJECTID; + + public String getType() { + return this.name().toLowerCase(); + } + + public static FieldTypeEnum parse(final String type) { + for (final FieldTypeEnum typeEnum : values()) { + if (typeEnum.name().toLowerCase().equals(type)) { + return typeEnum; + } + } + return null; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java index a3d352e24..98aee8f1d 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/MongoConfig.java @@ -2,87 +2,98 @@ package com.fangjia.datastore.config; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.dom4j.Node; - -import com.fangjia.datastore.ConfigurationException; import com.fangjia.datastore.utils.XmlUtils; +import org.dom4j.Node; +import com.mongodb.MongoURI; +import org.apache.commons.lang.StringUtils; +import com.fangjia.datastore.ConfigurationException; import com.mongodb.Mongo; -public final class MongoConfig { - - private String host; - private int port; - private final String id; - private Mongo mongo; - - MongoConfig(String id) throws ConfigurationException { - this.id = id; - } - - private synchronized void init() throws ConfigurationException { - if (null == mongo) { - try { - mongo = new Mongo(host, port); - } catch (Exception e) { - throw new ConfigurationException("error init mongo[" + host + ":" + port, e); - } - } - } - - public Mongo getMongo() throws ConfigurationException { - if (null == mongo) { - init(); - } - return mongo; - } - - synchronized void reload(Node mongoNode) throws ConfigurationException { - String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com"); - int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017); - if (host.equals(this.host) && this.port == port) { - return; - } - this.host = host; - this.port = port; - if (null != mongo) { - try { - init(); - } catch (Exception e) { - throw new ConfigurationException("error init mongo[" + host + ":" + port, e); - } - } - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(this.host).append(this.port).append(this.id).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (null == obj) { - return false; - } - if (!(obj instanceof MongoConfig)) { - return false; - } - MongoConfig other = (MongoConfig) obj; - return new EqualsBuilder().append(this.id, other.id).append(this.host, other.host).append(this.port, other.port) - .isEquals(); - } - - @Override - public String toString() { - StringBuilder ret = new StringBuilder(); - ret.append('{'); - ret.append("id:").append(this.id); - ret.append(", host:").append(this.host); - ret.append(", port:").append(this.port); - ret.append('}'); - return ret.toString(); - } - +public final class MongoConfig +{ + private String host; + private String uri; + private int port; + private final String id; + private Mongo mongo; + + MongoConfig(final String id) throws ConfigurationException { + this.id = id; + } + + private synchronized void init() throws ConfigurationException { + if (null == this.mongo) { + try { + MongoURI mongoURI; + if (StringUtils.isNotBlank(this.uri)) { + mongoURI = new MongoURI(this.uri); + } + else { + mongoURI = new MongoURI("mongodb://" + this.host + ":" + this.port); + } + this.mongo = new Mongo(mongoURI); + } + catch (Exception e) { + throw new ConfigurationException("error init mongo[" + this.host + ":" + this.port, e); + } + } + } + + public Mongo getMongo() throws ConfigurationException { + if (null == this.mongo) { + this.init(); + } + return this.mongo; + } + + synchronized void reload(final Node mongoNode) throws ConfigurationException { + final String host = XmlUtils.getNodeValue(mongoNode, "host", "datastore.mongo.db.fangjia.com"); + final int port = XmlUtils.getNodeValueAsInt(mongoNode, "port", 27017); + final String uri = XmlUtils.getNodeValue(mongoNode, "uri", ""); + if (host.equals(this.host) && this.port == port) { + return; + } + this.host = host; + this.port = port; + this.uri = uri; + if (null != this.mongo) { + try { + this.init(); + } + catch (Exception e) { + throw new ConfigurationException("error init mongo[" + host + ":" + port, e); + } + } + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append((Object)this.host).append(this.port).append((Object)this.id).toHashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (null == obj) { + return false; + } + if (!(obj instanceof MongoConfig)) { + return false; + } + final MongoConfig other = (MongoConfig)obj; + return new EqualsBuilder().append((Object)this.id, (Object)other.id).append((Object)this.host, (Object)other.host).append(this.port, other.port).isEquals(); + } + + @Override + public String toString() { + final StringBuilder ret = new StringBuilder(); + ret.append('{'); + ret.append("id:").append(this.id); + ret.append(", host:").append(this.host); + ret.append(", port:").append(this.port); + ret.append('}'); + return ret.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java index 07bd3e51b..ba98d1688 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Reloadable.java @@ -2,6 +2,7 @@ package com.fangjia.datastore.config; import com.fangjia.datastore.ConfigurationException; -public interface Reloadable { - void reload() throws ConfigurationException; +public interface Reloadable +{ + void reload() throws ConfigurationException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java index 9159563dc..3bd447036 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/ServerConfig.java @@ -1,74 +1,72 @@ package com.fangjia.datastore.config; +import com.fangjia.datastore.utils.XmlUtils; +import com.fangjia.datastore.ConfigurationException; import org.dom4j.Node; -import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.utils.XmlUtils; - -public final class ServerConfig { - private int port; - private boolean disableLogging; - private long performanceLogFilterMaxTime; - private int minaReadBufferSize; - private int cursorExpirationMinutes; - private int defaultCursorLimit; - private int maxConnections; - private int triggerBatchSize; - private int reIndexBatchSize; - - ServerConfig() { - super(); - } - - void reload(Node server) throws ConfigurationException { - if (null == server) { - throw new ConfigurationException("no server is configured!"); - } - this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017); - this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false); - this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000); - this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480); - this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5); - this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20); - this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10); - this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200); - this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200); - } - - public int getPort() { - return port; - } - - public boolean isDisableLogging() { - return disableLogging; - } - - public long getPerformanceLogFilterMaxTime() { - return performanceLogFilterMaxTime; - } - - public int getMinaReadBufferSize() { - return minaReadBufferSize; - } - - public int getCursorExpirationMinutes() { - return cursorExpirationMinutes; - } - - public int getDefaultCursorLimit() { - return defaultCursorLimit; - } - - public int getMaxConnections() { - return maxConnections; - } - - public int getTriggerBatchSize() { - return triggerBatchSize; - } - - public int getReIndexBatchSize() { - return reIndexBatchSize; - } - +public final class ServerConfig +{ + private int port; + private boolean disableLogging; + private long performanceLogFilterMaxTime; + private int minaReadBufferSize; + private int cursorExpirationMinutes; + private int defaultCursorLimit; + private int maxConnections; + private int triggerBatchSize; + private int reIndexBatchSize; + + ServerConfig() { + } + + void reload(final Node server) throws ConfigurationException { + if (null == server) { + throw new ConfigurationException("no server is configured!"); + } + this.port = XmlUtils.getNodeValueAsInt(server, "port", 27017); + this.disableLogging = XmlUtils.getNodeValueAsBoolean(server, "disableLogging", false); + this.performanceLogFilterMaxTime = XmlUtils.getNodeValueAsLong(server, "performanceLogFilterMaxTime", 1000L); + this.minaReadBufferSize = XmlUtils.getNodeValueAsInt(server, "minaReadBufferSize", 20480); + this.cursorExpirationMinutes = XmlUtils.getNodeValueAsInt(server, "cursorExpirationMinutes", 5); + this.defaultCursorLimit = XmlUtils.getNodeValueAsInt(server, "defaultCursorLimit", 20); + this.maxConnections = XmlUtils.getNodeValueAsInt(server, "maxConnections", 10); + this.triggerBatchSize = XmlUtils.getNodeValueAsInt(server, "triggerBatchSize", 200); + this.reIndexBatchSize = XmlUtils.getNodeValueAsInt(server, "reIndexBatchSize", 200); + } + + public int getPort() { + return this.port; + } + + public boolean isDisableLogging() { + return this.disableLogging; + } + + public long getPerformanceLogFilterMaxTime() { + return this.performanceLogFilterMaxTime; + } + + public int getMinaReadBufferSize() { + return this.minaReadBufferSize; + } + + public int getCursorExpirationMinutes() { + return this.cursorExpirationMinutes; + } + + public int getDefaultCursorLimit() { + return this.defaultCursorLimit; + } + + public int getMaxConnections() { + return this.maxConnections; + } + + public int getTriggerBatchSize() { + return this.triggerBatchSize; + } + + public int getReIndexBatchSize() { + return this.reIndexBatchSize; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java index 283d169e0..f08c37efd 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/SolrConfig.java @@ -1,132 +1,132 @@ package com.fangjia.datastore.config; +import com.fangjia.datastore.ConfigurationException; import org.apache.commons.lang.StringUtils; +import com.fangjia.datastore.utils.XmlUtils; import org.dom4j.Node; -import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.utils.XmlUtils; - -public class SolrConfig { - private final String id; - private String host; - private String home; - private final DataStoreConfig dataStoreConfig; - private String shards; - private String slaves; - private SolrConfig[] shardConfigs; - private SolrConfig[] slaveConfigs; - private String filter; - - SolrConfig(DataStoreConfig dataStoreConfig, String id) { - this.dataStoreConfig = dataStoreConfig; - this.id = id; - } - - synchronized void reload(Node solrNode) throws ConfigurationException { - this.host = removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null)); - this.shards = XmlUtils.getNodeValue(solrNode, "shards", null); - this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null); - this.home = XmlUtils.getNodeValue(solrNode, "home", null); - if (StringUtils.isNotEmpty(shards) && StringUtils.isEmpty(home)) { - throw new ConfigurationException("[" + id + "] invalid home attribute:" + home); - } - if (StringUtils.isEmpty(shards) && StringUtils.isEmpty(slaves) && StringUtils.isEmpty(host)) { - throw new ConfigurationException("[" + id + "] invalid host attribute:" + host); - } - this.filter = removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null)); - } - - private SolrConfig[] getConfigs(String ids) { - SolrConfig[] ret; - if (StringUtils.isBlank(ids)) { - ret = new SolrConfig[0]; - } else { - String[] idSplit = ids.split(","); - ret = new SolrConfig[idSplit.length]; - for (int i = 0; i < idSplit.length; i++) { - ret[i] = dataStoreConfig.getSolrConfigById(idSplit[i]); - } - } - return ret; - } - - public SolrConfig[] getShardsConfigs() { - if (null == shardConfigs) { - shardConfigs = this.getConfigs(shards); - } - return shardConfigs; - } - - public SolrConfig[] getSlaveConfigs() { - if (null == slaveConfigs) { - slaveConfigs = this.getConfigs(slaves); - } - return slaveConfigs; - } - - private String removeLastSlash(String s) { - if (StringUtils.isEmpty(s)) { - return s; - } - if (s.charAt(s.length() - 1) == '/') { - return s.substring(0, s.length() - 1); - } - return s; - } - - public String getShardsParam(String coreName) { - SolrConfig[] shardConfigs = getShardsConfigs(); - if (shardConfigs.length == 0) { - return ""; - } - StringBuilder ret = new StringBuilder(); - for (int i = 0; i < shardConfigs.length; i++) { - ret.append(shardConfigs[i].host).append('/').append(coreName); - if (i < shardConfigs.length - 1) { - ret.append(','); - } - } - return ret.toString(); - } - - public String getId() { - return id; - } - - public String getHost() { - return host; - } - - public DataStoreConfig getDataStoreConfig() { - return dataStoreConfig; - } - - public String getShards() { - return shards; - } - - public String getFilter() { - return filter; - } - - public String getHome() { - return home; - } - - @Override - public String toString() { - StringBuilder ret = new StringBuilder(); - ret.append('{'); - ret.append("id: ").append(this.id); - ret.append(", host: ").append(this.host); - ret.append(", shards: ").append(this.shards); - ret.append(", slaves: ").append(this.slaves); - ret.append('}'); - return ret.toString(); - } - - public String getSlaves() { - return slaves; - } - +public class SolrConfig +{ + private final String id; + private String host; + private String home; + private final DataStoreConfig dataStoreConfig; + private String shards; + private String slaves; + private SolrConfig[] shardConfigs; + private SolrConfig[] slaveConfigs; + private String filter; + + SolrConfig(final DataStoreConfig dataStoreConfig, final String id) { + this.dataStoreConfig = dataStoreConfig; + this.id = id; + } + + synchronized void reload(final Node solrNode) throws ConfigurationException { + this.host = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "host", null)); + this.shards = XmlUtils.getNodeValue(solrNode, "shards", null); + this.slaves = XmlUtils.getNodeValue(solrNode, "slaves", null); + this.home = XmlUtils.getNodeValue(solrNode, "home", null); + if (StringUtils.isNotEmpty(this.shards) && StringUtils.isEmpty(this.home)) { + throw new ConfigurationException("[" + this.id + "] invalid home attribute:" + this.home); + } + if (StringUtils.isEmpty(this.shards) && StringUtils.isEmpty(this.slaves) && StringUtils.isEmpty(this.host)) { + throw new ConfigurationException("[" + this.id + "] invalid host attribute:" + this.host); + } + this.filter = this.removeLastSlash(XmlUtils.getNodeValue(solrNode, "filter", null)); + } + + private SolrConfig[] getConfigs(final String ids) { + SolrConfig[] ret; + if (StringUtils.isBlank(ids)) { + ret = new SolrConfig[0]; + } + else { + final String[] idSplit = ids.split(","); + ret = new SolrConfig[idSplit.length]; + for (int i = 0; i < idSplit.length; ++i) { + ret[i] = this.dataStoreConfig.getSolrConfigById(idSplit[i]); + } + } + return ret; + } + + public SolrConfig[] getShardsConfigs() { + if (null == this.shardConfigs) { + this.shardConfigs = this.getConfigs(this.shards); + } + return this.shardConfigs; + } + + public SolrConfig[] getSlaveConfigs() { + if (null == this.slaveConfigs) { + this.slaveConfigs = this.getConfigs(this.slaves); + } + return this.slaveConfigs; + } + + private String removeLastSlash(final String s) { + if (StringUtils.isEmpty(s)) { + return s; + } + if (s.charAt(s.length() - 1) == '/') { + return s.substring(0, s.length() - 1); + } + return s; + } + + public String getShardsParam(final String coreName) { + final SolrConfig[] shardConfigs = this.getShardsConfigs(); + if (shardConfigs.length == 0) { + return ""; + } + final StringBuilder ret = new StringBuilder(); + for (int i = 0; i < shardConfigs.length; ++i) { + ret.append(shardConfigs[i].host).append('/').append(coreName); + if (i < shardConfigs.length - 1) { + ret.append(','); + } + } + return ret.toString(); + } + + public String getId() { + return this.id; + } + + public String getHost() { + return this.host; + } + + public DataStoreConfig getDataStoreConfig() { + return this.dataStoreConfig; + } + + public String getShards() { + return this.shards; + } + + public String getFilter() { + return this.filter; + } + + public String getHome() { + return this.home; + } + + @Override + public String toString() { + final StringBuilder ret = new StringBuilder(); + ret.append('{'); + ret.append("id: ").append(this.id); + ret.append(", host: ").append(this.host); + ret.append(", shards: ").append(this.shards); + ret.append(", slaves: ").append(this.slaves); + ret.append('}'); + return ret.toString(); + } + + public String getSlaves() { + return this.slaves; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java index fbeed7114..2d289057a 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/StoreConfig.java @@ -1,229 +1,208 @@ package com.fangjia.datastore.config; -import java.util.Collection; import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; +import java.util.Collection; import org.bson.types.ObjectId; +import java.util.Iterator; +import java.util.List; +import com.fangjia.datastore.utils.XmlUtils; +import com.fangjia.datastore.ConfigurationException; +import org.apache.commons.lang.StringUtils; import org.dom4j.Element; import org.dom4j.Node; +import java.util.HashMap; +import java.util.Map; -import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.utils.XmlUtils; - -public final class StoreConfig { - - private final String name; - private int solrMaxTotalConnections; - private int solrMaxConnectionsPerHost; - private SolrConfig[] masterSolrConfigs; - private SolrConfig masterSolrConfig; - private SolrConfig slaveSolrConfig; - private MongoConfig mongoConfig; - private boolean textFieldToGridFS; - private final Trigger trigger; - private final Map fields; - private final DataStoreConfig dataStoreConfig; - - StoreConfig(DataStoreConfig dataStoreConfig, String name) { - this.dataStoreConfig = dataStoreConfig; - fields = new HashMap(); - trigger = new Trigger(); - this.name = name; - } - - synchronized void reload(Node storeNode) throws ConfigurationException { - // init solr - String solrMastersAttr = ((Element) storeNode).attributeValue("solrMasters"); - if (StringUtils.isEmpty(solrMastersAttr)) { - throw new ConfigurationException("[" + name + "] invalid solrMasters attribute:" + solrMastersAttr); - } - String[] solrMasterIds = solrMastersAttr.split(","); - if (solrMasterIds.length > 2) { - throw new ConfigurationException("[" + name + "] too many solr server configured:" + solrMastersAttr); - } - masterSolrConfigs = new SolrConfig[solrMasterIds.length]; - for (int i = 0; i < solrMasterIds.length; i++) { - masterSolrConfigs[i] = dataStoreConfig.getSolrConfigById(solrMasterIds[i]); - if (null == masterSolrConfigs[i]) { - throw new ConfigurationException("[" + name + "] unknown solr id: " + solrMasterIds[i]); - } - } - - String solrSlaveAttr = System.getProperty("datastore." + name + ".solrSlave", ((Element) storeNode) - .attributeValue("solrSlave")); - if (StringUtils.isEmpty(solrSlaveAttr)) { - throw new ConfigurationException("[" + name + "] invalid solrSlave attribute:" + solrSlaveAttr); - } - slaveSolrConfig = dataStoreConfig.getSolrConfigById(solrSlaveAttr); - if (null == slaveSolrConfig) { - throw new ConfigurationException("[" + name + "] unknown solr id: " + solrSlaveAttr); - } - - String solrMasterAttr = System.getProperty("datastore." + name + ".solrMaster", ((Element) storeNode) - .attributeValue("solrMaster")); - if (StringUtils.isEmpty(solrMasterAttr)) { - throw new ConfigurationException("[" + name + "] invalid solrMaster attribute:" + solrMasterAttr); - } - masterSolrConfig = dataStoreConfig.getSolrConfigById(solrMasterAttr); - if (null == masterSolrConfig) { - throw new ConfigurationException("[" + name + "] unknown solr id: " + solrMasterAttr); - } - - if (slaveSolrConfig.getShardsConfigs().length != masterSolrConfig.getShardsConfigs().length) { - throw new ConfigurationException("[" + name + "] master and slave shard count must equals !"); - } - - // solr connections defaults - this.solrMaxConnectionsPerHost = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxConnectionsPerHost", 300); - if (slaveSolrConfig.getShardsConfigs().length > 0) { - this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", slaveSolrConfig - .getShardsConfigs().length - * solrMaxConnectionsPerHost); - } else { - this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", - solrMaxConnectionsPerHost); - } - - // init mongo - String mongoId = ((Element) storeNode).attributeValue("mongo"); - MongoConfig mongoConfig = dataStoreConfig.getMongoConfigById(mongoId); - if (null == mongoConfig) { - throw new ConfigurationException("[" + name + "] unknown mongo id: " + mongoId); - } - this.mongoConfig = mongoConfig; - - // init other config - this.textFieldToGridFS = XmlUtils.getNodeValueAsBoolean(storeNode, "textFieldToGridFS", false); - - // init fields - Map newFields = new HashMap(); - List fieldNodes = storeNode.selectNodes("schema//field"); - for (Node fieldNode : fieldNodes) { - Element element = (Element) fieldNode; - String fieldName = element.attributeValue("name"); - Field field = getOrCreateField(fieldName); - field.setName(fieldName); - field.setRequired(Boolean.parseBoolean(element.attributeValue("required"))); - field.setType(FieldTypeEnum.parse(element.attributeValue("type"))); - newFields.put(fieldName, field); - } - this.fields.clear(); - this.fields.putAll(newFields); - - // init triggers - Element triggerNode = (Element) storeNode.selectSingleNode("trigger"); - if(triggerNode != null) { - trigger.setInterface(triggerNode.attributeValue("interface")); - } - - } - - public Trigger getTrigger() { - return this.trigger; - } - - public Field getField(String fieldName) { - return this.fields.get(fieldName); - } - - public boolean containsField(String fieldName) { - return this.fields.containsKey(fieldName); - } - - public boolean isFieldValueLegal(String fieldName, Object value) { - if ("_id".equals(fieldName)) { - return value instanceof ObjectId; - } - if (fields.containsKey(fieldName)) { - Field field = fields.get(fieldName); - return field.validate(value); - } - return false; - } - - public boolean isBigField(String fieldName) { - return fields.get(fieldName).isBigField(); - } - - public Collection getFields() { - return Collections.unmodifiableCollection(this.fields.values()); - } - - synchronized Field getOrCreateField(String fieldName) { - if (StringUtils.isEmpty(fieldName)) { - throw new IllegalArgumentException("fieldName can not be null or empty!"); - } - Field ret = this.fields.get(fieldName); - if (null == ret) { - ret = new Field(); - ret.setName(fieldName); - this.fields.put(fieldName, ret); - } - return ret; - } - - public String getName() { - return name; - } - - public MongoConfig getMongoConfig() { - return mongoConfig; - } - - public DataStoreConfig getDataStoreConfig() { - return dataStoreConfig; - } - - public boolean isTextFieldToGridFS() { - return textFieldToGridFS; - } - - public SolrConfig getSlaveSolrConfig() { - return slaveSolrConfig; - } - - public SolrConfig[] getMasterSolrConfigs() { - return masterSolrConfigs; - } - - public SolrConfig getMasterSolrConfig() { - return masterSolrConfig; - } - - public int getSolrMaxTotalConnections() { - return solrMaxTotalConnections; - } - - public int getSolrMaxConnectionsPerHost() { - return solrMaxConnectionsPerHost; - } - - @Override - public String toString() { - StringBuilder ret = new StringBuilder(); - ret.append("\n{"); - ret.append("\n\tname: ").append(name); - ret.append(",\n\tmasterSolrConfig: ").append(masterSolrConfig); - ret.append(",\n\tslaveSolrConfig: ").append(this.slaveSolrConfig); - ret.append(",\n\tmongoConfig: ").append(mongoConfig); - ret.append(",\n\tsolrMaxTotalConnections: ").append(solrMaxTotalConnections); - ret.append(",\n\tsolrMaxConnectionsPerHost: ").append(this.solrMaxConnectionsPerHost); - ret.append(",\n\ttextFieldToGridFS: ").append(this.textFieldToGridFS); - ret.append(",\n\ttrigger: ").append(trigger); - ret.append(",\n\tmasterSolrConfigs: [\n"); - for (int i = 0; i < this.masterSolrConfigs.length; i++) { - ret.append('\t').append('\t').append(this.masterSolrConfigs[i]); - if (i < this.masterSolrConfigs.length - 1) { - ret.append(','); - } - ret.append('\n'); - } - ret.append("\t]\n}"); - return ret.toString(); - } - +public final class StoreConfig +{ + private final String name; + private int solrMaxTotalConnections; + private int solrMaxConnectionsPerHost; + private SolrConfig[] masterSolrConfigs; + private SolrConfig masterSolrConfig; + private SolrConfig slaveSolrConfig; + private MongoConfig mongoConfig; + private boolean textFieldToGridFS; + private final Trigger trigger; + private final Map fields; + private final DataStoreConfig dataStoreConfig; + + StoreConfig(final DataStoreConfig dataStoreConfig, final String name) { + this.dataStoreConfig = dataStoreConfig; + this.fields = new HashMap(); + this.trigger = new Trigger(); + this.name = name; + } + + synchronized void reload(final Node storeNode) throws ConfigurationException { + final String solrMastersAttr = ((Element)storeNode).attributeValue("solrMasters"); + if (StringUtils.isEmpty(solrMastersAttr)) { + throw new ConfigurationException("[" + this.name + "] invalid solrMasters attribute:" + solrMastersAttr); + } + final String[] solrMasterIds = solrMastersAttr.split(","); + if (solrMasterIds.length > 2) { + throw new ConfigurationException("[" + this.name + "] too many solr server configured:" + solrMastersAttr); + } + this.masterSolrConfigs = new SolrConfig[solrMasterIds.length]; + for (int i = 0; i < solrMasterIds.length; ++i) { + this.masterSolrConfigs[i] = this.dataStoreConfig.getSolrConfigById(solrMasterIds[i]); + if (null == this.masterSolrConfigs[i]) { + throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrMasterIds[i]); + } + } + final String solrSlaveAttr = System.getProperty("datastore." + this.name + ".solrSlave", ((Element)storeNode).attributeValue("solrSlave")); + if (StringUtils.isEmpty(solrSlaveAttr)) { + throw new ConfigurationException("[" + this.name + "] invalid solrSlave attribute:" + solrSlaveAttr); + } + this.slaveSolrConfig = this.dataStoreConfig.getSolrConfigById(solrSlaveAttr); + if (null == this.slaveSolrConfig) { + throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrSlaveAttr); + } + final String solrMasterAttr = System.getProperty("datastore." + this.name + ".solrMaster", ((Element)storeNode).attributeValue("solrMaster")); + if (StringUtils.isEmpty(solrMasterAttr)) { + throw new ConfigurationException("[" + this.name + "] invalid solrMaster attribute:" + solrMasterAttr); + } + this.masterSolrConfig = this.dataStoreConfig.getSolrConfigById(solrMasterAttr); + if (null == this.masterSolrConfig) { + throw new ConfigurationException("[" + this.name + "] unknown solr id: " + solrMasterAttr); + } + if (this.slaveSolrConfig.getShardsConfigs().length != this.masterSolrConfig.getShardsConfigs().length) { + throw new ConfigurationException("[" + this.name + "] master and slave shard count must equals !"); + } + this.solrMaxConnectionsPerHost = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxConnectionsPerHost", 300); + if (this.slaveSolrConfig.getShardsConfigs().length > 0) { + this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", this.slaveSolrConfig.getShardsConfigs().length * this.solrMaxConnectionsPerHost); + } + else { + this.solrMaxTotalConnections = XmlUtils.getNodeValueAsInt(storeNode, "solrMaxTotalConnections", this.solrMaxConnectionsPerHost); + } + final String mongoId = ((Element)storeNode).attributeValue("mongo"); + final MongoConfig mongoConfig = this.dataStoreConfig.getMongoConfigById(mongoId); + if (null == mongoConfig) { + throw new ConfigurationException("[" + this.name + "] unknown mongo id: " + mongoId); + } + this.mongoConfig = mongoConfig; + this.textFieldToGridFS = XmlUtils.getNodeValueAsBoolean(storeNode, "textFieldToGridFS", false); + final Map newFields = new HashMap(); + final List fieldNodes = (List)storeNode.selectNodes("schema//field"); + for (final Node fieldNode : fieldNodes) { + final Element element = (Element)fieldNode; + final String fieldName = element.attributeValue("name"); + final Field field = this.getOrCreateField(fieldName); + field.setName(fieldName); + field.setRequired(Boolean.parseBoolean(element.attributeValue("required"))); + field.setType(FieldTypeEnum.parse(element.attributeValue("type"))); + newFields.put(fieldName, field); + } + this.fields.clear(); + this.fields.putAll(newFields); + final Element triggerNode = (Element)storeNode.selectSingleNode("trigger"); + if (triggerNode != null) { + this.trigger.setInterface(triggerNode.attributeValue("interface")); + } + } + + public Trigger getTrigger() { + return this.trigger; + } + + public Field getField(final String fieldName) { + return this.fields.get(fieldName); + } + + public boolean containsField(final String fieldName) { + return this.fields.containsKey(fieldName); + } + + public boolean isFieldValueLegal(final String fieldName, final Object value) { + if ("_id".equals(fieldName)) { + return value instanceof ObjectId; + } + if (this.fields.containsKey(fieldName)) { + final Field field = this.fields.get(fieldName); + return field.validate(value); + } + return false; + } + + public boolean isBigField(final String fieldName) { + return this.fields.get(fieldName).isBigField(); + } + + public Collection getFields() { + return Collections.unmodifiableCollection((Collection)this.fields.values()); + } + + synchronized Field getOrCreateField(final String fieldName) { + if (StringUtils.isEmpty(fieldName)) { + throw new IllegalArgumentException("fieldName can not be null or empty!"); + } + Field ret = this.fields.get(fieldName); + if (null == ret) { + ret = new Field(); + ret.setName(fieldName); + this.fields.put(fieldName, ret); + } + return ret; + } + + public String getName() { + return this.name; + } + + public MongoConfig getMongoConfig() { + return this.mongoConfig; + } + + public DataStoreConfig getDataStoreConfig() { + return this.dataStoreConfig; + } + + public boolean isTextFieldToGridFS() { + return this.textFieldToGridFS; + } + + public SolrConfig getSlaveSolrConfig() { + return this.slaveSolrConfig; + } + + public SolrConfig[] getMasterSolrConfigs() { + return this.masterSolrConfigs; + } + + public SolrConfig getMasterSolrConfig() { + return this.masterSolrConfig; + } + + public int getSolrMaxTotalConnections() { + return this.solrMaxTotalConnections; + } + + public int getSolrMaxConnectionsPerHost() { + return this.solrMaxConnectionsPerHost; + } + + @Override + public String toString() { + final StringBuilder ret = new StringBuilder(); + ret.append("\n{"); + ret.append("\n\tname: ").append(this.name); + ret.append(",\n\tmasterSolrConfig: ").append(this.masterSolrConfig); + ret.append(",\n\tslaveSolrConfig: ").append(this.slaveSolrConfig); + ret.append(",\n\tmongoConfig: ").append(this.mongoConfig); + ret.append(",\n\tsolrMaxTotalConnections: ").append(this.solrMaxTotalConnections); + ret.append(",\n\tsolrMaxConnectionsPerHost: ").append(this.solrMaxConnectionsPerHost); + ret.append(",\n\ttextFieldToGridFS: ").append(this.textFieldToGridFS); + ret.append(",\n\ttrigger: ").append(this.trigger); + ret.append(",\n\tmasterSolrConfigs: [\n"); + for (int i = 0; i < this.masterSolrConfigs.length; ++i) { + ret.append('\t').append('\t').append(this.masterSolrConfigs[i]); + if (i < this.masterSolrConfigs.length - 1) { + ret.append(','); + } + ret.append('\n'); + } + ret.append("\t]\n}"); + return ret.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Trigger.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Trigger.java index 8608fffe0..08ec5471d 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Trigger.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/config/Trigger.java @@ -1,49 +1,49 @@ package com.fangjia.datastore.config; -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; +import org.apache.commons.lang.StringUtils; -public final class Trigger { - private String interface_; - - Trigger() { - } - - public String getInterface() { - return interface_; - } - - void setInterface(String interface_) { - if (StringUtils.isEmpty(interface_) || !interface_.startsWith("http://")) { - throw new IllegalArgumentException("invalid trigger interface: " + interface_); - } - this.interface_ = interface_; - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(this.interface_).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (null == obj) { - return false; - } - if (this == obj) { - return true; - } - if (!(obj instanceof Trigger)) { - return false; - } - Trigger other = (Trigger) obj; - return new EqualsBuilder().append(this.interface_, other.interface_).isEquals(); - } - - @Override - public String toString() { - return new StringBuilder().append("{interface=").append(interface_).append("}").toString(); - } - +public final class Trigger +{ + private String interface_; + + Trigger() { + } + + public String getInterface() { + return this.interface_; + } + + void setInterface(final String interface_) { + if (StringUtils.isEmpty(interface_) || !interface_.startsWith("http://")) { + throw new IllegalArgumentException("invalid trigger interface: " + interface_); + } + this.interface_ = interface_; + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append((Object)this.interface_).toHashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (null == obj) { + return false; + } + if (this == obj) { + return true; + } + if (!(obj instanceof Trigger)) { + return false; + } + final Trigger other = (Trigger)obj; + return new EqualsBuilder().append((Object)this.interface_, (Object)other.interface_).isEquals(); + } + + @Override + public String toString() { + return "{interface=" + this.interface_ + "}"; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/DeleteMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/DeleteMessageHandler.java index 5ed531fa7..2a853a2bc 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/DeleteMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/DeleteMessageHandler.java @@ -1,19 +1,20 @@ package com.fangjia.datastore.handler; +import com.fangjia.datastore.mongo.protocol.MongoMessage; import com.fangjia.datastore.DataStore; import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.mongo.protocol.DeleteMessage; -public class DeleteMessageHandler extends MessageHandler { - - public DeleteMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(DeleteMessage message,String ip) throws Exception { - assertNotReadOnly(); - DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); - dataStore.remove(message.getSelector()); - } - +public class DeleteMessageHandler extends MessageHandler +{ + public DeleteMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final DeleteMessage message) throws Exception { + this.assertNotReadOnly(); + final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); + dataStore.remove(message.getSelector()); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/GetMoreMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/GetMoreMessageHandler.java index 4baba97ee..a9c78e1ba 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/GetMoreMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/GetMoreMessageHandler.java @@ -1,54 +1,59 @@ package com.fangjia.datastore.handler; -import java.util.List; - -import org.bson.BSONObject; - +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import com.mongodb.DBObject; +import com.fangjia.datastore.QueryResponse; +import com.fangjia.datastore.mongo.protocol.QueryMessage; import com.fangjia.datastore.Cursor; import com.fangjia.datastore.DataStore; -import com.fangjia.datastore.DataStoreContext; -import com.fangjia.datastore.QueryResponse; -import com.fangjia.datastore.mongo.protocol.GetMoreMessage; -import com.fangjia.datastore.mongo.protocol.QueryMessage; import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import org.bson.BSONObject; +import java.util.List; +import com.fangjia.datastore.DataStoreContext; +import com.fangjia.datastore.mongo.protocol.GetMoreMessage; -public class GetMoreMessageHandler extends MessageHandler { - - public GetMoreMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(GetMoreMessage message) throws Exception { - String fullCollectionName = message.getFullCollectionName(); - if(fullCollectionName.contains("$cmd")){ - ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), (List) null); - reply.setStartingFrom(0); - this.writeMessage(reply); - this.dataStoreContext.removeCursor(message.getCursorID()); - return; - } - DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); - Cursor cursor = this.dataStoreContext.getCursor(message.getCursorID()); - if (null == cursor) { - ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), (List) null); - reply.setStartingFrom(0); - this.writeMessage(reply); - } else { - QueryMessage queryMessage = cursor.getQueryMessage(); - QueryResponse response = dataStore.find(queryMessage.getRealQuery(), queryMessage.getReturnFieldSelector(), - queryMessage.getOrderByQuery(), queryMessage.getFacetQuery(), cursor.nextSkip(), - message.getNumberToReturn() - 1, queryMessage.isBulkQuery(), queryMessage.isFromMaster()); - ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), response.getDocList()); - reply.setStartingFrom(cursor.getSkip()); - if (response.hasMore()) { - // cursor.setTotalCount(response.getNumFound()); - // cursor.setLimit((int) response.getNumReturned()); - // this.dataStoreContext.addCursor(cursor); - // reply.setCursorID(cursor.getId()); - } else { - this.dataStoreContext.removeCursor(cursor.getId()); - } - this.writeMessage(reply); - } - } +public class GetMoreMessageHandler extends MessageHandler +{ + public GetMoreMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final GetMoreMessage message) throws Exception { + final String fullCollectionName = message.getFullCollectionName(); + if (fullCollectionName.contains("$cmd")) { + final ReplyMessage reply = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List)null); + reply.setStartingFrom(0); + this.writeMessage(reply); + this.dataStoreContext.removeCursor(message.getCursorID()); + return; + } + final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); + final Cursor cursor = this.dataStoreContext.getCursor(message.getCursorID()); + if (null == cursor) { + final ReplyMessage reply2 = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List)null); + reply2.setStartingFrom(0); + this.writeMessage(reply2); + } + else { + final QueryMessage queryMessage = cursor.getQueryMessage(); + final QueryResponse response = dataStore.find(queryMessage.getRealQuery(), queryMessage.getReturnFieldSelector(), queryMessage.getOrderByQuery(), queryMessage.getFacetQuery(), cursor.nextSkip(), this.useCursor() ? message.getNumberToReturn() : (message.getNumberToReturn() - 1), queryMessage.isBulkQuery(), queryMessage.isFromMaster()); + final List doc = response.getDocList(); + if (this.useCursor()) { + doc.remove(0); + } + final ReplyMessage reply3 = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List)doc); + reply3.setStartingFrom(cursor.getSkip()); + if (response.hasMore() && this.useCursor()) { + cursor.setTotalCount(response.getNumFound()); + cursor.setLimit(response.getNumReturned()); + this.dataStoreContext.addCursor(cursor); + reply3.setCursorID(cursor.getId()); + } + else { + this.dataStoreContext.removeCursor(cursor.getId()); + } + this.writeMessage(reply3); + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/InsertMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/InsertMessageHandler.java index 27abe5e4f..54d6c41d9 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/InsertMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/InsertMessageHandler.java @@ -1,115 +1,117 @@ package com.fangjia.datastore.handler; -import java.util.ArrayList; -import java.util.List; - +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import java.util.Iterator; import org.apache.commons.lang.StringUtils; -import org.bson.BSONObject; - +import java.util.ArrayList; +import com.fangjia.datastore.solr.Index; +import java.util.Collection; import com.fangjia.datastore.DataStore; +import org.bson.BSONObject; +import java.util.List; import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.mongo.protocol.InsertMessage; -import com.fangjia.datastore.solr.Index; -import com.fangjia.datastore.solr.Index.IndexTypeEnum; -public class InsertMessageHandler extends MessageHandler { - - public InsertMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(InsertMessage message,String ip) throws Exception { - if (message.getFullCollectionName().endsWith("system.indexes")) { - ensureIndex(message); - } else { - insert(message); - } - } - - /** - * db.foo.insert() - * - * @param message - * @throws Exception - */ - private void insert(InsertMessage message) throws Exception { - assertNotReadOnly(); - DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); - dataStore.save(message.getDocuments()); - } - - /** - * db.foo.ensureIndex() - * - * @param message - * @throws Exception - */ - private void ensureIndex(InsertMessage message) throws Exception { - assertNotReadOnly(); - DataStore dataStore = this.dataStoreContext.getDataStore(String.valueOf(message.getDocuments().get(0).get("ns"))); - List indexList = addToIndexList(message.getDocuments()); - dataStore.ensureIndex(indexList); - } - - private List addToIndexList(List docList) { - if (null == docList || docList.isEmpty()) { - return null; - } - List indexList = new ArrayList(); - for (BSONObject bson : docList) { - BSONObject keys = (BSONObject) bson.get("key"); - if (null == keys) { - continue; - } - for (String key : keys.keySet()) { - Object value = keys.get(key); - if (value instanceof String) { - Index index = new Index(key, IndexTypeEnum.parse((String) value)); - indexList.add(index); - } else if (value instanceof BSONObject) { - // TODO: - Index index = new Index(); - index.setName(key); - for (String k : ((BSONObject) value).keySet()) { - Object v = ((BSONObject) value).get(k); - if (StringUtils.equalsIgnoreCase(k, "type")) { - if (v instanceof String) { - index.setType(IndexTypeEnum.parse((String) v)); - } - } else if (StringUtils.equalsIgnoreCase(k, "stored")) { - if (v instanceof Boolean) { - index.setStored((Boolean) v); - } - } else if (StringUtils.equalsIgnoreCase(k, "indexed")) { - if (v instanceof Boolean) { - index.setIndexed((Boolean) v); - } - - } else if (StringUtils.equalsIgnoreCase(k, "multiValued")) { - if (v instanceof Boolean) { - index.setMultiValued((Boolean) v); - } - } else if (StringUtils.equalsIgnoreCase(k, "dynamic")) { - if (v instanceof Boolean) { - index.setDynamic((Boolean) v); - } - } else if (StringUtils.equalsIgnoreCase(k, "comments")) { - if (v instanceof String) { - index.setComments((String) v); - } - } - } - if (index.getType() == null) { - throw new IllegalArgumentException("illegal schema field: type"); - } - indexList.add(index); - } else { - if (logger.isErrorEnabled()) { - logger.error("invalid key value of ensureIndex:{}", value); - } - } - } - } - return indexList; - } +public class InsertMessageHandler extends MessageHandler +{ + public InsertMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final InsertMessage message) throws Exception { + if (message.getFullCollectionName().endsWith("system.indexes")) { + this.ensureIndex(message); + } + else { + this.insert(message); + } + } + + private void insert(final InsertMessage message) throws Exception { + this.assertNotReadOnly(); + final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); + dataStore.save(message.getDocuments()); + } + + private void ensureIndex(final InsertMessage message) throws Exception { + this.assertNotReadOnly(); + final DataStore dataStore = this.dataStoreContext.getDataStore(String.valueOf(message.getDocuments().get(0).get("ns"))); + final List indexList = this.addToIndexList(message.getDocuments()); + dataStore.ensureIndex(indexList); + } + + private List addToIndexList(final List docList) { + if (null == docList || docList.isEmpty()) { + return null; + } + final List indexList = new ArrayList(); + for (final BSONObject bson : docList) { + final BSONObject keys = (BSONObject)bson.get("key"); + if (null == keys) { + continue; + } + for (final String key : keys.keySet()) { + final Object value = keys.get(key); + if (value instanceof String) { + final Index index = new Index(key, Index.IndexTypeEnum.parse((String)value)); + indexList.add(index); + } + else if (value instanceof BSONObject) { + final Index index = new Index(); + index.setName(key); + for (final String k : ((BSONObject)value).keySet()) { + final Object v = ((BSONObject)value).get(k); + if (StringUtils.equalsIgnoreCase(k, "type")) { + if (!(v instanceof String)) { + continue; + } + index.setType(Index.IndexTypeEnum.parse((String)v)); + } + else if (StringUtils.equalsIgnoreCase(k, "stored")) { + if (!(v instanceof Boolean)) { + continue; + } + index.setStored((Boolean) v); + } + else if (StringUtils.equalsIgnoreCase(k, "indexed")) { + if (!(v instanceof Boolean)) { + continue; + } + index.setIndexed((Boolean)v); + } + else if (StringUtils.equalsIgnoreCase(k, "multiValued")) { + if (!(v instanceof Boolean)) { + continue; + } + index.setMultiValued((Boolean)v); + } + else if (StringUtils.equalsIgnoreCase(k, "dynamic")) { + if (!(v instanceof Boolean)) { + continue; + } + index.setDynamic((Boolean)v); + } + else { + if (!StringUtils.equalsIgnoreCase(k, "comments") || !(v instanceof String)) { + continue; + } + index.setComments((String)v); + } + } + if (index.getType() == null) { + throw new IllegalArgumentException("illegal schema field: type"); + } + indexList.add(index); + } + else { + if (!this.logger.isErrorEnabled()) { + continue; + } + this.logger.error("invalid key value of ensureIndex:{}", value); + } + } + } + return indexList; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/KillCursorsMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/KillCursorsMessageHandler.java index a5104a111..8591eebc3 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/KillCursorsMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/KillCursorsMessageHandler.java @@ -1,23 +1,24 @@ package com.fangjia.datastore.handler; +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import java.util.Iterator; import java.util.List; - import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; -public class KillCursorsMessageHandler extends MessageHandler { - - public KillCursorsMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(KillCursorsMessage message,String ip) { - List cursorIds = message.getCursorIDs(); - if (cursorIds != null) { - for (Long cursorId : cursorIds) { - this.dataStoreContext.removeCursor(cursorId); - } - } - } - +public class KillCursorsMessageHandler extends MessageHandler +{ + public KillCursorsMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final KillCursorsMessage message) { + final List cursorIds = message.getCursorIDs(); + if (cursorIds != null) { + for (final Long cursorId : cursorIds) { + this.dataStoreContext.removeCursor(cursorId); + } + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/MessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/MessageHandler.java index a2d5cc6e0..9f4a4e995 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/MessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/MessageHandler.java @@ -1,184 +1,180 @@ package com.fangjia.datastore.handler; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.mina.core.session.IoSession; -import org.bson.BSONObject; -import org.bson.BasicBSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.DataStoreException; -import com.fangjia.datastore.mongo.protocol.MongoMessage; +import java.util.Arrays; +import org.bson.BasicBSONObject; +import java.util.ArrayList; +import org.bson.BSONObject; +import java.util.List; import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import java.net.InetSocketAddress; +import org.apache.mina.core.session.IoSession; +import org.slf4j.LoggerFactory; +import org.slf4j.Logger; +import com.fangjia.datastore.DataStoreContext; +import java.util.Map; +import com.fangjia.datastore.mongo.protocol.MongoMessage; -public abstract class MessageHandler { - private final static String LAST_ERROR_KEY = "com.goojia.datastore.LAST_ERROR_KEY"; - private final static String PREV_ERROR_KEY = "com.goojia.datastore.PREV_ERROR_KEY"; - - private final static String SESSION_KEY = "com.goojia.datastore.SESSION_KEY"; - private final static String ERROR_OCCURRED_KEY = "com.goojia.datastore.ERROR_OCCURRED_KEY"; - private final static ThreadLocal> THREAD_LOCAL = new ThreadLocal>() { - @Override - protected Map initialValue() { - return new HashMap(); - } - }; - - protected final DataStoreContext dataStoreContext; - - protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - - public MessageHandler(DataStoreContext dataStoreContext) { - this.dataStoreContext = dataStoreContext; - } - - /** - * 处理特定的消息 - * - * @param message - */ - public abstract void messageReceived(T message,String ip) throws Exception; - - @SuppressWarnings("unchecked") - public void messageReceived(IoSession session, MongoMessage message) throws Exception { - System.out.println(session.getRemoteAddress()); - try { - setSession(session); - messageReceived((T) message,session.getRemoteAddress().toString()); - if (!isErrorOccured()) { - this.clearLastError(); - } - } catch (Throwable e) { - String hostAddress = ""; - if(session.getRemoteAddress() instanceof InetSocketAddress) { - hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress(); - } - this.setLastError(buildExceptionMessage(e)); - logger.error("error processing host:" + hostAddress + ",message: " + message, e); - } finally { - clearErrorOccured(); - } - } - - private void clearErrorOccured() { - THREAD_LOCAL.get().remove(ERROR_OCCURRED_KEY); - } - - private void setErrorOccured() { - THREAD_LOCAL.get().put(ERROR_OCCURRED_KEY, true); - } - - private boolean isErrorOccured() { - Boolean b = (Boolean) THREAD_LOCAL.get().get(ERROR_OCCURRED_KEY); - return null == b ? false : b; - } - - protected IoSession getSession() { - return (IoSession) THREAD_LOCAL.get().get(SESSION_KEY); - } - - private void setSession(IoSession session) { - THREAD_LOCAL.get().put(SESSION_KEY, session); - } - - protected void setLastError(Object error) { - if (null != error) { - IoSession session = getSession(); - Object lastError = session.getAttribute(LAST_ERROR_KEY); - if (null != lastError) { - session.setAttribute(PREV_ERROR_KEY, lastError); - } - setErrorOccured(); - session.setAttribute(LAST_ERROR_KEY, String.valueOf(error)); - } - } - - private void clearLastError() { - getSession().removeAttribute(LAST_ERROR_KEY); - } - - protected Object getLastError() { - return getSession().getAttribute(LAST_ERROR_KEY); - } - - protected Object getPrevError() { - return getSession().getAttribute(PREV_ERROR_KEY); - } - - protected void writeMessage(ReplyMessage message) { - getSession().write(message); - } - - protected void writeAsReplyMessage(int responseTo, List docs) { - writeMessage(prepareReplyMessage(responseTo, docs)); - } - - @SuppressWarnings("unchecked") - protected ReplyMessage prepareReplyMessage(int responseTo, List docs) { - ReplyMessage reply = new ReplyMessage(); - reply.setNumberReturned(null == docs ? 0 : docs.size()); - reply.setResponseFlag(8); - reply.getMessageHeader().setResponseTo(responseTo); - if (null != docs && docs.size() > 0) { - reply.setDocuments((List) docs); - } - return reply; - } - - protected void writeAsReplyMessage(int responseTo, BSONObject... docs) { - writeMessage(prepareReplyMessage(responseTo, docs)); - } - - protected ReplyMessage prepareReplyMessage(int responseTo, BSONObject... docs) { - if (null != docs && docs.length > 0) { - List docList = new ArrayList(docs.length); - for (BSONObject doc : docs) { - docList.add(doc); - } - return prepareReplyMessage(responseTo, docList); - } else { - return prepareReplyMessage(responseTo, (List) null); - } - } - - protected void writeAsErrorMessage(int responseTo, Object errorMessage) { - writeMessage(prepareErrorMessage(responseTo, errorMessage)); - } - - protected ReplyMessage prepareErrorMessage(int responseTo, Object errorMessage) { - ReplyMessage reply = new ReplyMessage(); - reply.setNumberReturned(1); - reply.setResponseFlag(8); - reply.getMessageHeader().setResponseTo(responseTo); - BSONObject msg = new BasicBSONObject("$err", errorMessage == null ? null : errorMessage.toString()); - msg.put("err", errorMessage == null ? null : errorMessage.toString()); - msg.put("errmsg", errorMessage == null ? null : errorMessage.toString()); - msg.put("ok", 0.0); - reply.setDocuments(Arrays.asList(msg)); - return reply; - } - - protected String buildExceptionMessage(Throwable e) { - StringBuilder ret = new StringBuilder(); - ret.append(e.getClass().getName()).append(": ").append(e.getMessage()); - Throwable cause = e; - while ((cause = cause.getCause()) != null) { - ret.append(", caused by ").append(cause.getClass().getName()).append(": ").append(cause.getMessage()); - } - return ret.toString(); - } - - protected void assertNotReadOnly() throws DataStoreException { - if (this.dataStoreContext.getDataStoreConfig().isReadOnly()) { - throw new DataStoreException("read only server !"); - } - } +public abstract class MessageHandler +{ + private static final String LAST_ERROR_KEY = "com.goojia.datastore.LAST_ERROR_KEY"; + private static final String PREV_ERROR_KEY = "com.goojia.datastore.PREV_ERROR_KEY"; + private static final String SESSION_KEY = "com.goojia.datastore.SESSION_KEY"; + private static final String ERROR_OCCURRED_KEY = "com.goojia.datastore.ERROR_OCCURRED_KEY"; + private static final ThreadLocal> THREAD_LOCAL; + protected final DataStoreContext dataStoreContext; + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + public MessageHandler(final DataStoreContext dataStoreContext) { + this.dataStoreContext = dataStoreContext; + } + public abstract void messageReceived(T message) throws Exception; + + public void messageReceived(final IoSession session, final MongoMessage message) throws Exception { + try { + this.setSession(session); + this.messageReceived((T)message); + if (!this.isErrorOccured()) { + this.clearLastError(); + } + } + catch (Throwable e) { + String hostAddress = ""; + if (session.getRemoteAddress() instanceof InetSocketAddress) { + hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress(); + } + this.setLastError(this.buildExceptionMessage(e)); + this.logger.error("error processing host:" + hostAddress + ",message: " + message, e); + } + finally { + this.clearErrorOccured(); + } + } + + private void clearErrorOccured() { + MessageHandler.THREAD_LOCAL.get().remove("com.goojia.datastore.ERROR_OCCURRED_KEY"); + } + + private void setErrorOccured() { + MessageHandler.THREAD_LOCAL.get().put("com.goojia.datastore.ERROR_OCCURRED_KEY", true); + } + + private boolean isErrorOccured() { + final Boolean b = (Boolean) MessageHandler.THREAD_LOCAL.get().get("com.goojia.datastore.ERROR_OCCURRED_KEY"); + return null != b && b; + } + + protected IoSession getSession() { + return (IoSession)MessageHandler.THREAD_LOCAL.get().get("com.goojia.datastore.SESSION_KEY"); + } + + private void setSession(final IoSession session) { + MessageHandler.THREAD_LOCAL.get().put("com.goojia.datastore.SESSION_KEY", session); + } + + protected void setLastError(final Object error) { + if (null != error) { + final IoSession session = this.getSession(); + final Object lastError = session.getAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY"); + if (null != lastError) { + session.setAttribute((Object)"com.goojia.datastore.PREV_ERROR_KEY", lastError); + } + this.setErrorOccured(); + session.setAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY", (Object)String.valueOf(error)); + } + } + + private void clearLastError() { + this.getSession().removeAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY"); + } + + protected Object getLastError() { + return this.getSession().getAttribute((Object)"com.goojia.datastore.LAST_ERROR_KEY"); + } + + protected Object getPrevError() { + return this.getSession().getAttribute((Object)"com.goojia.datastore.PREV_ERROR_KEY"); + } + + protected void writeMessage(final ReplyMessage message) { + this.getSession().write((Object)message); + } + + protected void writeAsReplyMessage(final int responseTo, final List docs) { + this.writeMessage(this.prepareReplyMessage(responseTo, docs)); + } + + protected ReplyMessage prepareReplyMessage(final int responseTo, final List docs) { + final ReplyMessage reply = new ReplyMessage(); + reply.setNumberReturned((null == docs) ? 0 : docs.size()); + reply.setResponseFlag(8); + reply.getMessageHeader().setResponseTo(responseTo); + if (null != docs && docs.size() > 0) { + reply.setDocuments((List)docs); + } + return reply; + } + + protected void writeAsReplyMessage(final int responseTo, final BSONObject... docs) { + this.writeMessage(this.prepareReplyMessage(responseTo, docs)); + } + + protected ReplyMessage prepareReplyMessage(final int responseTo, final BSONObject... docs) { + if (null != docs && docs.length > 0) { + final List docList = new ArrayList(docs.length); + for (final BSONObject doc : docs) { + docList.add(doc); + } + return this.prepareReplyMessage(responseTo, docList); + } + return this.prepareReplyMessage(responseTo, (List)null); + } + + protected void writeAsErrorMessage(final int responseTo, final Object errorMessage) { + this.writeMessage(this.prepareErrorMessage(responseTo, errorMessage)); + } + + protected ReplyMessage prepareErrorMessage(final int responseTo, final Object errorMessage) { + final ReplyMessage reply = new ReplyMessage(); + reply.setNumberReturned(1); + reply.setResponseFlag(8); + reply.getMessageHeader().setResponseTo(responseTo); + final BSONObject msg = (BSONObject)new BasicBSONObject("$err", (Object)((errorMessage == null) ? null : errorMessage.toString())); + msg.put("err", (Object)((errorMessage == null) ? null : errorMessage.toString())); + msg.put("errmsg", (Object)((errorMessage == null) ? null : errorMessage.toString())); + msg.put("ok", (Object)0.0); + reply.setDocuments(Arrays.asList(msg)); + return reply; + } + + protected String buildExceptionMessage(final Throwable e) { + final StringBuilder ret = new StringBuilder(); + ret.append(e.getClass().getName()).append(": ").append(e.getMessage()); + Throwable cause = e; + while ((cause = cause.getCause()) != null) { + ret.append(", caused by ").append(cause.getClass().getName()).append(": ").append(cause.getMessage()); + } + return ret.toString(); + } + + protected void assertNotReadOnly() throws DataStoreException { + if (this.dataStoreContext.getDataStoreConfig().isReadOnly()) { + throw new DataStoreException("read only server !"); + } + } + + protected boolean useCursor() { + return (Boolean) this.getSession().getAttribute((Object)"useCursor", false); + } + + static { + THREAD_LOCAL = new ThreadLocal>() { + @Override + protected Map initialValue() { + return new HashMap(); + } + }; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/QueryMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/QueryMessageHandler.java index 8ae4e6ebb..fd2444c5b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/QueryMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/QueryMessageHandler.java @@ -1,490 +1,501 @@ package com.fangjia.datastore.handler; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Collection; +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import org.apache.commons.lang.StringUtils; import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import com.mongodb.BasicDBList; -import org.apache.mina.core.session.IoSession; -import org.bson.BSONObject; -import org.bson.BasicBSONObject; - -import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.DataStore; -import com.fangjia.datastore.DataStoreContext; -import com.fangjia.datastore.DataStoreException; -import com.fangjia.datastore.QueryResponse; -import com.fangjia.datastore.config.Field; import com.fangjia.datastore.config.StoreConfig; -import com.fangjia.datastore.mongo.protocol.QueryMessage; +import com.fangjia.datastore.config.Field; import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import com.fangjia.datastore.QueryResponse; +import com.fangjia.datastore.Cursor; +import java.util.Collection; import com.fangjia.datastore.solr.Index; -import com.fangjia.datastore.utils.MongoUtils; +import java.util.Iterator; +import java.util.Set; import com.mongodb.BasicDBObject; import com.mongodb.DBObject; +import java.util.ArrayList; +import com.fangjia.datastore.ConfigurationException; +import com.mongodb.BasicDBList; +import java.util.List; +import com.fangjia.datastore.DataStore; +import org.bson.types.BasicBSONList; +import org.bson.BasicBSONObject; +import org.bson.BSONObject; +import com.fangjia.datastore.DataStoreException; +import org.apache.mina.core.session.IoSession; +import java.net.InetSocketAddress; +import com.fangjia.datastore.utils.MongoUtils; +import com.fangjia.datastore.DataStoreContext; +import com.fangjia.datastore.mongo.protocol.QueryMessage; -public class QueryMessageHandler extends MessageHandler { - - public QueryMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(QueryMessage message,String ip) throws Exception { - Object error = null; - try { - String[] nameSplit = MongoUtils.parseFullCollectionName(message.getFullCollectionName()); - if (null == nameSplit) { - error = "invalid full collection name: " + message.getFullCollectionName(); - } else { - doMessageReceived(message, nameSplit[0], nameSplit[1]); - } - } catch (Throwable e) { - IoSession session = getSession(); - String hostAddress = ""; - if(session != null && session.getRemoteAddress() instanceof InetSocketAddress) { - hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress(); - } - - error = buildExceptionMessage(e); - this.writeAsErrorMessage(message.getMessageHeader().getRequestID(), error); - if (logger.isErrorEnabled()) { - logger.error("error process host:" + hostAddress + ",message: " + message, e); - } - } finally { - this.setLastError(error); - } - } - - private void doMessageReceived(QueryMessage message, String dbName, String collectionName) throws DataStoreException { - if (collectionName.equals("$cmd")) { - onCommand(message, dbName); - return; - } - if (collectionName.equals("system.namespaces")) { - showCollections(message, dbName, collectionName); - } else if (collectionName.equals("system.indexes")) { - getIndexes(message, dbName, collectionName); - } else { - find(message, dbName, collectionName); - } - } - - private void onCommand(QueryMessage message, String dbName) throws DataStoreException { - if (dbName.equals("admin")) { - onAdminCommand(message); - return; - } - BSONObject query = message.getQuery(); - if (query.containsField("getlasterror")) { - getLastError(message, dbName); - } else if (query.containsField("getpreverror")) { - getPrevError(message, dbName); - } else if (query.containsField("count")) { - count(message, dbName); - } else if (query.containsField("getSchema")) { - getSchema(message, dbName); - } else if (query.containsField("reIndex")) { - reIndex(message, dbName); - } else if (query.containsField("reload")) { - reload(message, dbName); - } else if(query.containsField("dbStats")){ - dbStats(message, dbName); - } else if(query.containsField("listCollections")){ - listCollections(message, dbName); - } else if (query.containsField("isMaster") || query.containsField("ismaster")) { - isMaster(message); - } else if(query.containsField("buildinfo") || query.containsField("buildInfo")){ - buildinfo(message); - } else if(query.containsField("serverStatus")){ - serverStatus(message); - } else if(query.containsField("connectionStatus")){ - connectionStatus(message); - } else if (query.containsField("find")) { - find(message, dbName, query.get("find").toString() ); - } else { - delegateCommand(message, dbName); - } - } - - private void onAdminCommand(QueryMessage message) throws DataStoreException { - BSONObject query = message.getQuery(); - if (query.containsField("whatsmyuri")) { - whatsMyUri(message); - } else if (query.containsField("listDatabases")) { - listDatabases(message); - } else if (query.containsField("nonce")) { - nonce(message); - } else if (query.containsField("isMaster") || query.containsField("ismaster")) { - isMaster(message); - } else if(query.containsField("buildinfo") || query.containsField("buildInfo")){ - buildinfo(message); - } else if (query.containsField("replSetGetStatus")) { - replSetGetStatus(message); - } else if(query.containsField("getlasterror")){ - getLastError(message, "admin"); - } else if(query.containsField("serverStatus")){ - serverStatus(message); - } else if(query.containsField("connectionStatus")){ - connectionStatus(message); - } else if (query.containsField("getLog")) { - getLog(message); - } else { - delegateAdminCommand(message); - } - } - - private void replSetGetStatus(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 0); - doc.put("errmsg", "not running with --replSet"); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void delegateCommand(QueryMessage message, String dbName) throws DataStoreException { - throw new DataStoreException("unsupported command:" + message.getQuery()); - } - - private void getLog(QueryMessage message){ - - BasicDBList names = new BasicDBList(); - names.add("global"); - names.add("startupWarnings"); - - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1); - doc.put("names", names); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - - } - - private void dbStats(QueryMessage message, String dbName) throws DataStoreException { - - BSONObject doc = new BasicBSONObject(); - doc.put("db", dbName); - doc.put("collections", this.dataStoreContext.getDataStoreConfig().getStoreNames()); - doc.put("ok", 1.0); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - - - private void delegateAdminCommand(QueryMessage message) throws DataStoreException { - throw new DataStoreException("unsupported command:" + message.getQuery()); - } - - private void reload(QueryMessage message, String dbName) throws DataStoreException { - assertNotReadOnly(); - try { - this.dataStoreContext.reload(); - this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("reload")).reload(); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } catch (ConfigurationException e) { - throw new DataStoreException("error reload config: ", e); - } - } - - /** - * show collections - * - * @param message - */ - private void showCollections(QueryMessage message, String dbName, String collectionName) throws DataStoreException { - Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); - List docList = new ArrayList(storeNames.size()); - for (String sn : storeNames) { - if (sn.startsWith(dbName + ".")) { - docList.add(new BasicDBObject("name", sn)); - } - } - writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList); - } - - /** - * db.foo.getIndexes() - * - * @param message - * @param dbName - * @param collectionName - * @throwstion - */ - private void getIndexes(QueryMessage message, String dbName, String collectionName) throws DataStoreException { - String ns = (String) message.getQuery().get("ns"); - Collection indexes = this.dataStoreContext.getDataStore(ns).getIndexes(); - List docList = new ArrayList(indexes.size()); - for (Index i : indexes) { - docList.add(i.toBSONObject()); - } - writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList); - } - - /** - * db.foo.find() - * - * @param message - * @param dbName - * @param collectionName - * @throws DataStoreException - */ - private void find(QueryMessage message, String dbName, String collectionName) throws DataStoreException { - - String fullCollectionName; - if(message.getFullCollectionName().contains("$cmd")){ - fullCollectionName = dbName + "." + collectionName; - }else{ - fullCollectionName = message.getFullCollectionName(); - } - DataStore dataStore = this.dataStoreContext.getDataStore(fullCollectionName); - QueryResponse response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message - .getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn() - 1, message - .isBulkQuery(), message.isFromMaster()); - ReplyMessage reply = prepareReplyMessage(message.getMessageHeader().getRequestID(), response.getDocList()); - reply.setStartingFrom(message.getNumberToSkip()); - // if (response.hasMore()) { - // Cursor cursor = new Cursor((int) message.getNumberToSkip(), (int) - // response.getNumReturned(), message, - // response.getNumFound()); - // this.dataStoreContext.addCursor(cursor); - // reply.setCursorID(cursor.getId()); - // } - this.writeMessage(reply); - } - - /** - * db.getLastError() - * - * @param message - * @param dbName - */ - private void getLastError(QueryMessage message, String dbName) { - Object lastError = this.getLastError(); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("n", 1); - doc.put("$err", lastError == null ? null : lastError.toString()); - doc.put("err", lastError == null ? null : lastError.toString()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * db.getPrevError() - * - * @param message - * @param dbName - */ - private void getPrevError(QueryMessage message, String dbName) { - Object prevError = this.getPrevError(); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("n", 0); - doc.put("$err", prevError == null ? null : prevError.toString()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * db.foo.count() - * - * @param message - * @param dbName - * @throws DataStoreException - */ - private void count(QueryMessage message, String dbName) throws DataStoreException { - BSONObject query = message.getQuery(); - int count = this.dataStoreContext.getDataStore(dbName + "." + query.get("count")).count(message.getRealQuery(), - message.isFromMaster()); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("n", (double) count); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * db.foo.rumCommnad('getSchema') - * - * @param message - * @param dbName - */ - private void getSchema(QueryMessage message, String dbName) throws DataStoreException { - BSONObject query = message.getQuery(); - String collectionName = (String) query.get("getSchema"); - StoreConfig storeConfig = this.dataStoreContext.getDataStore(dbName + "." + collectionName).getStoreConfig(); - Collection fields = storeConfig.getFields(); - List docList = new ArrayList(fields.size()); - for (Field field : fields) { - BasicDBObject f = new BasicDBObject(); - f.put("name", field.getName()); - f.put("type", field.getType().getType()); - docList.add(f); - } - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("schema", docList); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - *

    - * 不带参数重建:db.foo.reIndex() 或 db.foo.runCommand('reIndex') - *

    - * - *

    - * 带参数重建:db.foo.runCommand('reIndex',{query:{cat:'sell',city:'上海'}}) - *

    - * - * @param message - * @param dbName - * @throws Exception - */ - private void reIndex(QueryMessage message, String dbName) throws DataStoreException { - assertNotReadOnly(); - BSONObject query = message.getQuery(); - String collectionName = (String) query.get("reIndex"); - DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + collectionName); - // 新增查询条件 add by yukai 2011/07/14 - BSONObject doc = new BasicBSONObject(); - Object realQuery = query.get("query"); - if (null == realQuery) - doc.put("version", dataStore.reIndex(null)); - else if (realQuery instanceof BSONObject) - dataStore.reIndex((BSONObject) realQuery); - else - throw new DataStoreException("invalid re-index query: " + realQuery); - doc.put("ok", 1.0); - doc.put("n", 0D); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * db.runCommand('whatsmyuri') - * - * @param message - */ - private void whatsMyUri(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - InetSocketAddress address = (InetSocketAddress) getSession().getRemoteAddress(); - doc.put("you", address.getAddress().getHostAddress() + ":" + address.getPort()); - doc.put("ok", 1.0); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * show dbs - * - * @param message - */ - private void listDatabases(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); - Set dbNames = new HashSet(); - for (String sn : storeNames) { - int i = sn.indexOf('.'); - if (i > 0) { - String name = sn.substring(0, i); - dbNames.add(name); - } - } - List docs = new ArrayList(dbNames.size()); - for (String dn : dbNames) { - docs.add(new BasicDBObject("name", dn)); - } - doc.put("databases", docs); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - /** - * show dbs - * - * @param message - */ - private void listCollections(QueryMessage message, String dbName) { - System.out.println(message.getMessageHeader().getRequestID()); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - BSONObject cursor = new BasicDBObject(); - doc.put("cursor", cursor); - - cursor.put("id", (long)message.getMessageHeader().getRequestID()); - cursor.put("ns", dbName + ".$cmd.listCollections"); - BasicDBList firstBatch = new BasicDBList(); - cursor.put("firstBatch", firstBatch); - Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); - for (String sn : storeNames) { - if (sn.startsWith(dbName + ".")) { - BSONObject collection = new BasicDBObject(); - collection.put("name", sn.replace(dbName + ".", "")); - collection.put("options", new BasicDBObject()); - firstBatch.add(collection); - } - } - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - - - - /** - * db.runCommand('nonce') - * - * @param message - */ - private void nonce(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("errmsg", "no such cmd"); - doc.put("bad cmd", new BasicBSONObject("nonce", true)); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void isMaster(QueryMessage message) { - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("ismaster", true); - doc.put("secondary", false); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void buildinfo(QueryMessage message) { - BasicDBList versionArray = new BasicDBList(); - versionArray.add(3); - versionArray.add(2); - versionArray.add(9); - versionArray.add(0); - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - doc.put("bits", 64.0); - doc.put("debug", false); - doc.put("version" ,"3.2.9"); - doc.put("versionArray" ,versionArray); - doc.put("javascriptEngine", "V8"); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void serverStatus(QueryMessage message){ - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - private void connectionStatus(QueryMessage message){ - BSONObject doc = new BasicBSONObject(); - doc.put("ok", 1.0); - InetSocketAddress localAddress = (InetSocketAddress) this.getSession().getLocalAddress(); - doc.put("primary", localAddress.getAddress().getHostAddress()); - writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); - } - - +public class QueryMessageHandler extends MessageHandler +{ + public QueryMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final QueryMessage message) throws Exception { + Object error = null; + try { + final String[] nameSplit = MongoUtils.parseFullCollectionName(message.getFullCollectionName()); + if (null == nameSplit) { + error = "invalid full collection name: " + message.getFullCollectionName(); + } + else { + this.doMessageReceived(message, nameSplit[0], nameSplit[1]); + } + } + catch (Throwable e) { + final IoSession session = this.getSession(); + String hostAddress = ""; + if (session != null && session.getRemoteAddress() instanceof InetSocketAddress) { + hostAddress = ((InetSocketAddress)session.getRemoteAddress()).getAddress().getHostAddress(); + } + error = this.buildExceptionMessage(e); + this.writeAsErrorMessage(message.getMessageHeader().getRequestID(), error); + if (this.logger.isErrorEnabled()) { + this.logger.error("error process host:" + hostAddress + ",message: " + message, e); + } + } + finally { + this.setLastError(error); + } + } + + private void doMessageReceived(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException { + if (collectionName.equals("$cmd")) { + this.onCommand(message, dbName); + return; + } + if (collectionName.equals("system.namespaces")) { + this.showCollections(message, dbName, collectionName); + } + else if (collectionName.equals("system.indexes")) { + this.getIndexes(message, dbName, collectionName); + } + else { + this.find(message, dbName, collectionName); + } + } + + private void onCommand(final QueryMessage message, final String dbName) throws DataStoreException { + if (dbName.equals("admin")) { + this.onAdminCommand(message); + return; + } + final BSONObject query = message.getQuery(); + if (query.containsField("getlasterror")) { + this.getLastError(message, dbName); + } + else if (query.containsField("getpreverror")) { + this.getPrevError(message, dbName); + } + else if (query.containsField("count")) { + this.count(message, dbName); + } + else if (query.containsField("getSchema")) { + this.getSchema(message, dbName); + } + else if (query.containsField("reIndex")) { + this.reIndex(message, dbName); + } + else if (query.containsField("reload")) { + this.reload(message, dbName); + } + else if (query.containsField("dbStats")) { + this.dbStats(message, dbName); + } + else if (query.containsField("listCollections")) { + this.listCollections(message, dbName); + } + else if (query.containsField("isMaster") || query.containsField("ismaster")) { + this.isMaster(message); + } + else if (query.containsField("buildinfo") || query.containsField("buildInfo")) { + this.buildinfo(message); + } + else if (query.containsField("serverStatus")) { + this.serverStatus(message); + } + else if (query.containsField("connectionStatus")) { + this.connectionStatus(message); + } + else if (query.containsField("update")) { + this.update(message, dbName); + } + else if (query.containsField("insert")) { + this.insert(message, dbName); + } + else if (query.containsField("ping")) { + this.ping(message); + } + else if (query.containsField("find")) { + this.find(message, dbName, query.get("find").toString()); + } + else { + this.delegateCommand(message, dbName); + } + } + + private void onAdminCommand(final QueryMessage message) throws DataStoreException { + final BSONObject query = message.getQuery(); + if (query.containsField("whatsmyuri")) { + this.whatsMyUri(message); + } + else if (query.containsField("listDatabases")) { + this.listDatabases(message); + } + else if (query.containsField("nonce")) { + this.nonce(message); + } + else if (query.containsField("isMaster") || query.containsField("ismaster")) { + this.isMaster(message); + } + else if (query.containsField("buildinfo") || query.containsField("buildInfo")) { + this.buildinfo(message); + } + else if (query.containsField("replSetGetStatus")) { + this.replSetGetStatus(message); + } + else if (query.containsField("getlasterror")) { + this.getLastError(message, "admin"); + } + else if (query.containsField("serverStatus")) { + this.serverStatus(message); + } + else if (query.containsField("connectionStatus")) { + this.connectionStatus(message); + } + else if (query.containsField("collstats")) { + this.collstats(message); + } + else if (query.containsField("getLog")) { + this.getLog(message); + } + else if (query.containsField("ping")) { + this.ping(message); + } + else { + this.delegateAdminCommand(message); + } + } + + private void replSetGetStatus(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)0); + doc.put("errmsg", (Object)"not running with --replSet"); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void delegateCommand(final QueryMessage message, final String dbName) throws DataStoreException { + throw new DataStoreException("unsupported command:" + message.getQuery()); + } + + private void collstats(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1); + doc.put("ns", (Object)message.getFullCollectionName()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void ping(final QueryMessage message) throws DataStoreException { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void update(final QueryMessage message, final String dbName) throws DataStoreException { + final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("update")); + final BasicBSONList updates = (BasicBSONList)message.getQuery().get("updates"); + final BSONObject selector = (BSONObject)((BasicBSONObject)updates.get(0)).get("q"); + final BSONObject update = (BSONObject)((BasicBSONObject)updates.get(0)).get("u"); + final int count = dataStore.update(selector, update); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("nModified", (Object)count); + doc.put("n", (Object)count); + doc.put("ok", (Object)1); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void insert(final QueryMessage message, final String dbName) throws DataStoreException { + final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("insert")); + final List documents = (List)message.getQuery().get("documents"); + dataStore.save(documents); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("n", (Object)documents.size()); + doc.put("ok", (Object)1); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void getLog(final QueryMessage message) { + final BasicDBList names = new BasicDBList(); + names.add((Object)"global"); + names.add((Object)"startupWarnings"); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1); + doc.put("names", (Object)names); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void dbStats(final QueryMessage message, final String dbName) throws DataStoreException { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("db", (Object)dbName); + doc.put("collections", (Object)this.dataStoreContext.getDataStoreConfig().getStoreNames()); + doc.put("ok", (Object)1.0); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void delegateAdminCommand(final QueryMessage message) throws DataStoreException { + throw new DataStoreException("unsupported command:" + message.getQuery()); + } + + private void reload(final QueryMessage message, final String dbName) throws DataStoreException { + this.assertNotReadOnly(); + try { + this.dataStoreContext.reload(); + this.dataStoreContext.getDataStore(dbName + "." + message.getQuery().get("reload")).reload(); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + catch (ConfigurationException e) { + throw new DataStoreException("error reload config: ", e); + } + } + + private void showCollections(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException { + final Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); + final List docList = new ArrayList(storeNames.size()); + for (final String sn : storeNames) { + if (sn.startsWith(dbName + ".")) { + docList.add((DBObject)new BasicDBObject("name", (Object)sn)); + } + } + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), (List)docList); + } + + private void getIndexes(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException { + final String ns = (String)message.getQuery().get("ns"); + final Collection indexes = this.dataStoreContext.getDataStore(ns).getIndexes(); + final List docList = new ArrayList(indexes.size()); + for (final Index i : indexes) { + docList.add(i.toBSONObject()); + } + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), docList); + } + + private void find(final QueryMessage message, final String dbName, final String collectionName) throws DataStoreException { + String fullCollectionName; + if (message.getFullCollectionName().contains("$cmd")) { + fullCollectionName = dbName + "." + collectionName; + } + else { + fullCollectionName = message.getFullCollectionName(); + } + final DataStore dataStore = this.dataStoreContext.getDataStore(fullCollectionName); + QueryResponse response; + if (this.useCursor()) { + response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn(), message.isBulkQuery(), message.isFromMaster()); + } + else { + response = dataStore.find(message.getRealQuery(), message.getReturnFieldSelector(), message.getOrderByQuery(), message.getFacetQuery(), message.getNumberToSkip(), message.getNumberToReturn() - 1, message.isBulkQuery(), message.isFromMaster()); + } + final List doc = response.getDocList(); + if (this.useCursor()) { + doc.remove(0); + } + final ReplyMessage reply = this.prepareReplyMessage(message.getMessageHeader().getRequestID(), (List)doc); + reply.setStartingFrom(message.getNumberToSkip()); + if (response.hasMore() && this.useCursor()) { + final Cursor cursor = new Cursor(message.getNumberToSkip(), message.getNumberToReturn(), message, response.getNumFound()); + this.dataStoreContext.addCursor(cursor); + reply.setCursorID(cursor.getId()); + } + this.writeMessage(reply); + } + + private void getLastError(final QueryMessage message, final String dbName) { + final Object lastError = this.getLastError(); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("n", (Object)1); + doc.put("$err", (Object)((lastError == null) ? null : lastError.toString())); + doc.put("err", (Object)((lastError == null) ? null : lastError.toString())); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void getPrevError(final QueryMessage message, final String dbName) { + final Object prevError = this.getPrevError(); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("n", (Object)0); + doc.put("$err", (Object)((prevError == null) ? null : prevError.toString())); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void count(final QueryMessage message, final String dbName) throws DataStoreException { + final BSONObject query = message.getQuery(); + final int count = this.dataStoreContext.getDataStore(dbName + "." + query.get("count")).count(message.getRealQuery(), message.isFromMaster()); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("n", (Object)(double)count); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void getSchema(final QueryMessage message, final String dbName) throws DataStoreException { + final BSONObject query = message.getQuery(); + final String collectionName = (String)query.get("getSchema"); + final StoreConfig storeConfig = this.dataStoreContext.getDataStore(dbName + "." + collectionName).getStoreConfig(); + final Collection fields = storeConfig.getFields(); + final List docList = new ArrayList(fields.size()); + for (final Field field : fields) { + final BasicDBObject f = new BasicDBObject(); + f.put("name", field.getName()); + f.put("type", field.getType().getType()); + docList.add(f); + } + final BSONObject doc = new BasicBSONObject(); + doc.put("ok", 1.0); + doc.put("schema", docList); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void reIndex(final QueryMessage message, final String dbName) throws DataStoreException { + this.assertNotReadOnly(); + final BSONObject query = message.getQuery(); + final String collectionName = (String)query.get("reIndex"); + final DataStore dataStore = this.dataStoreContext.getDataStore(dbName + "." + collectionName); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + final Object realQuery = query.get("query"); + if (null == realQuery) { + doc.put("version", (Object)dataStore.reIndex(null)); + } + else { + if (!(realQuery instanceof BSONObject)) { + throw new DataStoreException("invalid re-index query: " + realQuery); + } + dataStore.reIndex((BSONObject)realQuery); + } + doc.put("ok", (Object)1.0); + doc.put("n", (Object)0.0); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void whatsMyUri(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + final InetSocketAddress address = (InetSocketAddress)this.getSession().getRemoteAddress(); + doc.put("you", (Object)(address.getAddress().getHostAddress() + ":" + address.getPort())); + doc.put("ok", (Object)1.0); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void listDatabases(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + final Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); + final Set dbNames = new HashSet(); + for (final String sn : storeNames) { + final int i = sn.indexOf(46); + if (i > 0) { + final String name = sn.substring(0, i); + dbNames.add(name); + } + } + final List docs = new ArrayList(dbNames.size()); + for (final String dn : dbNames) { + docs.add((DBObject)new BasicDBObject("name", (Object)dn)); + } + doc.put("databases", (Object)docs); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void listCollections(final QueryMessage message, final String dbName) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + final BSONObject cursor = (BSONObject)new BasicDBObject(); + doc.put("cursor", (Object)cursor); + cursor.put("id", (Object)(long)message.getMessageHeader().getRequestID()); + cursor.put("ns", (Object)(dbName + ".$cmd.listCollections")); + final BasicDBList firstBatch = new BasicDBList(); + cursor.put("firstBatch", (Object)firstBatch); + final Set storeNames = this.dataStoreContext.getDataStoreConfig().getStoreNames(); + for (final String sn : storeNames) { + if (sn.startsWith(dbName + ".")) { + final BSONObject collection = (BSONObject)new BasicDBObject(); + collection.put("name", (Object)sn.replace(dbName + ".", "")); + collection.put("options", (Object)new BasicDBObject()); + firstBatch.add((Object)collection); + } + } + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void nonce(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("errmsg", (Object)"no such cmd"); + doc.put("bad cmd", (Object)new BasicBSONObject("nonce", (Object)true)); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void isMaster(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("ismaster", (Object)true); + doc.put("secondary", (Object)false); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + doc.put("maxWireVersion", (Object)2); + doc.put("minWireVersion", (Object)0); + final BasicBSONObject client = (BasicBSONObject)message.getQuery().get("client"); + if (client != null) { + final String platform = client.getString("platform"); + if (platform != null && (platform.contains("python") || platform.contains("Python"))) { + this.getSession().setAttribute((Object)"useCursor", (Object)true); + } + final BasicBSONObject applicationInfo = (BasicBSONObject)client.get("application"); + if (applicationInfo != null && StringUtils.isNotBlank(applicationInfo.getString("name"))) { + this.getSession().setAttribute((Object)"useCursor", (Object)true); + } + } + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void buildinfo(final QueryMessage message) { + final BasicDBList versionArray = new BasicDBList(); + versionArray.add((Object)2); + versionArray.add((Object)6); + versionArray.add((Object)8); + versionArray.add((Object)0); + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + doc.put("bits", (Object)64.0); + doc.put("debug", (Object)false); + doc.put("version", (Object)"2.6.8"); + doc.put("versionArray", (Object)versionArray); + doc.put("javascriptEngine", (Object)"V8"); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void serverStatus(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } + + private void connectionStatus(final QueryMessage message) { + final BSONObject doc = (BSONObject)new BasicBSONObject(); + doc.put("ok", (Object)1.0); + final InetSocketAddress localAddress = (InetSocketAddress)this.getSession().getLocalAddress(); + doc.put("primary", (Object)localAddress.getAddress().getHostAddress()); + this.writeAsReplyMessage(message.getMessageHeader().getRequestID(), doc); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/UpdateMessageHandler.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/UpdateMessageHandler.java index 58f3a45df..1641938a6 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/UpdateMessageHandler.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/handler/UpdateMessageHandler.java @@ -1,27 +1,27 @@ package com.fangjia.datastore.handler; +import com.fangjia.datastore.mongo.protocol.MongoMessage; import com.fangjia.datastore.DataStore; -import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.DataStoreException; +import com.fangjia.datastore.DataStoreContext; import com.fangjia.datastore.mongo.protocol.UpdateMessage; -public class UpdateMessageHandler extends MessageHandler { - - public UpdateMessageHandler(DataStoreContext dataStoreContext) { - super(dataStoreContext); - } - - public void messageReceived(UpdateMessage message,String ip) throws Exception { - assertNotReadOnly(); - if (message.isUpsert()) { - throw new DataStoreException("upsert is not supported."); - } - if (!message.isMultiUpdate()) { - throw new DataStoreException("only multi update is supported."); - } - - DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); - dataStore.update(message.getSelector(), message.getUpdate(),ip); - } - +public class UpdateMessageHandler extends MessageHandler +{ + public UpdateMessageHandler(final DataStoreContext dataStoreContext) { + super(dataStoreContext); + } + + @Override + public void messageReceived(final UpdateMessage message) throws Exception { + this.assertNotReadOnly(); + if (message.isUpsert()) { + throw new DataStoreException("upsert is not supported."); + } + if (!message.isMultiUpdate()) { + throw new DataStoreException("only multi update is supported."); + } + final DataStore dataStore = this.dataStoreContext.getDataStore(message.getFullCollectionName()); + dataStore.update(message.getSelector(), message.getUpdate()); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONInputStream.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONInputStream.java index b77a769f9..b32f524c5 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONInputStream.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONInputStream.java @@ -1,92 +1,75 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.codec; -import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.ByteArrayOutputStream; +import org.apache.mina.core.buffer.IoBuffer; import java.io.InputStream; -import org.apache.mina.core.buffer.IoBuffer; - -public class BSONInputStream extends InputStream { - private final IoBuffer buffer; - - public BSONInputStream(IoBuffer buffer) { - this.buffer = buffer; - } - - @Override - public int read() { - return buffer.get(); - } - - public synchronized int read(byte b[], int off, int len) { - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); - } - buffer.get(b, off, len); - return len; - } - - public int readInteger() { - return this.buffer.getInt(); - } - - public long readLong() { - return this.buffer.getLong(); - } - - public double readDouble() { - return this.buffer.getDouble(); - } - - public String readCString() throws IOException { - ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); - int b; - while (true) { - b = read(); - if (b != 0) { - byteStream.write(b); - } else { - return new String(byteStream.toByteArray(), "UTF-8"); - } - } - } - - public String readString() throws IOException { - int len = readInteger(); - if (len < 0) { - throw new IOException("can not read string from the stream"); - } - if (len == 0) { - read(); - return ""; - } - byte[] buf = new byte[len]; - read(buf); - return new String(buf, 0, len - 1, "UTF-8"); - } - - @Override - public int available() throws IOException { - return buffer.remaining(); - } - +public class BSONInputStream extends InputStream +{ + private final IoBuffer buffer; + + public BSONInputStream(final IoBuffer buffer) { + this.buffer = buffer; + } + + @Override + public int read() { + return this.buffer.get(); + } + + @Override + public synchronized int read(final byte[] b, final int off, final int len) { + if (b == null) { + throw new NullPointerException(); + } + if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } + this.buffer.get(b, off, len); + return len; + } + + public int readInteger() { + return this.buffer.getInt(); + } + + public long readLong() { + return this.buffer.getLong(); + } + + public double readDouble() { + return this.buffer.getDouble(); + } + + public String readCString() throws IOException { + final ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + while (true) { + final int b = this.read(); + if (b == 0) { + break; + } + byteStream.write(b); + } + return new String(byteStream.toByteArray(), "UTF-8"); + } + + public String readString() throws IOException { + final int len = this.readInteger(); + if (len < 0) { + throw new IOException("can not read string from the stream"); + } + if (len == 0) { + this.read(); + return ""; + } + final byte[] buf = new byte[len]; + this.read(buf); + return new String(buf, 0, len - 1, "UTF-8"); + } + + @Override + public int available() throws IOException { + return this.buffer.remaining(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONIoBufferAllocator.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONIoBufferAllocator.java index 042264ec6..97e452e9f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONIoBufferAllocator.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONIoBufferAllocator.java @@ -1,95 +1,83 @@ package com.fangjia.datastore.mongo.codec; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - import org.apache.mina.core.buffer.AbstractIoBuffer; +import java.nio.ByteOrder; +import java.nio.ByteBuffer; import org.apache.mina.core.buffer.IoBuffer; import org.apache.mina.core.buffer.IoBufferAllocator; -public class BSONIoBufferAllocator implements IoBufferAllocator { - - public IoBuffer allocate(int capacity, boolean direct) { - return wrap(allocateNioBuffer(capacity, direct)); - } - - public ByteBuffer allocateNioBuffer(int capacity, boolean direct) { - ByteBuffer nioBuffer; - if (direct) { - nioBuffer = ByteBuffer.allocateDirect(capacity); - } else { - nioBuffer = ByteBuffer.allocate(capacity); - } - nioBuffer.order(ByteOrder.LITTLE_ENDIAN); - return nioBuffer; - } - - public IoBuffer wrap(ByteBuffer nioBuffer) { - return new SimpleBuffer(nioBuffer); - } - - public void dispose() { - // Do nothing - } - - private class SimpleBuffer extends AbstractIoBuffer { - private ByteBuffer buf; - - protected SimpleBuffer(ByteBuffer buf) { - super(BSONIoBufferAllocator.this, buf.capacity()); - this.buf = buf; - this.buf.order(ByteOrder.LITTLE_ENDIAN); - } - - protected SimpleBuffer(SimpleBuffer parent, ByteBuffer buf) { - super(parent); - this.buf = buf; - this.buf.order(ByteOrder.LITTLE_ENDIAN); - } - - @Override - public ByteBuffer buf() { - return buf; - } - - @Override - protected void buf(ByteBuffer buf) { - this.buf = buf; - } - - @Override - protected IoBuffer duplicate0() { - return new SimpleBuffer(this, this.buf.duplicate()); - } - - @Override - protected IoBuffer slice0() { - return new SimpleBuffer(this, this.buf.slice()); - } - - @Override - protected IoBuffer asReadOnlyBuffer0() { - return new SimpleBuffer(this, this.buf.asReadOnlyBuffer()); - } - - @Override - public byte[] array() { - return buf.array(); - } - - @Override - public int arrayOffset() { - return buf.arrayOffset(); - } - - @Override - public boolean hasArray() { - return buf.hasArray(); - } - - @Override - public void free() { - // Do nothing - } - } +public class BSONIoBufferAllocator implements IoBufferAllocator +{ + public IoBuffer allocate(final int capacity, final boolean direct) { + return this.wrap(this.allocateNioBuffer(capacity, direct)); + } + + public ByteBuffer allocateNioBuffer(final int capacity, final boolean direct) { + ByteBuffer nioBuffer; + if (direct) { + nioBuffer = ByteBuffer.allocateDirect(capacity); + } + else { + nioBuffer = ByteBuffer.allocate(capacity); + } + nioBuffer.order(ByteOrder.LITTLE_ENDIAN); + return nioBuffer; + } + + public IoBuffer wrap(final ByteBuffer nioBuffer) { + return (IoBuffer)new SimpleBuffer(nioBuffer); + } + + public void dispose() { + } + + private class SimpleBuffer extends AbstractIoBuffer + { + private ByteBuffer buf; + + protected SimpleBuffer(final ByteBuffer buf) { + super((IoBufferAllocator)BSONIoBufferAllocator.this, buf.capacity()); + (this.buf = buf).order(ByteOrder.LITTLE_ENDIAN); + } + + protected SimpleBuffer(final SimpleBuffer parent, final ByteBuffer buf) { + super((AbstractIoBuffer)parent); + (this.buf = buf).order(ByteOrder.LITTLE_ENDIAN); + } + + public ByteBuffer buf() { + return this.buf; + } + + protected void buf(final ByteBuffer buf) { + this.buf = buf; + } + + protected IoBuffer duplicate0() { + return (IoBuffer)new SimpleBuffer(this, this.buf.duplicate()); + } + + protected IoBuffer slice0() { + return (IoBuffer)new SimpleBuffer(this, this.buf.slice()); + } + + protected IoBuffer asReadOnlyBuffer0() { + return (IoBuffer)new SimpleBuffer(this, this.buf.asReadOnlyBuffer()); + } + + public byte[] array() { + return this.buf.array(); + } + + public int arrayOffset() { + return this.buf.arrayOffset(); + } + + public boolean hasArray() { + return this.buf.hasArray(); + } + + public void free() { + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONOutputStream.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONOutputStream.java index 77254eb6d..6e501ea79 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONOutputStream.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/BSONOutputStream.java @@ -1,102 +1,84 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.codec; -import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.util.Arrays; +import java.io.IOException; +import java.io.ByteArrayOutputStream; -public class BSONOutputStream extends ByteArrayOutputStream { - - private static final String ENCDOING = "UTF-8"; - - public BSONOutputStream() { - super(); - } - - public BSONOutputStream(int size) { - super(size); - } - - public void writeCString(String s) throws IOException { - write(s.getBytes(ENCDOING)); - write(0); - } - - public void writeInteger(int i) throws IOException { - write(i); - write(i >> 8); - write(i >> 16); - write(i >> 24); - } - - public void writeLong(long l) throws IOException { - write((byte) l); - write((byte) (l >> 8)); - write((byte) (l >> 16)); - write((byte) (l >> 24)); - write((byte) (l >> 32)); - write((byte) (l >> 40)); - write((byte) (l >> 48)); - write((byte) (l >> 56)); - } - - public void writeString(String s) throws IOException { - byte[] bs = s.getBytes(ENCDOING); - writeInteger(bs.length + 1); - write(bs); - write(0); - } - - public void set(int index, int i) { - int newcount = Math.max(count, index + 4); - if (newcount > buf.length) { - buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount)); - } - buf[index] = (byte) i; - buf[index + 1] = (byte) (i >> 8); - buf[index + 2] = (byte) (i >> 16); - buf[index + 3] = (byte) (i >> 24); - count = newcount; - } - - public void set(int index, byte[] b, int off, int len) { - if ((off < 0) || (off > b.length) || (len < 0) || ((off + len) > b.length) || ((off + len) < 0)) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return; - } - int newcount = Math.max(count, index + len); - if (newcount > buf.length) { - buf = Arrays.copyOf(buf, Math.max(buf.length << 1, newcount)); - } - System.arraycopy(b, off, buf, index, len); - count = newcount; - } - - public void setInteger(int index, int i) { - byte[] b = new byte[] { (byte) i, (byte) (i >> 8), (byte) (i >> 16), (byte) (i >> 24) }; - set(index, b, 0, b.length); - } - - public void setLong(int index, long l) { - byte[] b = new byte[] { (byte) l, (byte) (l >> 8), (byte) (l >> 16), (byte) (l >> 24), (byte) (l >> 32), - (byte) (l >> 40), (byte) (l >> 48), (byte) (l >> 56) }; - set(index, b, 0, b.length); - } +public class BSONOutputStream extends ByteArrayOutputStream +{ + private static final String ENCDOING = "UTF-8"; + + public BSONOutputStream() { + } + + public BSONOutputStream(final int size) { + super(size); + } + + public void writeCString(final String s) throws IOException { + this.write(s.getBytes("UTF-8")); + this.write(0); + } + + public void writeInteger(final int i) throws IOException { + this.write(i); + this.write(i >> 8); + this.write(i >> 16); + this.write(i >> 24); + } + + public void writeLong(final long l) throws IOException { + this.write((byte)l); + this.write((byte)(l >> 8)); + this.write((byte)(l >> 16)); + this.write((byte)(l >> 24)); + this.write((byte)(l >> 32)); + this.write((byte)(l >> 40)); + this.write((byte)(l >> 48)); + this.write((byte)(l >> 56)); + } + + public void writeString(final String s) throws IOException { + final byte[] bs = s.getBytes("UTF-8"); + this.writeInteger(bs.length + 1); + this.write(bs); + this.write(0); + } + + public void set(final int index, final int i) { + final int newcount = Math.max(this.count, index + 4); + if (newcount > this.buf.length) { + this.buf = Arrays.copyOf(this.buf, Math.max(this.buf.length << 1, newcount)); + } + this.buf[index] = (byte)i; + this.buf[index + 1] = (byte)(i >> 8); + this.buf[index + 2] = (byte)(i >> 16); + this.buf[index + 3] = (byte)(i >> 24); + this.count = newcount; + } + + public void set(final int index, final byte[] b, final int off, final int len) { + if (off < 0 || off > b.length || len < 0 || off + len > b.length || off + len < 0) { + throw new IndexOutOfBoundsException(); + } + if (len == 0) { + return; + } + final int newcount = Math.max(this.count, index + len); + if (newcount > this.buf.length) { + this.buf = Arrays.copyOf(this.buf, Math.max(this.buf.length << 1, newcount)); + } + System.arraycopy(b, off, this.buf, index, len); + this.count = newcount; + } + + public void setInteger(final int index, final int i) { + final byte[] b = { (byte)i, (byte)(i >> 8), (byte)(i >> 16), (byte)(i >> 24) }; + this.set(index, b, 0, b.length); + } + + public void setLong(final int index, final long l) { + final byte[] b = { (byte)l, (byte)(l >> 8), (byte)(l >> 16), (byte)(l >> 24), (byte)(l >> 32), (byte)(l >> 40), (byte)(l >> 48), (byte)(l >> 56) }; + this.set(index, b, 0, b.length); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolDecoder.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolDecoder.java index dbd156d26..d9302ac7f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolDecoder.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolDecoder.java @@ -1,111 +1,108 @@ package com.fangjia.datastore.mongo.codec; +import org.slf4j.LoggerFactory; import org.apache.mina.core.buffer.BufferDataException; +import com.fangjia.datastore.mongo.protocol.UpdateMessage; +import com.fangjia.datastore.mongo.protocol.ReplyMessage; +import com.fangjia.datastore.mongo.protocol.QueryMessage; +import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; +import com.fangjia.datastore.mongo.protocol.InsertMessage; +import com.fangjia.datastore.mongo.protocol.GetMoreMessage; +import com.fangjia.datastore.mongo.protocol.DeleteMessage; +import com.fangjia.datastore.mongo.protocol.MongoMessage; +import org.apache.mina.filter.codec.ProtocolDecoderException; +import com.fangjia.datastore.mongo.protocol.OperationCode; +import org.apache.mina.filter.codec.ProtocolDecoderOutput; import org.apache.mina.core.buffer.IoBuffer; import org.apache.mina.core.session.IoSession; -import org.apache.mina.filter.codec.CumulativeProtocolDecoder; -import org.apache.mina.filter.codec.ProtocolDecoderException; -import org.apache.mina.filter.codec.ProtocolDecoderOutput; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fangjia.datastore.mongo.protocol.DeleteMessage; -import com.fangjia.datastore.mongo.protocol.GetMoreMessage; -import com.fangjia.datastore.mongo.protocol.InsertMessage; -import com.fangjia.datastore.mongo.protocol.KillCursorsMessage; -import com.fangjia.datastore.mongo.protocol.MongoMessage; -import com.fangjia.datastore.mongo.protocol.OperationCode; -import com.fangjia.datastore.mongo.protocol.QueryMessage; -import com.fangjia.datastore.mongo.protocol.ReplyMessage; -import com.fangjia.datastore.mongo.protocol.UpdateMessage; - -public class MongoProtocolDecoder extends CumulativeProtocolDecoder { - - private final static Logger LOGGER = LoggerFactory.getLogger(MongoProtocolDecoder.class); - - @Override - protected boolean doDecode(IoSession session, IoBuffer in, ProtocolDecoderOutput out) throws Exception { - try { - if (!prefixedDataAvailable(in, 4, true)) { - return false; - } - int start = in.position(); - int limit = in.limit(); - int messageSize = in.getInt(start); - in.limit(start + messageSize); - try { - final IoBuffer buf = in.slice(); - int opCodeValue = buf.getInt(buf.position() + 12); - OperationCode opCode = OperationCode.valueOf(opCodeValue); - MongoMessage message = newInstance(opCode); - BSONInputStream input = new BSONInputStream(buf); - message.read(input); - out.write(message); - return true; - } finally { - in.position(start + messageSize); - in.limit(limit); - } - } catch (Exception e) { - ProtocolDecoderException pde = new ProtocolDecoderException("error decode message:", e); - pde.setHexdump(in.getHexDump()); - LOGGER.error("", pde); - throw pde; - } - } - - private MongoMessage newInstance(OperationCode opCode) { - switch (opCode) { - case OP_DELETE: - return new DeleteMessage(); - case OP_GET_MORE: - return new GetMoreMessage(); - case OP_INSERT: - return new InsertMessage(); - case OP_KILL_CURSORS: - return new KillCursorsMessage(); - case OP_QUERY: - return new QueryMessage(); - case OP_REPLY: - return new ReplyMessage(); - case OP_UPDATE: - return new UpdateMessage(); - case OP_MSG: - case RESERVED: - default: - return null; - } - } - - public boolean prefixedDataAvailable(IoBuffer buffer, int prefixLength, boolean includePrefix) { - if (buffer.remaining() < prefixLength) { - return false; - } - - int dataLength; - switch (prefixLength) { - case 1: - dataLength = buffer.getUnsigned(buffer.position()); - break; - case 2: - dataLength = buffer.getUnsignedShort(buffer.position()); - break; - case 4: - dataLength = buffer.getInt(buffer.position()); - break; - default: - throw new IllegalArgumentException("prefixLength: " + prefixLength); - } - - if (dataLength < 0) { - throw new BufferDataException("dataLength: " + dataLength); - } - - if (includePrefix) { - return buffer.remaining() >= dataLength; - } else { - return buffer.remaining() - prefixLength >= dataLength; - } - } +import org.apache.mina.filter.codec.CumulativeProtocolDecoder; +public class MongoProtocolDecoder extends CumulativeProtocolDecoder +{ + private static final Logger LOGGER; + + protected boolean doDecode(final IoSession session, final IoBuffer in, final ProtocolDecoderOutput out) throws Exception { + try { + if (!this.prefixedDataAvailable(in, 4, true)) { + return false; + } + final int start = in.position(); + final int limit = in.limit(); + final int messageSize = in.getInt(start); + in.limit(start + messageSize); + try { + final IoBuffer buf = in.slice(); + final int opCodeValue = buf.getInt(buf.position() + 12); + final OperationCode opCode = OperationCode.valueOf(opCodeValue); + final MongoMessage message = this.newInstance(opCode); + final BSONInputStream input = new BSONInputStream(buf); + message.read(input); + out.write((Object)message); + return true; + } + finally { + in.position(start + messageSize); + in.limit(limit); + } + } + catch (Exception e) { + final ProtocolDecoderException pde = new ProtocolDecoderException("error decode message:", (Throwable)e); + pde.setHexdump(in.getHexDump()); + MongoProtocolDecoder.LOGGER.error("", (Throwable)pde); + throw pde; + } + } + + private MongoMessage newInstance(final OperationCode opCode) { + switch (opCode) { + case OP_DELETE: + return new DeleteMessage(); + case OP_GET_MORE: + return new GetMoreMessage(); + case OP_INSERT: + return new InsertMessage(); + case OP_KILL_CURSORS: + return new KillCursorsMessage(); + case OP_QUERY: + return new QueryMessage(); + case OP_REPLY: + return new ReplyMessage(); + case OP_UPDATE: + return new UpdateMessage(); + default: + return null; + } + } + + public boolean prefixedDataAvailable(final IoBuffer buffer, final int prefixLength, final boolean includePrefix) { + if (buffer.remaining() < prefixLength) { + return false; + } + int dataLength = 0; + switch (prefixLength) { + case 1: + dataLength = buffer.getUnsigned(buffer.position()); + break; + case 2: + dataLength = buffer.getUnsignedShort(buffer.position()); + break; + case 4: + dataLength = buffer.getInt(buffer.position()); + break; + default: + throw new IllegalArgumentException("prefixLength: " + prefixLength); + } + if (dataLength < 0) { + throw new BufferDataException("dataLength: " + dataLength); + } + if (includePrefix) { + return buffer.remaining() >= dataLength; + } + return buffer.remaining() - prefixLength >= dataLength; + } + + static { + LOGGER = LoggerFactory.getLogger((Class)MongoProtocolDecoder.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolEncoder.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolEncoder.java index 786e1043a..0973fdb0d 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolEncoder.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/codec/MongoProtocolEncoder.java @@ -1,36 +1,38 @@ package com.fangjia.datastore.mongo.codec; -import org.apache.mina.core.buffer.IoBuffer; -import org.apache.mina.core.session.IoSession; -import org.apache.mina.filter.codec.ProtocolEncoder; -import org.apache.mina.filter.codec.ProtocolEncoderException; -import org.apache.mina.filter.codec.ProtocolEncoderOutput; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; - +import org.apache.mina.filter.codec.ProtocolEncoderException; +import org.apache.mina.core.buffer.IoBuffer; import com.fangjia.datastore.mongo.protocol.MongoMessage; +import org.apache.mina.filter.codec.ProtocolEncoderOutput; +import org.apache.mina.core.session.IoSession; +import org.slf4j.Logger; +import org.apache.mina.filter.codec.ProtocolEncoder; -public class MongoProtocolEncoder implements ProtocolEncoder { - - private final static Logger LOGGER = LoggerFactory.getLogger(MongoProtocolEncoder.class); - - public void encode(IoSession session, Object message, ProtocolEncoderOutput out) throws Exception { - try { - if (message instanceof MongoMessage) { - MongoMessage mongoMessage = (MongoMessage) message; - BSONOutputStream output = new BSONOutputStream(mongoMessage.getMessageHeader().getMessageLength()); - mongoMessage.write(output); - out.write(IoBuffer.wrap(output.toByteArray())); - } - } catch (Exception e) { - ProtocolEncoderException pe = new ProtocolEncoderException("error encode message: " + message, e); - LOGGER.error("", pe); - throw pe; - } - } - - public void dispose(IoSession session) throws Exception { - // empty - } - +public class MongoProtocolEncoder implements ProtocolEncoder +{ + private static final Logger LOGGER; + + public void encode(final IoSession session, final Object message, final ProtocolEncoderOutput out) throws Exception { + try { + if (message instanceof MongoMessage) { + final MongoMessage mongoMessage = (MongoMessage)message; + final BSONOutputStream output = new BSONOutputStream(mongoMessage.getMessageHeader().getMessageLength()); + mongoMessage.write(output); + out.write((Object)IoBuffer.wrap(output.toByteArray())); + } + } + catch (Exception e) { + final ProtocolEncoderException pe = new ProtocolEncoderException("error encode message: " + message, (Throwable)e); + MongoProtocolEncoder.LOGGER.error("", (Throwable)pe); + throw pe; + } + } + + public void dispose(final IoSession session) throws Exception { + } + + static { + LOGGER = LoggerFactory.getLogger((Class)MongoProtocolEncoder.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/DeleteMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/DeleteMessage.java index e421cd55b..43b369f75 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/DeleteMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/DeleteMessage.java @@ -1,111 +1,98 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; - -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; -import org.bson.BasicBSONObject; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; import com.fangjia.datastore.utils.BSONUtils; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BasicBSONObject; +import org.bson.BSONObject; -public class DeleteMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private String fullCollectionName; - private int flags; - private BSONObject selector; - - public DeleteMessage() { - messageHeader = new MessageHeader(OperationCode.OP_DELETE); - selector = new BasicBSONObject(); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeCString(this.fullCollectionName); - out.writeInteger(this.flags); - BSONEncoder encoder = new BSONEncoder(); - out.write(encoder.encode(selector)); - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - this.fullCollectionName = in.readCString(); - this.flags = in.readInteger(); - BSONDecoder decoder = new BasicBSONDecoder(); - selector = decoder.readObject(in); - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public int getFlags() { - return flags; - } - - public void setFlags(int flags) { - this.flags = flags; - } - - public void setSingleRemove(boolean b) { - this.flags = (int) BSONUtils.bitSet(this.flags, 0, b); - } - - public BSONObject getSelector() { - return selector; - } - - public void setSelector(BSONObject selector) { - this.selector = selector; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", flags:").append(flags); - builder.append(", selector:").append(selector); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class DeleteMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private String fullCollectionName; + private int flags; + private BSONObject selector; + + public DeleteMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_DELETE); + this.selector = (BSONObject)new BasicBSONObject(); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeCString(this.fullCollectionName); + out.writeInteger(this.flags); + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + out.write(encoder.encode(this.selector)); + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.fullCollectionName = in.readCString(); + this.flags = in.readInteger(); + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + this.selector = decoder.readObject((InputStream)in); + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public int getFlags() { + return this.flags; + } + + public void setFlags(final int flags) { + this.flags = flags; + } + + public void setSingleRemove(final boolean b) { + this.flags = (int)BSONUtils.bitSet(this.flags, 0, b); + } + + public BSONObject getSelector() { + return this.selector; + } + + public void setSelector(final BSONObject selector) { + this.selector = selector; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", flags:").append(this.flags); + builder.append(", selector:").append(this.selector); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/GetMoreMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/GetMoreMessage.java index dff96af93..8d44fe088 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/GetMoreMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/GetMoreMessage.java @@ -1,97 +1,83 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; - import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; import com.fangjia.datastore.mongo.codec.BSONOutputStream; -public class GetMoreMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private String fullCollectionName; - private int numberToReturn; - private long cursorID; - - public GetMoreMessage() { - messageHeader = new MessageHeader(OperationCode.OP_GET_MORE); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeCString(this.fullCollectionName); - out.writeInteger(this.numberToReturn); - out.writeLong(cursorID); - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - this.fullCollectionName = in.readCString(); - this.numberToReturn = in.readInteger(); - this.cursorID = in.readLong(); - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public int getNumberToReturn() { - return numberToReturn; - } - - public void setNumberToReturn(int numberToReturn) { - this.numberToReturn = numberToReturn; - } - - public long getCursorID() { - return cursorID; - } - - public void setCursorID(long cursorID) { - this.cursorID = cursorID; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", numberToReturn:").append(numberToReturn); - builder.append(", cursorID:").append(cursorID); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class GetMoreMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private String fullCollectionName; + private int numberToReturn; + private long cursorID; + + public GetMoreMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_GET_MORE); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeCString(this.fullCollectionName); + out.writeInteger(this.numberToReturn); + out.writeLong(this.cursorID); + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.fullCollectionName = in.readCString(); + this.numberToReturn = in.readInteger(); + this.cursorID = in.readLong(); + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public int getNumberToReturn() { + return this.numberToReturn; + } + + public void setNumberToReturn(final int numberToReturn) { + this.numberToReturn = numberToReturn; + } + + public long getCursorID() { + return this.cursorID; + } + + public void setCursorID(final long cursorID) { + this.cursorID = cursorID; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", numberToReturn:").append(this.numberToReturn); + builder.append(", cursorID:").append(this.cursorID); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/InsertMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/InsertMessage.java index 9c9093e88..ce30fc4a6 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/InsertMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/InsertMessage.java @@ -1,116 +1,105 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; import java.util.ArrayList; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import java.util.Iterator; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BSONObject; import java.util.List; -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; - -public class InsertMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private String fullCollectionName; - private List documents; - - public InsertMessage() { - messageHeader = new MessageHeader(OperationCode.OP_INSERT); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeCString(this.fullCollectionName); - if (null != documents && documents.size() > 0) { - BSONEncoder encoder = new BSONEncoder(); - for (BSONObject doc : documents) { - out.write(encoder.encode(doc)); - } - } - out.setInteger(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - this.fullCollectionName = in.readCString(); - while (in.available() > 0) { - if (null == documents) { - documents = new ArrayList(); - } - BSONDecoder decoder = new BasicBSONDecoder(); - documents.add(decoder.readObject(in)); - } - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public List getDocuments() { - return documents; - } - - public void setDocuments(List documents) { - this.documents = documents; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", documents:").append(documents); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - if (null != documents && documents.size() > 0) { - builder.append(", documents:").append(documents.get(0).get("_id")); - if (documents.size() > 1) { - builder.append("...").append(documents.size() - 1).append(" more"); - } - } else { - builder.append(", documents:").append(documents); - } - builder.append("}"); - return builder.toString(); - } - +public class InsertMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private String fullCollectionName; + private List documents; + + public InsertMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_INSERT); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeCString(this.fullCollectionName); + if (null != this.documents && this.documents.size() > 0) { + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + for (final BSONObject doc : this.documents) { + out.write(encoder.encode(doc)); + } + } + out.setInteger(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.fullCollectionName = in.readCString(); + while (in.available() > 0) { + if (null == this.documents) { + this.documents = new ArrayList(); + } + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + this.documents.add(decoder.readObject((InputStream)in)); + } + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public List getDocuments() { + return this.documents; + } + + public void setDocuments(final List documents) { + this.documents = documents; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", documents:").append(this.documents); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + if (null != this.documents && this.documents.size() > 0) { + builder.append(", documents:").append(this.documents.get(0).get("_id")); + if (this.documents.size() > 1) { + builder.append("...").append(this.documents.size() - 1).append(" more"); + } + } + else { + builder.append(", documents:").append(this.documents); + } + builder.append("}"); + return builder.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/KillCursorsMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/KillCursorsMessage.java index ee5f41a8d..3a36d8d4d 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/KillCursorsMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/KillCursorsMessage.java @@ -1,96 +1,83 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; import java.util.ArrayList; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import java.util.Iterator; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; import java.util.List; -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; - -public class KillCursorsMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private int numberOfCursorIDs; - private List cursorIDs; - - public KillCursorsMessage() { - messageHeader = new MessageHeader(OperationCode.OP_KILL_CURSORS); - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeInteger(numberOfCursorIDs); - if (null != cursorIDs) { - for (Long id : cursorIDs) { - out.writeLong(id); - } - } - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - numberOfCursorIDs = in.readInteger(); - while (in.available() > 0) { - if (null == cursorIDs) { - cursorIDs = new ArrayList(); - } - cursorIDs.add(in.readLong()); - } - } - - public int getNumberOfCursorIDs() { - return numberOfCursorIDs; - } - - public void setNumberOfCursorIDs(int numberOfCursorIDs) { - this.numberOfCursorIDs = numberOfCursorIDs; - } - - public List getCursorIDs() { - return cursorIDs; - } - - public void setCursorIDs(List cursorIDs) { - this.cursorIDs = cursorIDs; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", numberOfCursorIDs:").append(numberOfCursorIDs); - builder.append(", cursorIDs:").append(cursorIDs); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class KillCursorsMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private int numberOfCursorIDs; + private List cursorIDs; + + public KillCursorsMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_KILL_CURSORS); + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeInteger(this.numberOfCursorIDs); + if (null != this.cursorIDs) { + for (final Long id : this.cursorIDs) { + out.writeLong(id); + } + } + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.numberOfCursorIDs = in.readInteger(); + while (in.available() > 0) { + if (null == this.cursorIDs) { + this.cursorIDs = new ArrayList(); + } + this.cursorIDs.add(in.readLong()); + } + } + + public int getNumberOfCursorIDs() { + return this.numberOfCursorIDs; + } + + public void setNumberOfCursorIDs(final int numberOfCursorIDs) { + this.numberOfCursorIDs = numberOfCursorIDs; + } + + public List getCursorIDs() { + return this.cursorIDs; + } + + public void setCursorIDs(final List cursorIDs) { + this.cursorIDs = cursorIDs; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", numberOfCursorIDs:").append(this.numberOfCursorIDs); + builder.append(", cursorIDs:").append(this.cursorIDs); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MessageHeader.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MessageHeader.java index 9430053af..97fa2c238 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MessageHeader.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MessageHeader.java @@ -1,106 +1,93 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; -import java.io.Serializable; -import java.util.concurrent.atomic.AtomicInteger; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import java.io.IOException; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.util.concurrent.atomic.AtomicInteger; +import java.io.Serializable; -public final class MessageHeader implements Writable, Readable, Serializable { - private static final long serialVersionUID = 8505270142383450097L; - private static final AtomicInteger ID_GENERATOR = new AtomicInteger(1); - - private int messageLength; - private int requestID; - private int responseTo; - private OperationCode opCode; - - public MessageHeader(OperationCode opCode) { - this.opCode = opCode; - this.requestID = ID_GENERATOR.getAndIncrement(); - } - - public MessageHeader() { - } - - public void read(BSONInputStream in) throws IOException { - messageLength = in.readInteger(); - requestID = in.readInteger(); - responseTo = in.readInteger(); - opCode = OperationCode.valueOf(in.readInteger()); - } - - public void write(BSONOutputStream out) throws IOException { - out.writeInteger(messageLength); - out.writeInteger(requestID); - out.writeInteger(responseTo); - out.writeInteger(opCode.getValue()); - } - - public int size() { - return 16; - } - - public int getMessageLength() { - return messageLength; - } - - public void setMessageLength(int messageLength) { - this.messageLength = messageLength; - } - - public int getRequestID() { - return requestID; - } - - public void setRequestID(int requestID) { - this.requestID = requestID; - } - - public int getResponseTo() { - return responseTo; - } - - public void setResponseTo(int responseTo) { - this.responseTo = responseTo; - } - - public OperationCode getOpCode() { - return opCode; - } - - public void setOpCode(OperationCode opCode) { - this.opCode = opCode; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageLength:").append(messageLength); - builder.append(", requestID:").append(requestID); - builder.append(", responseTo:").append(responseTo); - builder.append(", opCode:").append(opCode); - builder.append("}"); - return builder.toString(); - } - +public final class MessageHeader implements Writable, Readable, Serializable +{ + private static final long serialVersionUID = 8505270142383450097L; + private static final AtomicInteger ID_GENERATOR; + private int messageLength; + private int requestID; + private int responseTo; + private OperationCode opCode; + + public MessageHeader(final OperationCode opCode) { + this.opCode = opCode; + this.requestID = MessageHeader.ID_GENERATOR.getAndIncrement(); + } + + public MessageHeader() { + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageLength = in.readInteger(); + this.requestID = in.readInteger(); + this.responseTo = in.readInteger(); + this.opCode = OperationCode.valueOf(in.readInteger()); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + out.writeInteger(this.messageLength); + out.writeInteger(this.requestID); + out.writeInteger(this.responseTo); + out.writeInteger(this.opCode.getValue()); + } + + public int size() { + return 16; + } + + public int getMessageLength() { + return this.messageLength; + } + + public void setMessageLength(final int messageLength) { + this.messageLength = messageLength; + } + + public int getRequestID() { + return this.requestID; + } + + public void setRequestID(final int requestID) { + this.requestID = requestID; + } + + public int getResponseTo() { + return this.responseTo; + } + + public void setResponseTo(final int responseTo) { + this.responseTo = responseTo; + } + + public OperationCode getOpCode() { + return this.opCode; + } + + public void setOpCode(final OperationCode opCode) { + this.opCode = opCode; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageLength:").append(this.messageLength); + builder.append(", requestID:").append(this.requestID); + builder.append(", responseTo:").append(this.responseTo); + builder.append(", opCode:").append(this.opCode); + builder.append("}"); + return builder.toString(); + } + + static { + ID_GENERATOR = new AtomicInteger(1); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MongoMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MongoMessage.java index 35f01104a..ea571f11c 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MongoMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/MongoMessage.java @@ -1,26 +1,10 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; import java.io.Serializable; -public interface MongoMessage extends Readable, Writable, Serializable { - public MessageHeader getMessageHeader(); - - public String toPrettyString(); +public interface MongoMessage extends Readable, Writable, Serializable +{ + MessageHeader getMessageHeader(); + + String toPrettyString(); } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/OperationCode.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/OperationCode.java index 0ceb27e6d..cf7fbde2b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/OperationCode.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/OperationCode.java @@ -1,82 +1,39 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; import java.util.HashMap; import java.util.Map; -public enum OperationCode { - /** - * Reply to a client request. responseTo is set - */ - OP_REPLY(1), - /** - * generic msg command followed by a string - */ - OP_MSG(1000), - /** - * update document - */ - OP_UPDATE(2001), - /** - * insert new document - */ - OP_INSERT(2002), - /** - * formerly used for OP_GET_BY_OID - */ - RESERVED(2003), - /** - * query a collection - */ - OP_QUERY(2004), - /** - * Get more data from a query. See Cursors - */ - OP_GET_MORE(2005), - /** - * Delete documents - */ - OP_DELETE(2006), - /** - * Tell database client is done with a cursor - */ - OP_KILL_CURSORS(2007); - - private final int value; - - private final static Map TYPE_MAP = new HashMap(); - static { - for (OperationCode e : OperationCode.values()) { - TYPE_MAP.put(e.value, e); - } - } - - private OperationCode(int value) { - this.value = value; - } - - public int getValue() { - return value; - } - - public static OperationCode valueOf(int value) { - return TYPE_MAP.get(value); - } - +public enum OperationCode +{ + OP_REPLY(1), + OP_MSG(1000), + OP_UPDATE(2001), + OP_INSERT(2002), + RESERVED(2003), + OP_QUERY(2004), + OP_GET_MORE(2005), + OP_DELETE(2006), + OP_KILL_CURSORS(2007); + + private final int value; + private static final Map TYPE_MAP; + + private OperationCode(final int value) { + this.value = value; + } + + public int getValue() { + return this.value; + } + + public static OperationCode valueOf(final int value) { + return OperationCode.TYPE_MAP.get(value); + } + + static { + TYPE_MAP = new HashMap(); + for (final OperationCode e : values()) { + OperationCode.TYPE_MAP.put(e.value, e); + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/QueryMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/QueryMessage.java index e82245fa0..ddf724569 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/QueryMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/QueryMessage.java @@ -1,213 +1,197 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; - -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; import com.fangjia.datastore.utils.BSONUtils; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BSONObject; -public class QueryMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private int options; - private int numberToSkip; - private int numberToReturn; - private String fullCollectionName; - private BSONObject query; - private BSONObject returnFieldSelector; - - public QueryMessage() { - messageHeader = new MessageHeader(OperationCode.OP_QUERY); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(options); - out.writeCString(this.fullCollectionName); - out.writeInteger(numberToSkip); - out.writeInteger(numberToReturn); - BSONEncoder encoder = new BSONEncoder(); - out.write(encoder.encode(query)); - if (null != returnFieldSelector) { - out.write(encoder.encode(returnFieldSelector)); - } - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - options = in.readInteger(); - this.fullCollectionName = in.readCString(); - this.numberToSkip = in.readInteger(); - this.numberToReturn = in.readInteger(); - BSONDecoder decoder = new BasicBSONDecoder(); - query = decoder.readObject(in); - if (in.available() > 0) { - returnFieldSelector = decoder.readObject(in); - } - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public int getOptions() { - return options; - } - - public void setOptions(int options) { - this.options = options; - } - - public void setNoCursorTimeout(boolean b) { - this.options = (int) BSONUtils.bitSet(options, 4, b); - } - - public void setTailable(boolean b) { - this.options = (int) BSONUtils.bitSet(options, 1, b); - } - - public void setSlaveOk(boolean b) { - this.options = (int) BSONUtils.bitSet(options, 2, b); - } - - public void setAwaitData(boolean b) { - this.options = (int) BSONUtils.bitSet(options, 5, b); - } - - public int getNumberToSkip() { - return numberToSkip; - } - - public void setNumberToSkip(int numberToSkip) { - this.numberToSkip = numberToSkip; - } - - public int getNumberToReturn() { - return numberToReturn; - } - - public void setNumberToReturn(int numberToReturn) { - this.numberToReturn = numberToReturn; - } - - public BSONObject getQuery() { - return query; - } - - public void setQuery(BSONObject query) { - this.query = query; - } - - public BSONObject getRealQuery() { - if (null == query) { - return null; - } - BSONObject q = (BSONObject) query.get("$query"); - if (null == q) { - q = (BSONObject) query.get("query"); - if (null == q) { - q = query; - } - } - return q; - } - - public BSONObject getOrderByQuery() { - if (null == query) { - return null; - } - BSONObject q = (BSONObject) query.get("$orderby"); - if (null == q) { - q = (BSONObject) query.get("orderby"); - } - return q; - } - - public BSONObject getFacetQuery() { - BSONObject q = getRealQuery(); - if (null == q) { - return null; - } - return (BSONObject) q.get("$facet"); - } - - public boolean isBulkQuery() { - BSONObject q = getRealQuery(); - if (null == q) { - return true; - } - Boolean bulk = (Boolean) q.get("$bulk"); - - return null == bulk ? true : bulk; - } - - public boolean isFromMaster() { - BSONObject q = getRealQuery(); - if (null == q) { - return false; - } - Boolean master = (Boolean) q.get("$master"); - - return null == master ? false : master; - } - - public BSONObject getReturnFieldSelector() { - return returnFieldSelector; - } - - public void setReturnFieldSelector(BSONObject returnFieldSelector) { - this.returnFieldSelector = returnFieldSelector; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", options:").append(options); - builder.append(", numberToSkip:").append(numberToSkip); - builder.append(", numberToReturn:").append(numberToReturn); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", query:").append(query); - builder.append(", returnFieldSelector:").append(returnFieldSelector); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class QueryMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private int options; + private int numberToSkip; + private int numberToReturn; + private String fullCollectionName; + private BSONObject query; + private BSONObject returnFieldSelector; + + public QueryMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_QUERY); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(this.options); + out.writeCString(this.fullCollectionName); + out.writeInteger(this.numberToSkip); + out.writeInteger(this.numberToReturn); + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + out.write(encoder.encode(this.query)); + if (null != this.returnFieldSelector) { + out.write(encoder.encode(this.returnFieldSelector)); + } + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + this.options = in.readInteger(); + this.fullCollectionName = in.readCString(); + this.numberToSkip = in.readInteger(); + this.numberToReturn = in.readInteger(); + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + this.query = decoder.readObject((InputStream)in); + if (in.available() > 0) { + this.returnFieldSelector = decoder.readObject((InputStream)in); + } + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public int getOptions() { + return this.options; + } + + public void setOptions(final int options) { + this.options = options; + } + + public void setNoCursorTimeout(final boolean b) { + this.options = (int)BSONUtils.bitSet(this.options, 4, b); + } + + public void setTailable(final boolean b) { + this.options = (int)BSONUtils.bitSet(this.options, 1, b); + } + + public void setSlaveOk(final boolean b) { + this.options = (int)BSONUtils.bitSet(this.options, 2, b); + } + + public void setAwaitData(final boolean b) { + this.options = (int)BSONUtils.bitSet(this.options, 5, b); + } + + public int getNumberToSkip() { + return this.numberToSkip; + } + + public void setNumberToSkip(final int numberToSkip) { + this.numberToSkip = numberToSkip; + } + + public int getNumberToReturn() { + return this.numberToReturn; + } + + public void setNumberToReturn(final int numberToReturn) { + this.numberToReturn = numberToReturn; + } + + public BSONObject getQuery() { + return this.query; + } + + public void setQuery(final BSONObject query) { + this.query = query; + } + + public BSONObject getRealQuery() { + if (null == this.query) { + return null; + } + BSONObject q = (BSONObject)this.query.get("$query"); + if (null == q) { + q = (BSONObject)this.query.get("query"); + if (null == q) { + q = this.query; + } + } + return q; + } + + public BSONObject getOrderByQuery() { + if (null == this.query) { + return null; + } + BSONObject q = (BSONObject)this.query.get("$orderby"); + if (null == q) { + q = (BSONObject)this.query.get("orderby"); + } + return q; + } + + public BSONObject getFacetQuery() { + final BSONObject q = this.getRealQuery(); + if (null == q) { + return null; + } + return (BSONObject)q.get("$facet"); + } + + public boolean isBulkQuery() { + final BSONObject q = this.getRealQuery(); + if (null == q) { + return true; + } + final Boolean bulk = (Boolean)q.get("$bulk"); + return null == bulk || bulk; + } + + public boolean isFromMaster() { + final BSONObject q = this.getRealQuery(); + if (null == q) { + return false; + } + final Boolean master = (Boolean)q.get("$master"); + return null != master && master; + } + + public BSONObject getReturnFieldSelector() { + return this.returnFieldSelector; + } + + public void setReturnFieldSelector(final BSONObject returnFieldSelector) { + this.returnFieldSelector = returnFieldSelector; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", options:").append(this.options); + builder.append(", numberToSkip:").append(this.numberToSkip); + builder.append(", numberToReturn:").append(this.numberToReturn); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", query:").append(this.query); + builder.append(", returnFieldSelector:").append(this.returnFieldSelector); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Readable.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Readable.java index 0aa14d6c1..cb0dbacf3 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Readable.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Readable.java @@ -1,26 +1,9 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; import java.io.IOException; - import com.fangjia.datastore.mongo.codec.BSONInputStream; -public interface Readable { - public void read(BSONInputStream in) throws IOException; +public interface Readable +{ + void read(final BSONInputStream p0) throws IOException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/ReplyMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/ReplyMessage.java index 806849823..d133315a3 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/ReplyMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/ReplyMessage.java @@ -1,138 +1,125 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; import java.util.ArrayList; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import java.util.Iterator; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BSONObject; import java.util.List; -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; - -public class ReplyMessage implements MongoMessage { - - private static final long serialVersionUID = -3350216587439425208L; - private MessageHeader messageHeader; - private int responseFlag; - private long cursorID; - private int startingFrom; - private int numberReturned; - private List documents; - - public ReplyMessage() { - messageHeader = new MessageHeader(OperationCode.OP_REPLY); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(responseFlag); - out.writeLong(cursorID); - out.writeInteger(startingFrom); - out.writeInteger(numberReturned); - if (null != this.documents) { - BSONEncoder encoder = new BSONEncoder(); - for (BSONObject doc : this.documents) { - out.write(encoder.encode(doc)); - } - } - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - responseFlag = in.readInteger(); - cursorID = in.readLong(); - startingFrom = in.readInteger(); - numberReturned = in.readInteger(); - documents = new ArrayList(numberReturned); - if (numberReturned > 0) { - BSONDecoder decoder = new BasicBSONDecoder(); - for (int i = 0; i < numberReturned; i++) { - documents.add(decoder.readObject(in)); - } - } - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public int getResponseFlag() { - return responseFlag; - } - - public void setResponseFlag(int responseFlag) { - this.responseFlag = responseFlag; - } - - public long getCursorID() { - return cursorID; - } - - public void setCursorID(long cursorID) { - this.cursorID = cursorID; - } - - public int getStartingFrom() { - return startingFrom; - } - - public void setStartingFrom(int startingFrom) { - this.startingFrom = startingFrom; - } - - public int getNumberReturned() { - return numberReturned; - } - - public void setNumberReturned(int numberReturned) { - this.numberReturned = numberReturned; - } - - public List getDocuments() { - return documents; - } - - public void setDocuments(List documents) { - this.documents = documents; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", responseFlag:").append(responseFlag); - builder.append(", cursorID:").append(cursorID); - builder.append(", startingFrom:").append(startingFrom); - builder.append(", numberReturned:").append(numberReturned); - builder.append(", documents:").append(documents); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class ReplyMessage implements MongoMessage +{ + private static final long serialVersionUID = -3350216587439425208L; + private MessageHeader messageHeader; + private int responseFlag; + private long cursorID; + private int startingFrom; + private int numberReturned; + private List documents; + + public ReplyMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_REPLY); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(this.responseFlag); + out.writeLong(this.cursorID); + out.writeInteger(this.startingFrom); + out.writeInteger(this.numberReturned); + if (null != this.documents) { + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + for (final BSONObject doc : this.documents) { + out.write(encoder.encode(doc)); + } + } + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + this.responseFlag = in.readInteger(); + this.cursorID = in.readLong(); + this.startingFrom = in.readInteger(); + this.numberReturned = in.readInteger(); + this.documents = new ArrayList(this.numberReturned); + if (this.numberReturned > 0) { + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + for (int i = 0; i < this.numberReturned; ++i) { + this.documents.add(decoder.readObject((InputStream)in)); + } + } + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public int getResponseFlag() { + return this.responseFlag; + } + + public void setResponseFlag(final int responseFlag) { + this.responseFlag = responseFlag; + } + + public long getCursorID() { + return this.cursorID; + } + + public void setCursorID(final long cursorID) { + this.cursorID = cursorID; + } + + public int getStartingFrom() { + return this.startingFrom; + } + + public void setStartingFrom(final int startingFrom) { + this.startingFrom = startingFrom; + } + + public int getNumberReturned() { + return this.numberReturned; + } + + public void setNumberReturned(final int numberReturned) { + this.numberReturned = numberReturned; + } + + public List getDocuments() { + return this.documents; + } + + public void setDocuments(final List documents) { + this.documents = documents; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", responseFlag:").append(this.responseFlag); + builder.append(", cursorID:").append(this.cursorID); + builder.append(", startingFrom:").append(this.startingFrom); + builder.append(", numberReturned:").append(this.numberReturned); + builder.append(", documents:").append(this.documents); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/UpdateMessage.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/UpdateMessage.java index 536d498fd..33bd4fdbd 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/UpdateMessage.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/UpdateMessage.java @@ -1,137 +1,123 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; -import java.io.IOException; - -import org.bson.BSONDecoder; -import org.bson.BSONEncoder; -import org.bson.BSONObject; -import org.bson.BasicBSONDecoder; -import org.bson.BasicBSONObject; - -import com.fangjia.datastore.mongo.codec.BSONInputStream; -import com.fangjia.datastore.mongo.codec.BSONOutputStream; import com.fangjia.datastore.utils.BSONUtils; +import org.bson.BSONDecoder; +import java.io.InputStream; +import org.bson.BasicBSONDecoder; +import com.fangjia.datastore.mongo.codec.BSONInputStream; +import java.io.IOException; +import org.bson.BSONEncoder; +import org.bson.BasicBSONEncoder; +import com.fangjia.datastore.mongo.codec.BSONOutputStream; +import org.bson.BasicBSONObject; +import org.bson.BSONObject; -public class UpdateMessage implements MongoMessage { - - private static final long serialVersionUID = -5278293474222136247L; - - private MessageHeader messageHeader; - private String fullCollectionName; - private int flags; - private BSONObject selector; - private BSONObject update; - - public UpdateMessage() { - messageHeader = new MessageHeader(OperationCode.OP_UPDATE); - this.selector = new BasicBSONObject(); - this.update = new BasicBSONObject(); - } - - public void write(BSONOutputStream out) throws IOException { - messageHeader.write(out); - out.writeInteger(0); - out.writeCString(this.fullCollectionName); - out.writeInteger(flags); - BSONEncoder encoder = new BSONEncoder(); - out.write(encoder.encode(selector)); - out.write(encoder.encode(update)); - out.set(0, out.size()); - } - - public void read(BSONInputStream in) throws IOException { - messageHeader.read(in); - in.readInteger(); - this.fullCollectionName = in.readCString(); - this.flags = in.readInteger(); - BSONDecoder decoder = new BasicBSONDecoder(); - selector = decoder.readObject(in); - update = decoder.readObject(in); - } - - public void setUpsert(boolean upsert) { - this.flags = (int) BSONUtils.bitSet(this.flags, 0, upsert); - } - - public boolean isUpsert() { - return BSONUtils.isBitSet(flags, 0); - } - - public void setMultiUpdate(boolean multiUpdate) { - this.flags = (int) BSONUtils.bitSet(this.flags, 1, multiUpdate); - } - - public boolean isMultiUpdate() { - return BSONUtils.isBitSet(flags, 1); - } - - public MessageHeader getMessageHeader() { - return messageHeader; - } - - public String getFullCollectionName() { - return fullCollectionName; - } - - public void setFullCollectionName(String fullCollectionName) { - this.fullCollectionName = fullCollectionName; - } - - public int getFlags() { - return flags; - } - - public void setFlags(int flags) { - this.flags = flags; - } - - public BSONObject getSelector() { - return selector; - } - - public void setSelector(BSONObject selector) { - this.selector = selector; - } - - public BSONObject getUpdate() { - return update; - } - - public void setUpdate(BSONObject update) { - this.update = update; - } - - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("{"); - builder.append("messageHeader:").append(messageHeader); - builder.append(", fullCollectionName:").append(fullCollectionName); - builder.append(", flags:").append(flags); - builder.append(", selector:").append(selector); - builder.append(", update:").append(update); - builder.append("}"); - return builder.toString(); - } - - public String toPrettyString() { - return toString(); - } - +public class UpdateMessage implements MongoMessage +{ + private static final long serialVersionUID = -5278293474222136247L; + private MessageHeader messageHeader; + private String fullCollectionName; + private int flags; + private BSONObject selector; + private BSONObject update; + + public UpdateMessage() { + this.messageHeader = new MessageHeader(OperationCode.OP_UPDATE); + this.selector = (BSONObject)new BasicBSONObject(); + this.update = (BSONObject)new BasicBSONObject(); + } + + @Override + public void write(final BSONOutputStream out) throws IOException { + this.messageHeader.write(out); + out.writeInteger(0); + out.writeCString(this.fullCollectionName); + out.writeInteger(this.flags); + final BSONEncoder encoder = (BSONEncoder)new BasicBSONEncoder(); + out.write(encoder.encode(this.selector)); + out.write(encoder.encode(this.update)); + out.set(0, out.size()); + } + + @Override + public void read(final BSONInputStream in) throws IOException { + this.messageHeader.read(in); + in.readInteger(); + this.fullCollectionName = in.readCString(); + this.flags = in.readInteger(); + final BSONDecoder decoder = (BSONDecoder)new BasicBSONDecoder(); + this.selector = decoder.readObject((InputStream)in); + this.update = decoder.readObject((InputStream)in); + } + + public void setUpsert(final boolean upsert) { + this.flags = (int)BSONUtils.bitSet(this.flags, 0, upsert); + } + + public boolean isUpsert() { + return BSONUtils.isBitSet(this.flags, 0); + } + + public void setMultiUpdate(final boolean multiUpdate) { + this.flags = (int)BSONUtils.bitSet(this.flags, 1, multiUpdate); + } + + public boolean isMultiUpdate() { + return BSONUtils.isBitSet(this.flags, 1); + } + + @Override + public MessageHeader getMessageHeader() { + return this.messageHeader; + } + + public String getFullCollectionName() { + return this.fullCollectionName; + } + + public void setFullCollectionName(final String fullCollectionName) { + this.fullCollectionName = fullCollectionName; + } + + public int getFlags() { + return this.flags; + } + + public void setFlags(final int flags) { + this.flags = flags; + } + + public BSONObject getSelector() { + return this.selector; + } + + public void setSelector(final BSONObject selector) { + this.selector = selector; + } + + public BSONObject getUpdate() { + return this.update; + } + + public void setUpdate(final BSONObject update) { + this.update = update; + } + + @Override + public String toString() { + final StringBuilder builder = new StringBuilder(); + builder.append("{"); + builder.append("messageHeader:").append(this.messageHeader); + builder.append(", fullCollectionName:").append(this.fullCollectionName); + builder.append(", flags:").append(this.flags); + builder.append(", selector:").append(this.selector); + builder.append(", update:").append(this.update); + builder.append("}"); + return builder.toString(); + } + + @Override + public String toPrettyString() { + return this.toString(); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Writable.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Writable.java index 0b43ef8ba..aa55f121b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Writable.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/mongo/protocol/Writable.java @@ -1,26 +1,9 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.mongo.protocol; import java.io.IOException; - import com.fangjia.datastore.mongo.codec.BSONOutputStream; -public interface Writable { - public void write(BSONOutputStream out) throws IOException; +public interface Writable +{ + void write(final BSONOutputStream p0) throws IOException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CommonsHttpSolrServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CommonsHttpSolrServer.java index 3c92e83bc..369c8af94 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CommonsHttpSolrServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CommonsHttpSolrServer.java @@ -1,11 +1,7 @@ package com.fangjia.datastore.solr; -import java.io.IOException; -import java.net.MalformedURLException; - import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.solr.client.solrj.SolrRequest; @@ -15,66 +11,61 @@ import org.apache.solr.common.util.NamedList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class CommonsHttpSolrServer extends HttpSolrServer { - private static final long serialVersionUID = -3881911436089005421L; +import java.io.IOException; +import java.net.MalformedURLException; - private final static Logger LOGGER = LoggerFactory.getLogger(CommonsHttpSolrServer.class); +public class CommonsHttpSolrServer extends HttpSolrServer +{ + private static final long serialVersionUID = -3881911436089005421L; + private static final Logger LOGGER; + private final HttpSolrClient solrServer; + private final String host; + private final String coreName; + + public CommonsHttpSolrServer(final String host, final String coreName, final HttpClient httpClient) throws MalformedURLException { + this.host = host; + this.coreName = coreName; + (this.solrServer = new HttpSolrClient("http://" + host + '/' + coreName)).setAllowCompression(true); + } + + @Override + public String getHost() { + return this.host; + } + + @Override + public String getCoreName() { + return this.coreName; + } + + @Override + public void reloadCore() throws SolrException { + try { + final HttpGet method = new HttpGet("http://" + this.host + "/admin/cores?action=RELOAD&core=" + this.coreName); + final HttpResponse response = this.solrServer.getHttpClient().execute(method); + if (response.getStatusLine().getStatusCode() != 200) { + throw new SolrException("[" + this.coreName + "] error reload core of solr[" + this.host + "]: " + IOUtils.toString(response.getEntity().getContent())); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info("[" + this.coreName + "] solr core of server[" + this.host + "] reloaded."); + } + if (LOGGER.isInfoEnabled()) { + LOGGER.info("[" + this.coreName + "] core reload success"); + } + } + catch (IOException e) { + throw new SolrException("[" + this.coreName + "] error reload core", e); + } + } - private final HttpSolrClient solrServer; - private final String host; - private final String coreName; + public NamedList request(final SolrRequest request, final String arg1) throws SolrServerException, IOException { + return this.solrServer.request(request); + } - public CommonsHttpSolrServer(String host, String coreName, HttpClient httpClient) throws MalformedURLException { - this.host = host; - this.coreName = coreName; - this.solrServer = new HttpSolrClient("http://" + host + '/' + coreName); - this.solrServer.setAllowCompression(true); - } - - @Override - public String getHost() { - return host; - } - - @Override - public String getCoreName() { - return coreName; - } - - @Override - public void reloadCore() throws SolrException { - try { - HttpGet method = new HttpGet("http://" + host + "/admin/cores?action=RELOAD&core=" + this.coreName); - HttpResponse response = solrServer.getHttpClient().execute(method); - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - throw new SolrException("[" + this.coreName + "] error reload core of solr[" + host + "]: " - + IOUtils.toString(response.getEntity().getContent())); - } else if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.coreName + "] solr core of server[" + host + "] reloaded."); - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.coreName + "] core reload success"); - } - } catch (IOException e) { - throw new SolrException("[" + this.coreName + "] error reload core", e); - } - } - - /*@Override - public NamedList request(SolrRequest request) throws SolrServerException, IOException { - return solrServer.request(request); - - }*/ - - @Override - public NamedList request(SolrRequest request, String arg1) - throws SolrServerException, IOException { - return solrServer.request(request); - } - - @Override - public void shutdown() { - - } + public void shutdown() { + } + static { + LOGGER = LoggerFactory.getLogger(CommonsHttpSolrServer.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Core.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Core.java index 740fe9854..f217c6248 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Core.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Core.java @@ -1,138 +1,32 @@ package com.fangjia.datastore.solr; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; - -import org.apache.commons.lang.time.FastDateFormat; -import org.bson.BSONObject; -import org.bson.types.ObjectId; - import com.fangjia.datastore.QueryResponse; +import org.bson.types.ObjectId; +import java.util.List; +import org.bson.BSONObject; +import java.util.Iterator; +import java.util.Collection; +import org.apache.commons.lang.time.FastDateFormat; import com.fangjia.datastore.config.Reloadable; -/** - * 一个Core对应一个solr索引 - * - * @author velna - * - */ -public interface Core extends Reloadable { - - public final static int INDEX_BATCH_SIZE = 30000; - public final static FastDateFormat VERSION_DATE_FORMAT = FastDateFormat.getInstance("yyMMddHHmmssSSS"); - - /** - * 得到索引名称 - * - * @return - */ - public String getName(); - - /** - * 得到所有索引字段 - * - * @return - * @throws SolrException - */ - Collection getIndexes() throws SolrException; - - /** - * 添加索引字段,必要的话重建索引 - * - * @param indexes - * 添加的索引字段列表 - * @param iterator - * 用于遍历数据 - * @throws SolrException - */ - void ensureIndex(Collection indexes, Iterator iterator) throws SolrException; - - /** - * 添加索引字段,必要的话重建索引 - * - * @param index - * 添加的索引字段列表 - * @param iterator - * 用于遍历数据 - * @throws SolrException - */ - void ensureIndex(Index index, Iterator iterator) throws SolrException; - - /** - * 重建索引 - * - * @param objIdList - * 待重建的ObjIdList,全部重建时为null - * @param iterator - * 用于遍历数据 - */ - void reIndex(List objIdList, Iterator iterator) throws SolrException; - - /** - * 保存索引文档 - * - * @param doc - * @throws SolrException - */ - // void save(BSONObject doc) throws SolrException; - /** - * 保存多个索引文档 - * - * @param docList - * @throws SolrException - */ - // void save(List docList) throws SolrException; - /** - * 保存多个索引文档 - * - * @param iterator - * @throws SolrException - */ - void save(Iterator iterator) throws SolrException; - - /** - * 根据ID删除索引文档 - * - * @param id - * @throws SolrException - */ - // void deleteById(String id) throws SolrException; - /** - * 根据ID列表删除索引文档 - * - * @param ids - * @throws SolrException - */ - // void deleteById(List ids) throws SolrException; - /** - * 根据查询删除索引文档 - * - * @param query - * @throws SolrException - */ - void deleteByQuery(BSONObject query) throws SolrException; - - /** - * 查询索引 - * - * @param query - * 查询条件 - * @param sort - * 排序条件 - * @param facet - * facet查询条件 - * @param skip - * 从第skip条开始返回结果 - * @param limit - * 返回limit条结果 - * @param fromMaster - * 是否从主索引中读取数据 - * @param bulk - * 是否使用bulk查询 - * @return - * @throws SolrException - */ - QueryResponse find(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, boolean fromMaster, boolean bulk) - throws SolrException; +public interface Core extends Reloadable +{ + public static final int INDEX_BATCH_SIZE = 30000; + public static final FastDateFormat VERSION_DATE_FORMAT = FastDateFormat.getInstance("yyMMddHHmmssSSS"); + + String getName(); + + Collection getIndexes() throws SolrException; + + void ensureIndex(final Collection p0, final Iterator p1) throws SolrException; + + void ensureIndex(final Index p0, final Iterator p1) throws SolrException; + + void reIndex(final List p0, final Iterator p1) throws SolrException; + + void save(final Iterator p0) throws SolrException; + + void deleteByQuery(final BSONObject p0) throws SolrException; + + QueryResponse find(final BSONObject p0, final BSONObject p1, final BSONObject p2, final int p3, final int p4, final boolean p5, final boolean p6) throws SolrException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CoreImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CoreImpl.java index 7b4ea2efd..6467ed951 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CoreImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/CoreImpl.java @@ -1,485 +1,480 @@ package com.fangjia.datastore.solr; +import com.fangjia.datastore.utils.LoggerFactory; +import com.fangjia.datastore.TriggerExecuteException; +import com.fangjia.datastore.config.Trigger; +import com.fangjia.datastore.TriggerExecutor; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.common.SolrDocument; import java.util.ArrayList; +import com.fangjia.datastore.QueryResponse; +import org.apache.commons.lang.StringUtils; +import com.mongodb.BasicDBObject; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.commons.collections.CollectionUtils; +import org.bson.types.ObjectId; +import java.util.List; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.LinkedBlockingQueue; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrQuery.ORDER; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.SpatialParams; -import org.bson.BSONObject; -import org.bson.types.ObjectId; -import org.slf4j.Logger; -import org.sothis.core.util.BlockingThreadPoolExecutor; - import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.QueryResponse; -import com.fangjia.datastore.TriggerExecuteException; -import com.fangjia.datastore.TriggerExecutor; -import com.fangjia.datastore.config.ServerConfig; -import com.fangjia.datastore.config.StoreConfig; -import com.fangjia.datastore.config.Trigger; +import java.util.concurrent.LinkedBlockingQueue; +import org.sothis.core.util.BlockingThreadPoolExecutor; +import java.util.concurrent.Executor; +import org.bson.BSONObject; +import java.util.concurrent.BlockingQueue; import com.fangjia.datastore.utils.AsyncOperation; -import com.fangjia.datastore.utils.LoggerFactory; -import com.mongodb.BasicDBObject; +import com.fangjia.datastore.config.StoreConfig; +import com.fangjia.datastore.config.ServerConfig; +import org.slf4j.Logger; -public class CoreImpl implements Core { - private final static Logger LOGGER = LoggerFactory.getLogger(CoreImpl.class); - private final static Logger PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(CoreImpl.class); - private final static int TRIIGER_POOL_SIZE = Integer.getInteger("datastore.triggerPoolSize", 10); +public class CoreImpl implements Core +{ + private static final Logger LOGGER; + private static final Logger PERFORMANCE_LOGGER; + private static final int TRIIGER_POOL_SIZE; + private final String name; + private final SolrServer solrServer; + private final ServerConfig serverConfig; + private final StoreConfig storeConfig; + private final AsyncOperation asynchOperation; + private final BlockingQueue solrAddQueue; + private final BlockingQueue triggerQueue; + private final Executor triggerExecutorService; + + public CoreImpl(final StoreConfig storeConfig) throws ConfigurationException { + this.name = storeConfig.getName(); + this.storeConfig = storeConfig; + this.solrServer = new SolrServerImpl(this.storeConfig); + this.serverConfig = storeConfig.getDataStoreConfig().getServerConfig(); + if (CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("init solr core: " + this.name + ", storeConfig: " + storeConfig); + } + this.asynchOperation = AsyncOperation.getInstance(); + this.triggerExecutorService = new BlockingThreadPoolExecutor(this.name + "-Trigger", 1, CoreImpl.TRIIGER_POOL_SIZE, CoreImpl.TRIIGER_POOL_SIZE, 60000L); + this.solrAddQueue = new LinkedBlockingQueue(60000); + final Thread solrIndexAddThread = new Thread(new SolrAddRunnable(), this.name + "-SolrAdd"); + solrIndexAddThread.setDaemon(true); + solrIndexAddThread.start(); + this.triggerQueue = new LinkedBlockingQueue(this.serverConfig.getTriggerBatchSize() * 4); + final Thread triggerExecuteThread = new Thread(new TriggerRunnable(), this.name + "-Trigger"); + triggerExecuteThread.setDaemon(true); + triggerExecuteThread.start(); + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + while (!CoreImpl.this.solrAddQueue.isEmpty()) { + CoreImpl.LOGGER.info(CoreImpl.this.name + ": wait for solr queue, current size: " + CoreImpl.this.solrAddQueue.size()); + try { + Thread.sleep(1000L); + } + catch (InterruptedException e) { + CoreImpl.LOGGER.error("", (Throwable)e); + } + } + CoreImpl.LOGGER.info(CoreImpl.this.name + ": shutdown."); + } + }); + if (CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("solr core " + this.name + " initialized"); + } + } + + private void saveToSolr(final Iterator iterator, final String operationName) throws SolrException { + long c = 0L; + final long startTime = System.currentTimeMillis(); + while (iterator.hasNext()) { + try { + this.triggerQueue.put(iterator.next()); + } + catch (InterruptedException e) { + throw new SolrException("error put object: ", e); + } + ++c; + if (null != operationName && c % this.serverConfig.getTriggerBatchSize() == 0L && CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("{}: {}, {}/s", new Object[] { operationName, c, (int)(c / ((System.currentTimeMillis() - startTime) / 1000.0)) }); + } + } + } + + @Override + public void reload() throws ConfigurationException { + this.solrServer.reload(); + try { + this.solrServer.reloadCores(); + } + catch (SolrException e) { + throw new ConfigurationException(e); + } + } + + @Override + public Collection getIndexes() throws SolrException { + return this.solrServer.getSolrSchema().getIndexes(); + } + + @Override + public void ensureIndex(final Collection indexes, final Iterator iterator) throws SolrException { + try { + final String operationName = "ensureIndex_" + System.currentTimeMillis(); + if (CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("{}: ensure index[{}] {} ...", new Object[] { this.getName(), operationName, indexes }); + } + if (this.solrServer.getSolrSchema().saveIndex(indexes) > 0) { + this.solrServer.reloadCores(); + } + this.saveToSolr(iterator, operationName); + if (CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("{}: ensure index[{}] {} finished", new Object[] { this.getName(), operationName, indexes }); + } + } + catch (SolrException e) { + CoreImpl.LOGGER.error("ensureIndex: ", (Throwable)e); + } + } + + @Override + public void ensureIndex(final Index index, final Iterator iterator) throws SolrException { + this.ensureIndex(Arrays.asList(index), iterator); + } + + @Override + public void reIndex(final List objIdList, final Iterator iterator) throws SolrException { + final boolean logInfo = CollectionUtils.isEmpty((Collection)objIdList); + final long startTime = System.currentTimeMillis(); + final String operationName = logInfo ? (this.getName() + "_re-index_" + startTime) : null; + if (logInfo && CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info(operationName + " started."); + } + this.saveToSolr(iterator, operationName); + if (logInfo && CoreImpl.LOGGER.isInfoEnabled()) { + CoreImpl.LOGGER.info("{} finished, total time: {}s.", operationName, (System.currentTimeMillis() - startTime) / 1000L); + } + } + + @Override + public void save(final Iterator iterator) throws SolrException { + this.asynchOperation.run(new Runnable() { + @Override + public void run() { + try { + CoreImpl.this.saveToSolr(iterator, "save_" + System.currentTimeMillis()); + } + catch (SolrException e) { + CoreImpl.LOGGER.error(CoreImpl.this.name + ": error save objects into solr:", (Throwable)e); + } + } + }); + } + + @Override + public void deleteByQuery(final BSONObject query) throws SolrException { + final String solrQuery = this.toSolrQuery(query).get("q"); + if (CoreImpl.LOGGER.isDebugEnabled()) { + CoreImpl.LOGGER.debug("deleteByQuery: {}", (Object)solrQuery); + } + this.solrServer.deleteByQuery(solrQuery); + } + + private SolrQuery toSolrQuery(final BSONObject query) throws SolrException { + return this.toSolrQuery(query, false); + } + + private SolrQuery toSolrQuery(final BSONObject orgQuery, final boolean useFilterQuery) throws SolrException { + System.err.println(orgQuery.toString()); + final BSONObject query = (BSONObject)new BasicDBObject(); + query.putAll(orgQuery); + final SolrQueryBuilder queryBuilder = new SolrQueryBuilder(); + final SolrQuery solrQuery = new SolrQuery(); + if (useFilterQuery && query != null && query.containsField("visible") && query.containsField("city") && query.containsField("cat") && query.get("cat") instanceof String) { + final StringBuilder fq = new StringBuilder(); + fq.append("+cat:").append(query.get("cat")); + fq.append(" +city:").append(query.get("city")); + fq.append(" +visible:").append(((Number)query.get("visible")).intValue()); + solrQuery.addFilterQuery(fq.toString()); + query.removeField("cat"); + query.removeField("city"); + query.removeField("visible"); + } + for (final String key : query.keySet()) { + if (key.charAt(0) == '$') { + continue; + } + final Object value = query.get(key); + final Index index = this.assertFieldExistence(key); + if (value instanceof BSONObject) { + final BSONObject subQuery = (BSONObject)value; + Object g = subQuery.get("$gt"); + boolean ge = false; + if (null == g) { + g = subQuery.get("$gte"); + ge = (g != null); + } + Object l = subQuery.get("$lt"); + boolean le = false; + if (null == l) { + l = subQuery.get("$lte"); + le = (l != null); + } + if (ge && null == l) { + le = true; + } + if (le && null == g) { + ge = true; + } + if (null != g || null != l) { + queryBuilder.range(key, index.normalizeValue(g), index.normalizeValue(l), ge, le); + } + final Object in = subQuery.get("$in"); + if (null != in) { + queryBuilder.in(key, index.normalizeValue((List)in)); + } + final Object ne = subQuery.get("$ne"); + if (null == ne) { + continue; + } + queryBuilder.ne(key, index.normalizeValue(ne)); + } + else { + queryBuilder.eq(key, index.normalizeValue(value)); + } + } + String q = queryBuilder.toQuery().toString(); + if (query.get("$sr") != null) { + final Object value2 = query.get("$sr"); + final BSONObject bsonObject = (BSONObject)value2; + final String val = bsonObject.get("sr.query").toString(); + if (val != null && !"".equals(val)) { + q = q + " +(" + val + ")"; + } + } + if (query.get("$sd") != null) { + final Object value2 = query.get("$sd"); + final BSONObject bsonObject = (BSONObject)value2; + final String distance = (String)bsonObject.get("sd.distance"); + final String field = (String)bsonObject.get("sd.sfield"); + final String point = (String)bsonObject.get("sd.point"); + if (!StringUtils.isNotBlank(point) || !StringUtils.isNotBlank(distance) || !StringUtils.isNotBlank(field)) { + throw new IllegalArgumentException("paramName $sd value illegal : " + value2); + } + solrQuery.setParam("sfield", field); + solrQuery.setParam("d", distance); + solrQuery.setParam("pt", point); + solrQuery.addSort("geodist()", SolrQuery.ORDER.asc); + solrQuery.addFilterQuery("{!geofilt}"); + } + if (query.get("$fangjiawang_query_type") != null) { + final Object value2 = query.get("$fangjiawang_query_type"); + solrQuery.add("q.op", value2.toString()); + } + if (StringUtils.isEmpty(q)) { + solrQuery.setQuery("*:*"); + } + else { + solrQuery.setQuery(q); + } + return solrQuery; + } + + private Index assertFieldExistence(final String key) throws SolrException { + final Index index = this.solrServer.getSolrSchema().getIndex(key); + if (null == index) { + throw new SolrException("index '" + key + "' is not exist!"); + } + return index; + } + + @Override + public QueryResponse find(final BSONObject query, final BSONObject sort, final BSONObject facet, int skip, int limit, final boolean fromMaster, final boolean bulk) throws SolrException { + final long start = System.currentTimeMillis(); + if (limit < 0) { + limit = this.serverConfig.getDefaultCursorLimit(); + } + if (skip < 0) { + skip = 0; + } + final SolrQuery solrQuery = this.toSolrQuery(query, true); + solrQuery.setFields("_id"); + solrQuery.setStart(skip); + solrQuery.setRows(limit); + if (null != sort) { + for (final String key : sort.keySet()) { + final int value = ((Number)sort.get(key)).intValue(); + if (!"score".equals(key)) { + this.assertFieldExistence(key); + } + solrQuery.addSort(key, (value > 0) ? SolrQuery.ORDER.asc : SolrQuery.ORDER.desc); + } + } + if (null != facet) { + final List fields = (List)facet.get("fields"); + if (null != fields) { + solrQuery.setFacet(true); + Integer facetMinCount = null; + if (null != facet.get("minCount")) { + facetMinCount = ((Number)facet.get("minCount")).intValue(); + } + solrQuery.setFacetMinCount((null == facetMinCount) ? 1 : ((int)facetMinCount)); + Integer facetLimit = null; + if (null != facet.get("limit")) { + facetLimit = ((Number)facet.get("limit")).intValue(); + } + solrQuery.setFacetLimit((null == facetLimit) ? 100 : facetLimit); + for (final String field : fields) { + this.assertFieldExistence(field); + solrQuery.addFacetField(field); + } + } + } + final org.apache.solr.client.solrj.response.QueryResponse queryResponse = this.solrServer.query(solrQuery, fromMaster, bulk); + final long time = System.currentTimeMillis() - start; + if (time > 100L) { + String host = solrQuery.get("__host"); + solrQuery.remove("__host"); + final StringBuilder sb = new StringBuilder(); + sb.append("query solr: ").append(solrQuery); + if(StringUtils.isNotBlank(host)){ + sb.append(", host: ").append(host); + } + sb.append(", time: ").append(time); + sb.append(", qtime: ").append(queryResponse.getQTime()); + sb.append(", numFound: ").append(queryResponse.getResults().getNumFound()); + sb.append(", numReturned: ").append(queryResponse.getResults().size()); + sb.append(", master: ").append(fromMaster); + CoreImpl.PERFORMANCE_LOGGER.info(sb.toString()); + if (time < 1000L) { + CoreImpl.PERFORMANCE_LOGGER.info(sb.toString()); + } + else if (time < 3000L) { + CoreImpl.PERFORMANCE_LOGGER.warn(sb.toString()); + } + else { + CoreImpl.PERFORMANCE_LOGGER.error(sb.toString()); + } + } + final QueryResponse response = this.toSolrResponse(queryResponse); + return response; + } + + private QueryResponse toSolrResponse(final org.apache.solr.client.solrj.response.QueryResponse response) { + if (response == null) { + return QueryResponse.EMPTY_RESPONSE; + } + final SolrDocumentList solrDocumentList = response.getResults(); + final int numReturned = solrDocumentList.size(); + final List idList = new ArrayList(numReturned); + for (int i = 0; i < numReturned; ++i) { + final String value = ((SolrDocument)solrDocumentList.get(i)).getFirstValue("_id").toString(); + idList.add(new ObjectId(value)); + } + final QueryResponse solrResponse = new QueryResponse(); + solrResponse.setNumFound((int)solrDocumentList.getNumFound()); + solrResponse.setNumReturned(numReturned); + solrResponse.setNumToSkip((int)solrDocumentList.getStart()); + solrResponse.setIdList(idList); + solrResponse.setFacetFieldList(response.getFacetFields()); + return solrResponse; + } + + @Override + public String getName() { + return this.name; + } + + private List executeTrigger(final List list) throws TriggerExecuteException { + final Trigger trigger = this.storeConfig.getTrigger(); + if (null == trigger) { + if (CoreImpl.LOGGER.isWarnEnabled()) { + CoreImpl.LOGGER.warn("null trigger!"); + } + return list; + } + if (null == trigger.getInterface()) { + return list; + } + return TriggerExecutor.getInstance().execute(trigger, list); + } + + static { + LOGGER = LoggerFactory.getLogger(CoreImpl.class); + PERFORMANCE_LOGGER = LoggerFactory.getPerformanceLogger(CoreImpl.class); + TRIIGER_POOL_SIZE = Integer.getInteger("datastore.triggerPoolSize", 10); + } - private final String name; - private final SolrServer solrServer; - private final ServerConfig serverConfig; - private final StoreConfig storeConfig; - private final AsyncOperation asynchOperation; - private final BlockingQueue solrAddQueue; - private final BlockingQueue triggerQueue; - private final Executor triggerExecutorService; + private class TriggerRunnable implements Runnable { - public CoreImpl(StoreConfig storeConfig) throws ConfigurationException { - this.name = storeConfig.getName(); - this.storeConfig = storeConfig; - this.solrServer = new SolrServerImpl(this.storeConfig); - this.serverConfig = storeConfig.getDataStoreConfig().getServerConfig(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("init solr core: " + name + ", storeConfig: " + storeConfig); - } + public void run() { + int sleepTime = 0; + while (true) { + try { + final List dbObjects = new ArrayList(serverConfig.getTriggerBatchSize()); + triggerQueue.drainTo(dbObjects, serverConfig.getTriggerBatchSize()); + if (!dbObjects.isEmpty()) { + triggerExecutorService.execute(new Runnable() { - this.asynchOperation = AsyncOperation.getInstance(); + public void run() { + try { + long time = System.currentTimeMillis(); + List list = executeTrigger(dbObjects); + time = System.currentTimeMillis() - time; + if (LOGGER.isInfoEnabled() && time > 0) { + LOGGER.info("execute trigger: {}/{} objects, {}ms, {}/s", new Object[] { list.size(), + dbObjects.size(), time, dbObjects.size() * 1000 / time }); + } + for (BSONObject obj : list) { + solrAddQueue.put(obj); + } + } catch (Exception e) { + LOGGER.error("error execute trigger: ", e); + } + } - // trigger executor - this.triggerExecutorService = new BlockingThreadPoolExecutor(name + "-Trigger", 1, TRIIGER_POOL_SIZE, TRIIGER_POOL_SIZE, - 60000); + }); + sleepTime = serverConfig.getTriggerBatchSize() - dbObjects.size(); + if (sleepTime > 100) { + Thread.sleep(sleepTime * 3); + } + } else { + Thread.sleep(serverConfig.getTriggerBatchSize()); + } + } catch (Exception e) { + LOGGER.error("error execute trigger: ", e); + } + } + } - // solr add thread - this.solrAddQueue = new LinkedBlockingQueue(INDEX_BATCH_SIZE * 2); - Thread solrIndexAddThread = new Thread(new SolrAddRunnable(), name + "-SolrAdd"); - solrIndexAddThread.setDaemon(true); - solrIndexAddThread.start(); - - // trigger thread - this.triggerQueue = new LinkedBlockingQueue(this.serverConfig.getTriggerBatchSize() * 4); - Thread triggerExecuteThread = new Thread(new TriggerRunnable(), name + "-Trigger"); - triggerExecuteThread.setDaemon(true); - triggerExecuteThread.start(); - - // shutdown hook - Runtime.getRuntime().addShutdownHook(new Thread() { - - @Override - public void run() { - while (!solrAddQueue.isEmpty()) { - LOGGER.info(CoreImpl.this.name + ": wait for solr queue, current size: " + solrAddQueue.size()); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - LOGGER.error("", e); - } - } - LOGGER.info(CoreImpl.this.name + ": shutdown."); - } - - }); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("solr core " + name + " initialized"); - } - } - - private class TriggerRunnable implements Runnable { - - public void run() { - int sleepTime = 0; - while (true) { - try { - final List dbObjects = new ArrayList(serverConfig.getTriggerBatchSize()); - triggerQueue.drainTo(dbObjects, serverConfig.getTriggerBatchSize()); - if (!dbObjects.isEmpty()) { - triggerExecutorService.execute(new Runnable() { - - public void run() { - try { - long time = System.currentTimeMillis(); - List list = executeTrigger(dbObjects); - time = System.currentTimeMillis() - time; - if (LOGGER.isInfoEnabled() && time > 0) { - LOGGER.info("execute trigger: {}/{} objects, {}ms, {}/s", new Object[] { list.size(), - dbObjects.size(), time, dbObjects.size() * 1000 / time }); - } - for (BSONObject obj : list) { - solrAddQueue.put(obj); - } - } catch (Exception e) { - LOGGER.error("error execute trigger: ", e); - } - } - - }); - sleepTime = serverConfig.getTriggerBatchSize() - dbObjects.size(); - if (sleepTime > 100) { - Thread.sleep(sleepTime * 3); - } - } else { - Thread.sleep(serverConfig.getTriggerBatchSize()); - } - } catch (Exception e) { - LOGGER.error("error execute trigger: ", e); - } - } - } - - } - - private class SolrAddRunnable implements Runnable { - - public void run() { - final List dbObjects = new ArrayList(INDEX_BATCH_SIZE); - final int maxSleepTime = 10000; - int sleepTime = 0; - while (true) { - try { - solrAddQueue.drainTo(dbObjects, INDEX_BATCH_SIZE); - if (!dbObjects.isEmpty()) { - solrServer.add(dbObjects); - sleepTime = maxSleepTime - dbObjects.size(); - if (sleepTime > 100) { - Thread.sleep(sleepTime); - } - } else { - Thread.sleep(maxSleepTime); - } - } catch (Exception e) { - LOGGER.error("error save to solr: ", e); - } finally { - dbObjects.clear(); - } - } - } - } - - private void saveToSolr(final Iterator iterator, String operationName) throws SolrException { - long c = 0; - long startTime = System.currentTimeMillis(); - while (iterator.hasNext()) { - try { - triggerQueue.put(iterator.next()); - } catch (InterruptedException e) { - throw new SolrException("error put object: ", e); - } - c++; - if (null != operationName && c % serverConfig.getTriggerBatchSize() == 0 && LOGGER.isInfoEnabled()) { - LOGGER.info("{}: {}, {}/s", new Object[] { operationName, c, - (int) (c / ((System.currentTimeMillis() - startTime) / 1000.0)) }); - } - } - } - - public void reload() throws ConfigurationException { - this.solrServer.reload(); - try { - this.solrServer.reloadCores(); - } catch (SolrException e) { - throw new ConfigurationException(e); - } - } - - public Collection getIndexes() throws SolrException { - return this.solrServer.getSolrSchema().getIndexes(); - } - - public void ensureIndex(final Collection indexes, final Iterator iterator) throws SolrException { - try { - String operationName = "ensureIndex_" + System.currentTimeMillis(); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("{}: ensure index[{}] {} ...", new Object[] { CoreImpl.this.getName(), operationName, indexes }); - } - if (solrServer.getSolrSchema().saveIndex(indexes) > 0) { - this.solrServer.reloadCores(); - } - saveToSolr(iterator, operationName); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("{}: ensure index[{}] {} finished", new Object[] { CoreImpl.this.getName(), operationName, indexes }); - } - } catch (SolrException e) { - LOGGER.error("ensureIndex: ", e); - } - } - - public void ensureIndex(Index index, Iterator iterator) throws SolrException { - ensureIndex(Arrays.asList(index), iterator); - } - - public void reIndex(final List objIdList, final Iterator iterator) throws SolrException { - boolean logInfo = CollectionUtils.isEmpty(objIdList); - long startTime = System.currentTimeMillis(); - String operationName = logInfo ? CoreImpl.this.getName() + "_re-index_" + startTime : null; - if (logInfo && LOGGER.isInfoEnabled()) { - LOGGER.info(operationName + " started."); - } - saveToSolr(iterator, operationName); - if (logInfo && LOGGER.isInfoEnabled()) { - LOGGER.info("{} finished, total time: {}s.", operationName, (System.currentTimeMillis() - startTime) / 1000); - } - } - - public void save(final Iterator iterator) throws SolrException { - asynchOperation.run(new Runnable() { - public void run() { - try { - saveToSolr(iterator, "save_" + System.currentTimeMillis()); - } catch (SolrException e) { - LOGGER.error(name + ": error save objects into solr:", e); - } - } - }); - } - - public void deleteByQuery(final BSONObject query) throws SolrException { - String solrQuery = toSolrQuery(query).get(CommonParams.Q); - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("deleteByQuery: {}", solrQuery); - } - this.solrServer.deleteByQuery(solrQuery); - } - - private SolrQuery toSolrQuery(BSONObject query) throws SolrException { - return toSolrQuery(query, false); - } - - private SolrQuery toSolrQuery(final BSONObject orgQuery, boolean useFilterQuery) throws SolrException { - System.err.println(orgQuery.toString()); - BSONObject query = new BasicDBObject(); - query.putAll(orgQuery); - SolrQueryBuilder queryBuilder = new SolrQueryBuilder(); - - SolrQuery solrQuery = new SolrQuery(); - if (useFilterQuery && query != null && query.containsField("visible") && query.containsField("city") - && query.containsField("cat") && query.get("cat") instanceof String) { - StringBuilder fq = new StringBuilder(); - fq.append("+cat:").append(query.get("cat")); - fq.append(" +city:").append(query.get("city")); - fq.append(" +visible:").append(((Number) query.get("visible")).intValue()); - solrQuery.addFilterQuery(fq.toString()); - query.removeField("cat"); - query.removeField("city"); - query.removeField("visible"); - } - - if (null != query) { - for (String key : query.keySet()) { - if (key.charAt(0) == '$') { - continue; - } - Object value = query.get(key); - Index index = assertFieldExistence(key); - if (value instanceof BSONObject) { - BSONObject subQuery = (BSONObject) value; - Object g = subQuery.get("$gt"); - boolean ge = false; - if (null == g) { - g = subQuery.get("$gte"); - ge = g != null; - } - Object l = subQuery.get("$lt"); - boolean le = false; - if (null == l) { - l = subQuery.get("$lte"); - le = l != null; - } - if (ge && null == l) { - le = true; - } - if (le && null == g) { - ge = true; - } - if (null != g || null != l) { - queryBuilder.range(key, index.normalizeValue(g), index.normalizeValue(l), ge, le); - } - Object in = subQuery.get("$in"); - if (null != in) { - queryBuilder.in(key, index.normalizeValue((List) in)); - } - Object ne = subQuery.get("$ne"); - if (null != ne) { - queryBuilder.ne(key, index.normalizeValue(ne)); - } - - } else { - queryBuilder.eq(key, index.normalizeValue(value)); - } - } - } - String q = queryBuilder.toQuery().toString(); - - // $sr模式 - if (null != query) { - if (query.get("$sr") != null) { - Object value = query.get("$sr"); - BSONObject bsonObject = (BSONObject) value; - String val = bsonObject.get("sr.query").toString(); - if (val != null && !"".equals(val)) { - q += " +(" + val + ")"; - } - } - if (query.get("$sd") != null) { - Object value = query.get("$sd"); - BSONObject bsonObject = (BSONObject) value; - String distance = (String) bsonObject.get("sd.distance"); - String field = (String) bsonObject.get("sd.sfield"); - String point = (String) bsonObject.get("sd.point"); - if (StringUtils.isNotBlank(point) && StringUtils.isNotBlank(distance) && StringUtils.isNotBlank(field)) { - solrQuery.setParam(SpatialParams.FIELD, field); - solrQuery.setParam(SpatialParams.DISTANCE, distance); - solrQuery.setParam(SpatialParams.POINT, point); - solrQuery.addSort("geodist()", ORDER.asc); - solrQuery.addFilterQuery("{!geofilt}"); - } else { - throw new IllegalArgumentException("paramName $sd value illegal : " + value); - } - } - - if (query.get("$fangjiawang_query_type") != null) { - Object value = query.get("$fangjiawang_query_type"); - solrQuery.add("q.op", value.toString()); - } - } - - - - if (StringUtils.isEmpty(q)) { - solrQuery.setQuery("*:*"); - } else { - solrQuery.setQuery(q); - } - return solrQuery; - } - - private Index assertFieldExistence(String key) throws SolrException { - Index index = this.solrServer.getSolrSchema().getIndex(key); - if (null == index) { - throw new SolrException("index '" + key + "' is not exist!"); - } - return index; - } - - public QueryResponse find(BSONObject query, BSONObject sort, BSONObject facet, int skip, int limit, boolean fromMaster, - boolean bulk) throws SolrException { - long start = System.currentTimeMillis(); - if (limit < 0) { - limit = serverConfig.getDefaultCursorLimit(); - } - if (skip < 0) { - skip = 0; - } - SolrQuery solrQuery = toSolrQuery(query, true); - - // fields, start, limit - solrQuery.setFields(SolrSchema.ID); - solrQuery.setStart(skip); - solrQuery.setRows(limit); - - // sort - if (null != sort) { - for (String key : sort.keySet()) { - int value = ((Number) sort.get(key)).intValue(); - if (!"score".equals(key)) { // pseudo-field score - assertFieldExistence(key); - } - solrQuery.addSort(key, value > 0 ? ORDER.asc : ORDER.desc); - } - } - - // facet - if (null != facet) { - List fields = (List) facet.get("fields"); - if (null != fields) { - solrQuery.setFacet(true); - Integer facetMinCount = null; - if (null != facet.get("minCount")) { - facetMinCount = ((Number) facet.get("minCount")).intValue(); - } - solrQuery.setFacetMinCount(null == facetMinCount ? 1 : facetMinCount); - Integer facetLimit = null; - if (null != facet.get("limit")) { - facetLimit = ((Number) facet.get("limit")).intValue(); - } - solrQuery.setFacetLimit(null == facetLimit ? 100 : facetLimit); - for (String field : fields) { - assertFieldExistence(field); - solrQuery.addFacetField(field); - } - } - } - - org.apache.solr.client.solrj.response.QueryResponse queryResponse = this.solrServer.query(solrQuery, fromMaster, bulk); - long time = System.currentTimeMillis() - start; - if (time > 100) { - StringBuilder sb = new StringBuilder(); - sb.append("query solr: ").append(solrQuery); - sb.append(", time: ").append(time); - sb.append(", qtime: ").append(queryResponse.getQTime()); - sb.append(", numFound: ").append(queryResponse.getResults().getNumFound()); - sb.append(", numReturned: ").append(queryResponse.getResults().size()); - sb.append(", master: ").append(fromMaster); - if (time < 1000) { - PERFORMANCE_LOGGER.info(sb.toString()); - } else if (time < 3000) { - PERFORMANCE_LOGGER.warn(sb.toString()); - } else { - PERFORMANCE_LOGGER.error(sb.toString()); - } - } - QueryResponse response = toSolrResponse(queryResponse); - return response; - } - - private QueryResponse toSolrResponse(org.apache.solr.client.solrj.response.QueryResponse response) { - if (response == null) { - return QueryResponse.EMPTY_RESPONSE; - } - - SolrDocumentList solrDocumentList = response.getResults(); - int numReturned = solrDocumentList.size(); - List idList = new ArrayList(numReturned); - for (int i = 0; i < numReturned; i++) { - String value = solrDocumentList.get(i).getFirstValue(SolrSchema.ID).toString(); - idList.add(new ObjectId(value)); - } - - QueryResponse solrResponse = new QueryResponse(); - solrResponse.setNumFound((int) solrDocumentList.getNumFound()); - solrResponse.setNumReturned(numReturned); - solrResponse.setNumToSkip((int) solrDocumentList.getStart()); - solrResponse.setIdList(idList); - solrResponse.setFacetFieldList(response.getFacetFields()); - return solrResponse; - } - - public String getName() { - return name; - } - - private List executeTrigger(List list) throws TriggerExecuteException { - Trigger trigger = this.storeConfig.getTrigger(); - if (null == trigger) { - if (LOGGER.isWarnEnabled()) { - LOGGER.warn("null trigger!"); - } - return list; - } - - if (null == trigger.getInterface()) { - return list; - } - return TriggerExecutor.getInstance().execute(trigger, list); - } + } + + private class SolrAddRunnable implements Runnable + { + @Override + public void run() { + final List dbObjects = new ArrayList(30000); + final int maxSleepTime = 10000; + int sleepTime = 0; + while (true) { + try { + CoreImpl.this.solrAddQueue.drainTo(dbObjects, 30000); + if (!dbObjects.isEmpty()) { + CoreImpl.this.solrServer.add(dbObjects); + sleepTime = 10000 - dbObjects.size(); + if (sleepTime <= 100) { + continue; + } + Thread.sleep(sleepTime); + } + else { + Thread.sleep(10000L); + } + } + catch (Exception e) { + CoreImpl.LOGGER.error("error save to solr: ", (Throwable)e); + } + finally { + dbObjects.clear(); + } + } + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/HttpSolrServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/HttpSolrServer.java index 12a416398..388c29f65 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/HttpSolrServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/HttpSolrServer.java @@ -2,13 +2,13 @@ package com.fangjia.datastore.solr; import org.apache.solr.client.solrj.SolrServer; -public abstract class HttpSolrServer extends SolrServer { - - private static final long serialVersionUID = -3967599614339379847L; - - abstract public void reloadCore() throws SolrException; - - abstract public String getCoreName(); - - abstract public String getHost(); +public abstract class HttpSolrServer extends SolrServer +{ + private static final long serialVersionUID = -3967599614339379847L; + + public abstract void reloadCore() throws SolrException; + + public abstract String getCoreName(); + + public abstract String getHost(); } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Index.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Index.java index 0ece19d97..a8a7c1415 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Index.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/Index.java @@ -1,191 +1,196 @@ package com.fangjia.datastore.solr; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - import org.apache.commons.lang.builder.EqualsBuilder; import org.apache.commons.lang.builder.HashCodeBuilder; -import org.bson.BSONObject; import org.bson.BasicBSONObject; +import org.bson.BSONObject; +import java.util.Date; +import java.util.Iterator; +import java.util.ArrayList; +import java.util.List; -public class Index { - public static enum IndexTypeEnum { - STRING, INTEGER, INT, DOUBLE, TEXT_STD, TEXT_CJK, DATE, LONG, LOCATION; - public String getType() { - return this.name().toLowerCase(); - } - - public static IndexTypeEnum parse(String type) { - for (IndexTypeEnum e : IndexTypeEnum.values()) { - if (e.getType().equals(type)) { - return e; - } - } - return null; - } - } - - private String name; - private IndexTypeEnum type; - private boolean stored; - private boolean indexed; - private boolean multiValued; - private String comments; - private boolean dynamic; - - public Index(String name, IndexTypeEnum type, boolean stored, boolean indexed, boolean multiValued, boolean dynamic, - String comments) { - this.name = name; - this.type = type; - this.stored = stored; - this.indexed = indexed; - this.multiValued = multiValued; - this.dynamic = dynamic; - this.comments = comments; - } - - public Index(String name, IndexTypeEnum type) { - this.name = name; - this.type = type; - this.stored = false; - this.indexed = true; - this.multiValued = false; - this.dynamic = false; - this.comments = ""; - } - - public List normalizeValue(List values) { - if (null == values) { - return null; - } - List ret = new ArrayList(values.size()); - for (Object v : values) { - ret.add(normalizeValue(v)); - } - return ret; - } - - public Object normalizeValue(Object value) { - if (null == value) { - return null; - } - switch (this.type) { - case STRING: - case TEXT_STD: - return value.toString(); - case INTEGER: - return ((Number) value).intValue(); - case LONG: - return ((Number) value).longValue(); - case DOUBLE: - return ((Number) value).doubleValue(); - case DATE: - return (Date) value; - } - return value; - } - - public Index() { - this(null, null); - } - - public String getName() { - return name; - } - - public IndexTypeEnum getType() { - return type; - } - - public boolean isStored() { - return stored; - } - - public boolean isIndexed() { - return indexed; - } - - public boolean isMultiValued() { - return multiValued; - } - - public String getComments() { - return comments; - } - - public boolean isDynamic() { - return dynamic; - } - - public void setName(String name) { - this.name = name; - } - - public void setType(IndexTypeEnum type) { - this.type = type; - } - - public void setStored(boolean stored) { - this.stored = stored; - } - - public void setIndexed(boolean indexed) { - this.indexed = indexed; - } - - public void setMultiValued(boolean multiValued) { - this.multiValued = multiValued; - } - - public void setComments(String comments) { - this.comments = comments; - } - - public void setDynamic(boolean dynamic) { - this.dynamic = dynamic; - } - - public BSONObject toBSONObject() { - BSONObject ret = new BasicBSONObject(); - ret.put("name", name); - ret.put("type", this.type.getType()); - ret.put("stored", this.stored); - ret.put("indexed", this.indexed); - ret.put("multiValued", this.multiValued); - ret.put("dynamic", this.dynamic); - ret.put("comments", this.comments); - return ret; - } - - @Override - public int hashCode() { - return new HashCodeBuilder().append(this.dynamic).append(this.indexed).append(this.multiValued).append(this.stored) - .append(this.comments).append(this.name).append(this.type).toHashCode(); - } - - @Override - public boolean equals(Object obj) { - if (null == obj) { - return false; - } - if (this == obj) { - return true; - } - if (!(obj instanceof Index)) { - return false; - } - Index other = (Index) obj; - return new EqualsBuilder().append(this.dynamic, other.dynamic).append(this.indexed, other.indexed).append( - this.multiValued, other.multiValued).append(this.stored, other.stored).append(this.comments, other.comments) - .append(this.name, other.name).append(this.type, other.type).isEquals(); - } - - @Override - public String toString() { - return new StringBuilder().append("{name=").append(name).append(", type=").append(type).append(", stored=") - .append(stored).append(", indexed=").append(indexed).append(", multiValued=").append(multiValued).append( - ", dynamic=").append(dynamic).append(", comments=").append(this.comments).append("}").toString(); - } - +public class Index +{ + private String name; + private IndexTypeEnum type; + private boolean stored; + private boolean indexed; + private boolean multiValued; + private String comments; + private boolean dynamic; + + public Index(final String name, final IndexTypeEnum type, final boolean stored, final boolean indexed, final boolean multiValued, final boolean dynamic, final String comments) { + this.name = name; + this.type = type; + this.stored = stored; + this.indexed = indexed; + this.multiValued = multiValued; + this.dynamic = dynamic; + this.comments = comments; + } + + public Index(final String name, final IndexTypeEnum type) { + this.name = name; + this.type = type; + this.stored = false; + this.indexed = true; + this.multiValued = false; + this.dynamic = false; + this.comments = ""; + } + + public List normalizeValue(final List values) { + if (null == values) { + return null; + } + final List ret = new ArrayList(values.size()); + for (final Object v : values) { + ret.add(this.normalizeValue(v)); + } + return ret; + } + + public Object normalizeValue(final Object value) { + if (null == value) { + return null; + } + switch (this.type) { + case STRING: + case TEXT_STD: + return value.toString(); + case INTEGER: + return ((Number)value).intValue(); + case LONG: + return ((Number)value).longValue(); + case DOUBLE: + return ((Number)value).doubleValue(); + case DATE: + return value; + default: + return value; + } + } + + public Index() { + this(null, null); + } + + public String getName() { + return this.name; + } + + public IndexTypeEnum getType() { + return this.type; + } + + public boolean isStored() { + return this.stored; + } + + public boolean isIndexed() { + return this.indexed; + } + + public boolean isMultiValued() { + return this.multiValued; + } + + public String getComments() { + return this.comments; + } + + public boolean isDynamic() { + return this.dynamic; + } + + public void setName(final String name) { + this.name = name; + } + + public void setType(final IndexTypeEnum type) { + this.type = type; + } + + public void setStored(final boolean stored) { + this.stored = stored; + } + + public void setIndexed(final boolean indexed) { + this.indexed = indexed; + } + + public void setMultiValued(final boolean multiValued) { + this.multiValued = multiValued; + } + + public void setComments(final String comments) { + this.comments = comments; + } + + public void setDynamic(final boolean dynamic) { + this.dynamic = dynamic; + } + + public BSONObject toBSONObject() { + final BSONObject ret = (BSONObject)new BasicBSONObject(); + ret.put("name", (Object)this.name); + ret.put("type", (Object)this.type.getType()); + ret.put("stored", (Object)this.stored); + ret.put("indexed", (Object)this.indexed); + ret.put("multiValued", (Object)this.multiValued); + ret.put("dynamic", (Object)this.dynamic); + ret.put("comments", (Object)this.comments); + return ret; + } + + @Override + public int hashCode() { + return new HashCodeBuilder().append(this.dynamic).append(this.indexed).append(this.multiValued).append(this.stored).append((Object)this.comments).append((Object)this.name).append((Object)this.type).toHashCode(); + } + + @Override + public boolean equals(final Object obj) { + if (null == obj) { + return false; + } + if (this == obj) { + return true; + } + if (!(obj instanceof Index)) { + return false; + } + final Index other = (Index)obj; + return new EqualsBuilder().append(this.dynamic, other.dynamic).append(this.indexed, other.indexed).append(this.multiValued, other.multiValued).append(this.stored, other.stored).append((Object)this.comments, (Object)other.comments).append((Object)this.name, (Object)other.name).append((Object)this.type, (Object)other.type).isEquals(); + } + + @Override + public String toString() { + return "{name=" + this.name + ", type=" + this.type + ", stored=" + this.stored + ", indexed=" + this.indexed + ", multiValued=" + this.multiValued + ", dynamic=" + this.dynamic + ", comments=" + this.comments + "}"; + } + + public enum IndexTypeEnum + { + STRING, + INTEGER, + INT, + DOUBLE, + TEXT_STD, + TEXT_CJK, + DATE, + LONG, + LOCATION; + + public String getType() { + return this.name().toLowerCase(); + } + + public static IndexTypeEnum parse(final String type) { + for (final IndexTypeEnum e : values()) { + if (e.getType().equals(type)) { + return e; + } + } + return null; + } + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/LBHttpSolrServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/LBHttpSolrServer.java index bbfaae8af..f54925e00 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/LBHttpSolrServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/LBHttpSolrServer.java @@ -1,262 +1,223 @@ package com.fangjia.datastore.solr; -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.RandomUtils; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; +import com.fangjia.datastore.utils.LoggerFactory; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.params.ShardParams; +import java.io.IOException; +import org.apache.commons.lang.StringUtils; import org.apache.solr.common.util.NamedList; +import org.apache.solr.client.solrj.SolrRequest; +import java.util.concurrent.TimeUnit; +import java.lang.ref.WeakReference; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.Executors; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.params.SolrParams; +import java.util.Iterator; +import org.apache.commons.lang.math.RandomUtils; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.locks.ReentrantLock; +import org.apache.solr.client.solrj.SolrQuery; import org.slf4j.Logger; -import com.fangjia.datastore.utils.LoggerFactory; - -/** - * - * @author velna - * @see org.apache.solr.client.solrj.impl.LBHttpSolrServer - */ -public class LBHttpSolrServer extends HttpSolrServer { - private static final long serialVersionUID = -6175098554678462150L; - private final static Logger LOGGER = LoggerFactory.getLogger(LBHttpSolrServer.class); - - private static final SolrQuery SOLR_QUERY = new SolrQuery("*:*"); - private static final int CHECK_INTERVAL = 10 * 1000; - - private final String coreName; - - private final ReentrantLock checkLock = new ReentrantLock(); - private ScheduledExecutorService aliveCheckExecutor; - private final CopyOnWriteArrayList aliveServers = new CopyOnWriteArrayList(); - private final CopyOnWriteArrayList zombieServers = new CopyOnWriteArrayList(); - - public LBHttpSolrServer(HttpSolrServer... httpSolrServers) { - if (null == httpSolrServers || httpSolrServers.length == 0) { - throw new IllegalArgumentException("length of httpSolrServers must greater than 0"); - } - for (HttpSolrServer httpSolrServer : httpSolrServers) { - this.aliveServers.add(new WrappedHttpSolrServer(httpSolrServer)); - } - coreName = httpSolrServers[0].getCoreName(); - } - - private WrappedHttpSolrServer selectRandomServer() { - for (;;) { - if (aliveServers.isEmpty()) { - throw new RuntimeException("no solr server available, all are down!"); - } - try { - if (aliveServers.size() == 1) { - return aliveServers.get(0); - } - return aliveServers.get(RandomUtils.nextInt(aliveServers.size())); - } catch (IndexOutOfBoundsException e) { - continue; - } - } - } - - /*@Override - public NamedList request(final SolrRequest request) throws SolrServerException, IOException { - int attempts = 0; - int startSize = aliveServers.size(); - Exception ex; - String zombieHost = null; - for (;;) { - WrappedHttpSolrServer httpSolrServer = selectRandomServer(); - try { - if (null != zombieHost && request.getParams() instanceof SolrQuery) { - SolrQuery query = (SolrQuery) request.getParams(); - String shards = query.get(ShardParams.SHARDS); - if (null != shards) { - shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost()); - query.set(ShardParams.SHARDS, shards); - } - } - return httpSolrServer.httpSolrServer.request(request); - } catch (SolrServerException e) { - if (e.getRootCause() instanceof IOException) { - ex = e; - zombieHost = httpSolrServer.httpSolrServer.getHost(); - LOGGER.error("error request solr server: " + zombieHost, e); - moveAliveToDead(httpSolrServer); - } else { - throw e; - } - } - attempts++; - if (attempts >= startSize) - throw new SolrServerException("No live SolrServers available to handle this request", ex); - } - }*/ - - @Override - public String getHost() { - for (WrappedHttpSolrServer server : aliveServers) { - LOGGER.info("---"+server.httpSolrServer.getHost()); - } - return selectRandomServer().httpSolrServer.getHost(); - } - - @Override - public String getCoreName() { - return coreName; - } - - @Override - public void reloadCore() throws com.fangjia.datastore.solr.SolrException { - // does noting - } - - private void moveAliveToDead(WrappedHttpSolrServer solrServer) { - checkLock.lock(); - try { - boolean result = aliveServers.remove(solrServer); - if (result) { - if (zombieServers.addIfAbsent(solrServer)) { - if (LOGGER.isWarnEnabled()) { - LOGGER.warn("solr server [" + solrServer.httpSolrServer.getHost() + "] is marked as down."); - } - startAliveCheckExecutor(); - } - } - } finally { - checkLock.unlock(); - } - } - - private void checkAZombieServer(WrappedHttpSolrServer zombieServer) { - checkLock.lock(); - try { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("check solr server [" + zombieServer.httpSolrServer.getHost() + "] ..."); - } - QueryResponse resp = zombieServer.httpSolrServer.query(SOLR_QUERY); - if (resp.getStatus() == 0) { - // server has come back up - zombieServers.remove(zombieServer); - aliveServers.add(zombieServer); - if (LOGGER.isInfoEnabled()) { - LOGGER.info("solr server [" + zombieServer.httpSolrServer.getHost() + "] has come back."); - } - zombieServer.failedPings = 0; - } - } catch (Exception e) { - zombieServer.failedPings++; - // Expected . The server is still down - } finally { - checkLock.unlock(); - } - } - - private void startAliveCheckExecutor() { - if (aliveCheckExecutor == null) { - synchronized (this) { - if (aliveCheckExecutor == null) { - aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory()); - aliveCheckExecutor.scheduleAtFixedRate(getAliveCheckRunner(new WeakReference(this)), - CHECK_INTERVAL, CHECK_INTERVAL, TimeUnit.MILLISECONDS); - } - } - } - } - - private static Runnable getAliveCheckRunner(final WeakReference lbHttpSolrServer) { - return new Runnable() { - public void run() { - LBHttpSolrServer solrServer = lbHttpSolrServer.get(); - if (solrServer != null && solrServer.zombieServers != null) { - for (WrappedHttpSolrServer zombieServer : solrServer.zombieServers) { - solrServer.checkAZombieServer(zombieServer); - } - } - } - }; - } - - /** - * The default thread factory - */ - static class DefaultThreadFactory implements ThreadFactory { - static final AtomicInteger poolNumber = new AtomicInteger(1); - final ThreadGroup group; - final AtomicInteger threadNumber = new AtomicInteger(1); - final String namePrefix; - - DefaultThreadFactory() { - SecurityManager s = System.getSecurityManager(); - group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup(); - namePrefix = "solrcheck-" + poolNumber.getAndIncrement() + "-thread-"; - } - - public Thread newThread(Runnable r) { - Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); - t.setDaemon(true); - if (t.getPriority() != Thread.NORM_PRIORITY) - t.setPriority(Thread.NORM_PRIORITY); - return t; - } - } - - private class WrappedHttpSolrServer { - private final HttpSolrServer httpSolrServer; - private int failedPings = 0; - - public WrappedHttpSolrServer(HttpSolrServer httpSolrServer) { - this.httpSolrServer = httpSolrServer; - } - - } - - @Override - public NamedList request(SolrRequest paramSolrRequest, - String paramString) throws SolrServerException, IOException { - int attempts = 0; - int startSize = aliveServers.size(); - Exception ex; - String zombieHost = null; - for (;;) { - WrappedHttpSolrServer httpSolrServer = selectRandomServer(); - try { - if (null != zombieHost && paramSolrRequest.getParams() instanceof SolrQuery) { - SolrQuery query = (SolrQuery) paramSolrRequest.getParams(); - String shards = query.get(ShardParams.SHARDS); - if (null != shards) { - shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost()); - query.set(ShardParams.SHARDS, shards); - } - } - return httpSolrServer.httpSolrServer.request(paramSolrRequest); - } catch (SolrServerException e) { - if (e.getRootCause() instanceof IOException) { - ex = e; - zombieHost = httpSolrServer.httpSolrServer.getHost(); - LOGGER.error("error request solr server: " + zombieHost, e); - moveAliveToDead(httpSolrServer); - } else { - throw e; - } - } - attempts++; - if (attempts >= startSize) - throw new SolrServerException("No live SolrServers available to handle this request", ex); - } - } - - @Override - public void shutdown() { - // TODO Auto-generated method stub - - } -} \ No newline at end of file +public class LBHttpSolrServer extends HttpSolrServer +{ + private static final long serialVersionUID = -6175098554678462150L; + private static final Logger LOGGER; + private static final SolrQuery SOLR_QUERY; + private static final int CHECK_INTERVAL = 10000; + private final String coreName; + private final ReentrantLock checkLock; + private ScheduledExecutorService aliveCheckExecutor; + private final CopyOnWriteArrayList aliveServers; + private final CopyOnWriteArrayList zombieServers; + + public LBHttpSolrServer(final HttpSolrServer... httpSolrServers) { + this.checkLock = new ReentrantLock(); + this.aliveServers = new CopyOnWriteArrayList(); + this.zombieServers = new CopyOnWriteArrayList(); + if (null == httpSolrServers || httpSolrServers.length == 0) { + throw new IllegalArgumentException("length of httpSolrServers must greater than 0"); + } + for (final HttpSolrServer httpSolrServer : httpSolrServers) { + this.aliveServers.add(new WrappedHttpSolrServer(httpSolrServer)); + } + this.coreName = httpSolrServers[0].getCoreName(); + } + + private WrappedHttpSolrServer selectRandomServer() { + while (!this.aliveServers.isEmpty()) { + try { + if (this.aliveServers.size() == 1) { + return this.aliveServers.get(0); + } + return this.aliveServers.get(RandomUtils.nextInt(this.aliveServers.size())); + } + catch (IndexOutOfBoundsException e) {} + } + throw new RuntimeException("no solr server available, all are down!"); + } + + @Override + public String getHost() { + for (final WrappedHttpSolrServer server : this.aliveServers) { + LBHttpSolrServer.LOGGER.info("---" + server.httpSolrServer.getHost()); + } + return this.selectRandomServer().httpSolrServer.getHost(); + } + + @Override + public String getCoreName() { + return this.coreName; + } + + @Override + public void reloadCore() throws SolrException { + } + + private void moveAliveToDead(final WrappedHttpSolrServer solrServer) { + this.checkLock.lock(); + try { + final boolean result = this.aliveServers.remove(solrServer); + if (result && this.zombieServers.addIfAbsent(solrServer)) { + if (LBHttpSolrServer.LOGGER.isWarnEnabled()) { + LBHttpSolrServer.LOGGER.warn("solr server [" + solrServer.httpSolrServer.getHost() + "] is marked as down."); + } + this.startAliveCheckExecutor(); + } + } + finally { + this.checkLock.unlock(); + } + } + + private void checkAZombieServer(final WrappedHttpSolrServer zombieServer) { + this.checkLock.lock(); + try { + if (LBHttpSolrServer.LOGGER.isInfoEnabled()) { + LBHttpSolrServer.LOGGER.info("check solr server [" + zombieServer.httpSolrServer.getHost() + "] ..."); + } + final QueryResponse resp = zombieServer.httpSolrServer.query((SolrParams)LBHttpSolrServer.SOLR_QUERY); + if (resp.getStatus() == 0) { + this.zombieServers.remove(zombieServer); + this.aliveServers.add(zombieServer); + if (LBHttpSolrServer.LOGGER.isInfoEnabled()) { + LBHttpSolrServer.LOGGER.info("solr server [" + zombieServer.httpSolrServer.getHost() + "] has come back."); + } + zombieServer.failedPings = 0; + } + } + catch (Exception e) { + zombieServer.failedPings++; + } + finally { + this.checkLock.unlock(); + } + } + + private void startAliveCheckExecutor() { + if (this.aliveCheckExecutor == null) { + synchronized (this) { + if (this.aliveCheckExecutor == null) { + (this.aliveCheckExecutor = Executors.newSingleThreadScheduledExecutor(new DefaultThreadFactory())).scheduleAtFixedRate(getAliveCheckRunner(new WeakReference(this)), 10000L, 10000L, TimeUnit.MILLISECONDS); + } + } + } + } + + private static Runnable getAliveCheckRunner(final WeakReference lbHttpSolrServer) { + return new Runnable() { + @Override + public void run() { + final LBHttpSolrServer solrServer = (LBHttpSolrServer)lbHttpSolrServer.get(); + if (solrServer != null && solrServer.zombieServers != null) { + for (final WrappedHttpSolrServer zombieServer : solrServer.zombieServers) { + solrServer.checkAZombieServer(zombieServer); + } + } + } + }; + } + + public NamedList request(final SolrRequest paramSolrRequest, final String paramString) throws SolrServerException, IOException { + int attempts = 0; + final int startSize = this.aliveServers.size(); + String zombieHost = null; + while (true) { + final WrappedHttpSolrServer httpSolrServer = this.selectRandomServer(); + try { + if (null != zombieHost && paramSolrRequest.getParams() instanceof SolrQuery) { + final SolrQuery query = (SolrQuery)paramSolrRequest.getParams(); + String shards = query.get("shards"); + if (null != shards) { + shards = StringUtils.replace(shards, zombieHost, httpSolrServer.httpSolrServer.getHost()); + query.set("shards", shards); + } + } + return httpSolrServer.httpSolrServer.request(paramSolrRequest); + } + catch (SolrServerException e) { + if (!(e.getRootCause() instanceof IOException)) { + throw e; + } + final Exception ex = (Exception)e; + zombieHost = httpSolrServer.httpSolrServer.getHost(); + LBHttpSolrServer.LOGGER.error("error request solr server: " + zombieHost, (Throwable)e); + this.moveAliveToDead(httpSolrServer); + if (++attempts >= startSize) { + throw new SolrServerException("No live SolrServers available to handle this request", (Throwable)ex); + } + continue; + } + } + } + + public void shutdown() { + } + + static { + LOGGER = LoggerFactory.getLogger(LBHttpSolrServer.class); + SOLR_QUERY = new SolrQuery("*:*"); + } + + static class DefaultThreadFactory implements ThreadFactory + { + static final AtomicInteger poolNumber; + final ThreadGroup group; + final AtomicInteger threadNumber; + final String namePrefix; + + DefaultThreadFactory() { + this.threadNumber = new AtomicInteger(1); + final SecurityManager s = System.getSecurityManager(); + this.group = ((s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup()); + this.namePrefix = "solrcheck-" + DefaultThreadFactory.poolNumber.getAndIncrement() + "-thread-"; + } + + @Override + public Thread newThread(final Runnable r) { + final Thread t = new Thread(this.group, r, this.namePrefix + this.threadNumber.getAndIncrement(), 0L); + t.setDaemon(true); + if (t.getPriority() != 5) { + t.setPriority(5); + } + return t; + } + + static { + poolNumber = new AtomicInteger(1); + } + } + + private class WrappedHttpSolrServer + { + private final HttpSolrServer httpSolrServer; + private int failedPings; + + public WrappedHttpSolrServer(final HttpSolrServer httpSolrServer) { + this.failedPings = 0; + this.httpSolrServer = httpSolrServer; + } + } +} diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrException.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrException.java index d69e3595f..4f29518db 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrException.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrException.java @@ -1,23 +1,21 @@ package com.fangjia.datastore.solr; -public class SolrException extends Exception { - - private static final long serialVersionUID = -1105081175581071875L; - - public SolrException() { - super(); - } - - public SolrException(String message, Throwable cause) { - super(message, cause); - } - - public SolrException(String message) { - super(message); - } - - public SolrException(Throwable cause) { - super(cause); - } - +public class SolrException extends Exception +{ + private static final long serialVersionUID = -1105081175581071875L; + + public SolrException() { + } + + public SolrException(final String message, final Throwable cause) { + super(message, cause); + } + + public SolrException(final String message) { + super(message); + } + + public SolrException(final Throwable cause) { + super(cause); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrQueryBuilder.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrQueryBuilder.java index 43a840b47..fa56f6ba6 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrQueryBuilder.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrQueryBuilder.java @@ -1,177 +1,176 @@ package com.fangjia.datastore.solr; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; import java.util.TimeZone; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.time.FastDateFormat; -import org.apache.lucene.index.Term; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.NumericRangeQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; +import java.util.ArrayList; import org.apache.solr.client.solrj.util.ClientUtils; +import org.apache.commons.lang.StringUtils; +import java.util.Iterator; +import java.util.List; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeQuery; +import java.util.Date; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.commons.lang.time.FastDateFormat; +import org.apache.lucene.search.BooleanQuery; -public class SolrQueryBuilder { - private BooleanQuery booleanQuery = new BooleanQuery(); - public final static FastDateFormat DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", TimeZone - .getTimeZone("UTC")); - /** - * 约定:以$$开头的字符串不需要特殊字符转义 - */ - public final static String NO_ESCAPSE = "$$"; - - public SolrQueryBuilder range(String field, Object min, Object max, boolean minInclusive, boolean maxInclusive, - BooleanClause.Occur occur) { - if (min != null || max != null) { - Object obj = null == min ? max : min; - Query query = null; - if (Integer.class.isInstance(obj)) { - query = NumericRangeQuery.newIntRange(field, (Integer) min, (Integer) max, minInclusive, maxInclusive); - } else if (Long.class.isInstance(obj)) { - query = NumericRangeQuery.newLongRange(field, (Long) min, (Long) max, minInclusive, maxInclusive); - } else if (Double.class.isInstance(obj)) { - query = NumericRangeQuery.newDoubleRange(field, (Double) min, (Double) max, minInclusive, maxInclusive); - } else if (Float.class.isInstance(obj)) { - query = NumericRangeQuery.newFloatRange(field, (Float) min, (Float) max, minInclusive, maxInclusive); - } else if (Date.class.isInstance(obj)) { - query = new TermRangeQuery(field, null == min ? null : DATE_FORMAT.format((Date) min), null == max ? null - : DATE_FORMAT.format((Date) max), minInclusive, maxInclusive); - } else { - query = new TermRangeQuery(field, null == min ? (String) min : min.toString(), null == max ? (String) max : max - .toString(), minInclusive, maxInclusive); - } - booleanQuery.add(query, occur); - } - return this; - } - - public SolrQueryBuilder range(String field, Object min, Object max, boolean minInclusive, boolean maxInclusive) { - return range(field, min, max, minInclusive, maxInclusive, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder range(String field, Object min, Object max) { - return range(field, min, max, true, true, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder range(String field, Object min, Object max, BooleanClause.Occur occur) { - return range(field, min, max, true, true, occur); - } - - public SolrQueryBuilder gt(String field, Object value) { - return range(field, value, null, false, true, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder gt(String field, Object value, BooleanClause.Occur occur) { - return range(field, value, null, false, true, occur); - } - - public SolrQueryBuilder gte(String field, Object value) { - return range(field, value, null, true, true, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder gte(String field, Object value, BooleanClause.Occur occur) { - return range(field, value, null, true, true, occur); - } - - public SolrQueryBuilder lt(String field, Object value) { - return range(field, null, value, true, false, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder lt(String field, Object value, BooleanClause.Occur occur) { - return range(field, null, value, true, false, occur); - } - - public SolrQueryBuilder lte(String field, Object value) { - return range(field, null, value, true, true, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder lte(String field, Object value, BooleanClause.Occur occur) { - return range(field, null, value, true, true, occur); - } - - public SolrQueryBuilder eq(String field, Object value, BooleanClause.Occur occur) { - if (null != value) { - Query query = new TermQuery(new Term(field, value.toString())); - booleanQuery.add(query, occur); - } - return this; - } - - public SolrQueryBuilder eq(String field, Object value) { - return eq(field, getRealValue(value), BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder eq(String field, Object value, boolean escapse) { - return eq(field, escapse ? QueryParser.escape(value.toString()) : value.toString(), BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder ne(String field, Object value) { - return eq(field, getRealValue(value), BooleanClause.Occur.MUST_NOT); - } - - public SolrQueryBuilder in(String field, List value, boolean escapse, BooleanClause.Occur occur) { - if (null == value || value.isEmpty()) { - throw new IllegalArgumentException("in query with null or empty list!!"); - } - BooleanQuery bQuery = new BooleanQuery(); - for (Object v : value) { - Query query = new TermQuery(new Term(field, escapse ? QueryParser.escape(v.toString()) : v.toString())); - bQuery.add(query, BooleanClause.Occur.SHOULD); - } - booleanQuery.add(bQuery, occur); - return this; - } - - public SolrQueryBuilder in(String field, List value) { - return in(field, getRealValue(value), false, BooleanClause.Occur.MUST); - } - - public SolrQueryBuilder in(String field, List value, BooleanClause.Occur occur) { - return in(field, getRealValue(value), false, occur); - } - - public SolrQueryBuilder in(String field, List value, boolean escapse) { - return in(field, value, escapse, BooleanClause.Occur.MUST); - } - - public Query toQuery() { - return this.booleanQuery; - } - - /** - * 获取真实value
    - * 如果是非string的value,返回转义后的原值
    - * 如果是string型,如果以$$开头,则截取$$后面的字符且不需要转义 - * - * @param value - * @return - */ - private Object getRealValue(Object value) { - if (value instanceof String) { - if (NO_ESCAPSE.equals(value)) { - return value; - } else if (StringUtils.startsWith((String) value, NO_ESCAPSE)) { - return StringUtils.substring((String) value, NO_ESCAPSE.length()); - } - } - return ClientUtils.escapeQueryChars(value.toString()); -// return QueryParser.escape(value.toString()); - } - - private List getRealValue(List value) { - if (null == value || value.isEmpty()) { - return value; - } - List ret = new ArrayList(value.size()); - for (Object v : value) { - ret.add(getRealValue(v)); - } - return ret; - } +public class SolrQueryBuilder +{ + private BooleanQuery booleanQuery; + public static final FastDateFormat DATE_FORMAT; + public static final String NO_ESCAPSE = "$$"; + + public SolrQueryBuilder() { + this.booleanQuery = new BooleanQuery(); + } + + public SolrQueryBuilder range(final String field, final Object min, final Object max, final boolean minInclusive, final boolean maxInclusive, final BooleanClause.Occur occur) { + if (min != null || max != null) { + final Object obj = (null == min) ? max : min; + Query query = null; + if (Integer.class.isInstance(obj)) { + query = (Query)NumericRangeQuery.newIntRange(field, (Integer)min, (Integer)max, minInclusive, maxInclusive); + } + else if (Long.class.isInstance(obj)) { + query = (Query)NumericRangeQuery.newLongRange(field, (Long)min, (Long)max, minInclusive, maxInclusive); + } + else if (Double.class.isInstance(obj)) { + query = (Query)NumericRangeQuery.newDoubleRange(field, (Double)min, (Double)max, minInclusive, maxInclusive); + } + else if (Float.class.isInstance(obj)) { + query = (Query)NumericRangeQuery.newFloatRange(field, (Float)min, (Float)max, minInclusive, maxInclusive); + } + else if (Date.class.isInstance(obj)) { + query = (Query)new TermRangeQuery(field, (null == min) ? null : SolrQueryBuilder.DATE_FORMAT.format((Date)min), (null == max) ? null : SolrQueryBuilder.DATE_FORMAT.format((Date)max), minInclusive, maxInclusive); + } + else { + query = (Query)new TermRangeQuery(field, (null == min) ? ((String)min) : min.toString(), (null == max) ? ((String)max) : max.toString(), minInclusive, maxInclusive); + } + this.booleanQuery.add(query, occur); + } + return this; + } + + public SolrQueryBuilder range(final String field, final Object min, final Object max, final boolean minInclusive, final boolean maxInclusive) { + return this.range(field, min, max, minInclusive, maxInclusive, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder range(final String field, final Object min, final Object max) { + return this.range(field, min, max, true, true, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder range(final String field, final Object min, final Object max, final BooleanClause.Occur occur) { + return this.range(field, min, max, true, true, occur); + } + + public SolrQueryBuilder gt(final String field, final Object value) { + return this.range(field, value, null, false, true, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder gt(final String field, final Object value, final BooleanClause.Occur occur) { + return this.range(field, value, null, false, true, occur); + } + + public SolrQueryBuilder gte(final String field, final Object value) { + return this.range(field, value, null, true, true, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder gte(final String field, final Object value, final BooleanClause.Occur occur) { + return this.range(field, value, null, true, true, occur); + } + + public SolrQueryBuilder lt(final String field, final Object value) { + return this.range(field, null, value, true, false, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder lt(final String field, final Object value, final BooleanClause.Occur occur) { + return this.range(field, null, value, true, false, occur); + } + + public SolrQueryBuilder lte(final String field, final Object value) { + return this.range(field, null, value, true, true, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder lte(final String field, final Object value, final BooleanClause.Occur occur) { + return this.range(field, null, value, true, true, occur); + } + + public SolrQueryBuilder eq(final String field, final Object value, final BooleanClause.Occur occur) { + if (null != value) { + final Query query = (Query)new TermQuery(new Term(field, value.toString())); + this.booleanQuery.add(query, occur); + } + return this; + } + + public SolrQueryBuilder eq(final String field, final Object value) { + return this.eq(field, this.getRealValue(value), BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder eq(final String field, final Object value, final boolean escapse) { + return this.eq(field, escapse ? QueryParser.escape(value.toString()) : value.toString(), BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder ne(final String field, final Object value) { + return this.eq(field, this.getRealValue(value), BooleanClause.Occur.MUST_NOT); + } + + public SolrQueryBuilder in(final String field, final List value, final boolean escapse, final BooleanClause.Occur occur) { + if (null == value || value.isEmpty()) { + throw new IllegalArgumentException("in query with null or empty list!!"); + } + final BooleanQuery bQuery = new BooleanQuery(); + for (final Object v : value) { + final Query query = (Query)new TermQuery(new Term(field, escapse ? QueryParser.escape(v.toString()) : v.toString())); + bQuery.add(query, BooleanClause.Occur.SHOULD); + } + this.booleanQuery.add((Query)bQuery, occur); + return this; + } + + public SolrQueryBuilder in(final String field, final List value) { + return this.in(field, this.getRealValue(value), false, BooleanClause.Occur.MUST); + } + + public SolrQueryBuilder in(final String field, final List value, final BooleanClause.Occur occur) { + return this.in(field, this.getRealValue(value), false, occur); + } + + public SolrQueryBuilder in(final String field, final List value, final boolean escapse) { + return this.in(field, value, escapse, BooleanClause.Occur.MUST); + } + + public Query toQuery() { + return (Query)this.booleanQuery; + } + + private Object getRealValue(final Object value) { + if (value instanceof String) { + if ("$$".equals(value)) { + return value; + } + if (StringUtils.startsWith((String)value, "$$")) { + return StringUtils.substring((String)value, "$$".length()); + } + } + return ClientUtils.escapeQueryChars(value.toString()); + } + + private List getRealValue(final List value) { + if (null == value || value.isEmpty()) { + return value; + } + final List ret = new ArrayList(value.size()); + for (final Object v : value) { + ret.add(this.getRealValue(v)); + } + return ret; + } + + static { + DATE_FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", TimeZone.getTimeZone("UTC")); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchema.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchema.java index 5b4f06359..d04b48bb0 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchema.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchema.java @@ -1,26 +1,26 @@ package com.fangjia.datastore.solr; import java.io.File; -import java.util.Collection; import java.util.Set; - +import java.util.Collection; import com.fangjia.datastore.config.Reloadable; -public interface SolrSchema extends Reloadable { - public final static String VERTION = "_version"; - public final static String ID = "_id"; - - int saveIndex(Collection indexes) throws SolrException; - - boolean removeIndex(String name) throws SolrException; - - Index getIndex(String name); - - Collection getIndexes(); - - Set getIndexNames(); - - File getSchemaFile(); - - void setSchemaFile(File schemaFile); -} \ No newline at end of file +public interface SolrSchema extends Reloadable +{ + public static final String VERTION = "_version"; + public static final String ID = "_id"; + + int saveIndex(final Collection p0) throws SolrException; + + boolean removeIndex(final String p0) throws SolrException; + + Index getIndex(final String p0); + + Collection getIndexes(); + + Set getIndexNames(); + + File getSchemaFile(); + + void setSchemaFile(final File p0); +} diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchemaImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchemaImpl.java index e2312d7d4..d76c7202b 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchemaImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrSchemaImpl.java @@ -1,189 +1,194 @@ package com.fangjia.datastore.solr; -import java.io.File; -import java.io.FileOutputStream; +import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.dom4j.Document; +import java.io.OutputStream; +import org.dom4j.io.XMLWriter; +import java.io.FileOutputStream; import org.dom4j.DocumentException; +import java.util.Set; +import java.util.List; +import org.apache.commons.lang.StringUtils; +import org.dom4j.Node; +import java.util.Iterator; import org.dom4j.DocumentHelper; import org.dom4j.Element; -import org.dom4j.Node; -import org.dom4j.io.SAXReader; -import org.dom4j.io.XMLWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import java.util.Collection; import com.fangjia.datastore.ConfigurationException; -import com.fangjia.datastore.solr.Index.IndexTypeEnum; +import java.util.HashMap; +import org.slf4j.Logger; +import org.dom4j.Document; +import java.util.Map; +import java.io.File; +import org.dom4j.io.SAXReader; -public class SolrSchemaImpl implements SolrSchema { - private final SAXReader reader = new SAXReader(); - private File schemaFile; - private final Map indexes = new HashMap(); - private Document document; - private final static Logger LOGGER = LoggerFactory.getLogger(SolrSchemaImpl.class); - - public static void main(String[] args) throws Exception { - new SolrSchemaImpl(new File("D:\\viewstore\\solrhome\\house\\conf\\schema.xml")); - } - - public SolrSchemaImpl(File schemaFile) throws ConfigurationException { - if (!schemaFile.exists() || schemaFile.isDirectory()) { - throw new IllegalArgumentException("schema file not exists or is a directory:" + schemaFile); - } - this.schemaFile = schemaFile; - reload(); - } - - public int saveIndex(Collection newIndexes) throws SolrException { - int ret = 0; - try { - for (Index index : newIndexes) { - Index i = indexes.get(index.getName()); - if (index.equals(i)) { - continue; - } - ret++; - Element e; - if (indexes.containsKey(index.getName())) { - String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" - + index.getName() + "\"]"; - e = (Element) this.document.selectSingleNode(xpath); - } else { - e = DocumentHelper.createElement(index.isDynamic() ? "dynamicField" : "field"); - Element fields = (Element) document.selectSingleNode("//schema//fields"); - fields.add(e); - } - e.addAttribute("name", index.getName()); - e.addAttribute("type", index.getType().getType()); - e.addAttribute("stored", String.valueOf(index.isStored())); - e.addAttribute("indexed", String.valueOf(index.isIndexed())); - e.addAttribute("multiValued", String.valueOf(index.isMultiValued())); - e.addAttribute("comments", String.valueOf(index.getComments())); - this.indexes.put(index.getName(), index); - } - } finally { - write(); - } - return ret; - } - - public synchronized boolean removeIndex(String name) throws SolrException { - Index index = this.indexes.get(name); - if (null == index) { - return false; - } - if (ID.equals(name) || VERTION.equals(name)) { - throw new SolrException("delete of field \"" + name + "\" is not allowed!"); - } - String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() - + "\"]"; - Element e = (Element) this.document.selectSingleNode(xpath); - if (null != e) { - e.detach(); - } - write(); - this.indexes.remove(name); - return true; - } - - public Collection getIndexes() { - return indexes.values(); - } - - @SuppressWarnings("unchecked") - public void reload() throws ConfigurationException { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("refresh solr schema: {}", this.schemaFile); - } - Map newIndexes = new HashMap(); - - Document newDocument = read(); - - List list = newDocument.selectNodes("//schema"); - if (null == list || list.size() > 1) { - throw new ConfigurationException("zero or more than one \"fields\" tag found, expect only one \"fields\" tag"); - } - Node fieldsNode = list.get(0); - list = fieldsNode.selectNodes("field"); - for (Node node : list) { - Element e = (Element) node; - - String name = e.attributeValue("name"); - if (StringUtils.isEmpty(name)) { - throw new ConfigurationException("invalid field name \"" + name + "\" in solr schema: " + this.schemaFile); - } - - IndexTypeEnum type = IndexTypeEnum.parse(e.attributeValue("type")); - if (null == type) { - throw new ConfigurationException("invalid field type \"" + e.attributeValue("type") + "\" in solr schema: " - + this.schemaFile); - } - - boolean stored = Boolean.parseBoolean(e.attributeValue("stored")); - boolean indexed = Boolean.parseBoolean(e.attributeValue("indexed")); - boolean multiValued = Boolean.parseBoolean(e.attributeValue("multiValued")); - String comments = e.attributeValue("comments"); - boolean dynamic = e.getName().equals("dynamicField"); - - Index index = new Index(name, type, stored, indexed, multiValued, dynamic, comments); - newIndexes.put(name, index); - } - if (!newIndexes.containsKey(ID)) { - throw new ConfigurationException("no field names \"" + ID + "\" find in schema: " + this.schemaFile); - } - if (!newIndexes.containsKey(VERTION)) { - throw new ConfigurationException("no field names \"" + VERTION + "\" find in schema: " + this.schemaFile); - } - indexes.clear(); - indexes.putAll(newIndexes); - this.document = newDocument; - - } - - public Set getIndexNames() { - return this.indexes.keySet(); - } - - private Document read() throws ConfigurationException { - try { - return reader.read(schemaFile); - } catch (DocumentException e) { - throw new ConfigurationException("error read solr xml file: " + this, e); - } - } - - private void write() throws SolrException { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("write solr schema: {}" + this.schemaFile); - } - XMLWriter writer; - try { - writer = new XMLWriter(new FileOutputStream(schemaFile)); - writer.write(document); - writer.close(); - } catch (IOException e) { - throw new SolrException("error write solr xml file: " + this, e); - } - } - - public Index getIndex(String name) { - return this.indexes.get(name); - } - - public File getSchemaFile() { - return schemaFile; - } - - public void setSchemaFile(File schemaFile) { - this.schemaFile = schemaFile; - } - -} \ No newline at end of file +public class SolrSchemaImpl implements SolrSchema +{ + private final SAXReader reader; + private File schemaFile; + private final Map indexes; + private Document document; + private static final Logger LOGGER; + + public static void main(final String[] args) throws Exception { + new SolrSchemaImpl(new File("D:\\viewstore\\solrhome\\house\\conf\\schema.xml")); + } + + public SolrSchemaImpl(final File schemaFile) throws ConfigurationException { + this.reader = new SAXReader(); + this.indexes = new HashMap(); + if (!schemaFile.exists() || schemaFile.isDirectory()) { + throw new IllegalArgumentException("schema file not exists or is a directory:" + schemaFile); + } + this.schemaFile = schemaFile; + this.reload(); + } + + @Override + public int saveIndex(final Collection newIndexes) throws SolrException { + int ret = 0; + try { + for (final Index index : newIndexes) { + final Index i = this.indexes.get(index.getName()); + if (index.equals(i)) { + continue; + } + ++ret; + Element e; + if (this.indexes.containsKey(index.getName())) { + final String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() + "\"]"; + e = (Element)this.document.selectSingleNode(xpath); + } + else { + e = DocumentHelper.createElement(index.isDynamic() ? "dynamicField" : "field"); + final Element fields = (Element)this.document.selectSingleNode("//schema//fields"); + fields.add(e); + } + e.addAttribute("name", index.getName()); + e.addAttribute("type", index.getType().getType()); + e.addAttribute("stored", String.valueOf(index.isStored())); + e.addAttribute("indexed", String.valueOf(index.isIndexed())); + e.addAttribute("multiValued", String.valueOf(index.isMultiValued())); + e.addAttribute("comments", String.valueOf(index.getComments())); + this.indexes.put(index.getName(), index); + } + } + finally { + this.write(); + } + return ret; + } + + @Override + public synchronized boolean removeIndex(final String name) throws SolrException { + final Index index = this.indexes.get(name); + if (null == index) { + return false; + } + if ("_id".equals(name) || "_version".equals(name)) { + throw new SolrException("delete of field \"" + name + "\" is not allowed!"); + } + final String xpath = "//schema//fields//" + (index.isDynamic() ? "dynamicField" : "field") + "[name=\"" + index.getName() + "\"]"; + final Element e = (Element)this.document.selectSingleNode(xpath); + if (null != e) { + e.detach(); + } + this.write(); + this.indexes.remove(name); + return true; + } + + @Override + public Collection getIndexes() { + return this.indexes.values(); + } + + @Override + public void reload() throws ConfigurationException { + if (SolrSchemaImpl.LOGGER.isDebugEnabled()) { + SolrSchemaImpl.LOGGER.debug("refresh solr schema: {}", (Object)this.schemaFile); + } + final Map newIndexes = new HashMap(); + final Document newDocument = this.read(); + List list = (List)newDocument.selectNodes("//schema"); + if (null == list || list.size() > 1) { + throw new ConfigurationException("zero or more than one \"fields\" tag found, expect only one \"fields\" tag"); + } + final Node fieldsNode = list.get(0); + list = (List)fieldsNode.selectNodes("field"); + for (final Node node : list) { + final Element e = (Element)node; + final String name = e.attributeValue("name"); + if (StringUtils.isEmpty(name)) { + throw new ConfigurationException("invalid field name \"" + name + "\" in solr schema: " + this.schemaFile); + } + final Index.IndexTypeEnum type = Index.IndexTypeEnum.parse(e.attributeValue("type")); + if (null == type) { + throw new ConfigurationException("invalid field type \"" + e.attributeValue("type") + "\" in solr schema: " + this.schemaFile); + } + final boolean stored = Boolean.parseBoolean(e.attributeValue("stored")); + final boolean indexed = Boolean.parseBoolean(e.attributeValue("indexed")); + final boolean multiValued = Boolean.parseBoolean(e.attributeValue("multiValued")); + final String comments = e.attributeValue("comments"); + final boolean dynamic = e.getName().equals("dynamicField"); + final Index index = new Index(name, type, stored, indexed, multiValued, dynamic, comments); + newIndexes.put(name, index); + } + if (!newIndexes.containsKey("_id")) { + throw new ConfigurationException("no field names \"_id\" find in schema: " + this.schemaFile); + } + if (!newIndexes.containsKey("_version")) { + throw new ConfigurationException("no field names \"_version\" find in schema: " + this.schemaFile); + } + this.indexes.clear(); + this.indexes.putAll(newIndexes); + this.document = newDocument; + } + + @Override + public Set getIndexNames() { + return this.indexes.keySet(); + } + + private Document read() throws ConfigurationException { + try { + return this.reader.read(this.schemaFile); + } + catch (DocumentException e) { + throw new ConfigurationException("error read solr xml file: " + this, (Throwable)e); + } + } + + private void write() throws SolrException { + if (SolrSchemaImpl.LOGGER.isDebugEnabled()) { + SolrSchemaImpl.LOGGER.debug("write solr schema: {}" + this.schemaFile); + } + try { + final XMLWriter writer = new XMLWriter((OutputStream)new FileOutputStream(this.schemaFile)); + writer.write(this.document); + writer.close(); + } + catch (IOException e) { + throw new SolrException("error write solr xml file: " + this, e); + } + } + + @Override + public Index getIndex(final String name) { + return this.indexes.get(name); + } + + @Override + public File getSchemaFile() { + return this.schemaFile; + } + + @Override + public void setSchemaFile(final File schemaFile) { + this.schemaFile = schemaFile; + } + + static { + LOGGER = LoggerFactory.getLogger((Class)SolrSchemaImpl.class); + } +} diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServer.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServer.java index 8ce424980..684ba0e35 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServer.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServer.java @@ -1,23 +1,22 @@ package com.fangjia.datastore.solr; -import java.util.List; - -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.response.QueryResponse; import org.bson.BSONObject; - +import java.util.List; import com.fangjia.datastore.ConfigurationException; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.SolrQuery; -public interface SolrServer { - QueryResponse query(SolrQuery params, boolean fromMaster, boolean bulk) throws SolrException; - - void deleteByQuery(String query) throws SolrException; - - void reload() throws ConfigurationException; - - void reloadCores() throws SolrException; - - SolrSchema getSolrSchema(); - - void add(List dbObjects) throws SolrException; +public interface SolrServer +{ + QueryResponse query(final SolrQuery p0, final boolean p1, final boolean p2) throws SolrException; + + void deleteByQuery(final String p0) throws SolrException; + + void reload() throws ConfigurationException; + + void reloadCores() throws SolrException; + + SolrSchema getSolrSchema(); + + void add(final List p0) throws SolrException; } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServerImpl.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServerImpl.java index 8bd4287db..9f4931e8f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServerImpl.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/solr/SolrServerImpl.java @@ -1,475 +1,455 @@ package com.fangjia.datastore.solr; -import java.io.File; -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.math.RandomUtils; -import org.apache.http.client.HttpClient; -import org.apache.http.client.config.CookieSpecs; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrException.ErrorCode; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.SolrInputField; -import org.apache.solr.common.params.ShardParams; -import org.bson.BSONObject; -import org.bson.types.ObjectId; -import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sothis.core.util.BlockingThreadPoolExecutor; - -import com.fangjia.datastore.ConfigurationException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.commons.lang.math.RandomUtils; +import java.io.IOException; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.commons.lang.StringUtils; +import java.util.Collections; +import org.apache.commons.collections.CollectionUtils; +import org.bson.types.ObjectId; +import org.apache.solr.common.SolrInputField; +import java.util.Iterator; +import java.util.Date; +import java.util.ArrayList; +import java.util.concurrent.CountDownLatch; +import org.bson.BSONObject; +import java.util.Collection; +import org.apache.solr.common.SolrInputDocument; +import java.util.List; +import java.util.HashMap; +import java.net.MalformedURLException; import com.fangjia.datastore.config.SolrConfig; +import com.fangjia.datastore.ConfigurationException; +import org.sothis.core.util.BlockingThreadPoolExecutor; +import java.io.File; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.Executor; +import java.util.Map; +import org.apache.http.client.HttpClient; import com.fangjia.datastore.config.StoreConfig; +import org.slf4j.Logger; -public class SolrServerImpl implements SolrServer { - - private final static Logger LOGGER = LoggerFactory.getLogger(SolrServerImpl.class); - private final static int SOLR_COMMIT_MAX_RETRY_COUNT = 3; - private final static int SOLR_QUERY_MAX_RETRY_COUNT = 3; - private final static int MAX_NONE_BULK_SKIP = 100000; - - private final String name; - private final StoreConfig storeConfig; - private final HttpClient httpClient; - private int masterCommitSolrServerCount; - private final Map allSolrServers; - private String[] masterCommitFilters; - private HttpSolrServer[][] masterCommitSolrServers; - private HttpSolrServer[] masterSolrServers; - private HttpSolrServer[] slaveSolrServers; - private SolrSchema[] solrSchemas; - private final SolrSchema solrSchema; - private final Executor solrAddExecutorService; - private final AtomicBoolean filtering = new AtomicBoolean(false); - private long lastFilterTime = 0; - - public SolrServerImpl(StoreConfig storeConfig) throws ConfigurationException { - this.storeConfig = storeConfig; - this.name = storeConfig.getName(); - - /*MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager(); - connectionManager.getParams().setTcpNoDelay(true); - connectionManager.getParams().setStaleCheckingEnabled(true); - this.httpClient = new HttpClient(connectionManager); - httpClient.getParams().setConnectionManagerTimeout(0); - DefaultHttpMethodRetryHandler retryhandler = new DefaultHttpMethodRetryHandler(0, false); - this.httpClient.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, retryhandler); - */ - PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); - cm.setMaxTotal(5); - /* RequestConfig requestConfig = RequestConfig.custom() - .setConnectTimeout(1000 * 10) - .setSocketTimeout(1000 * 10) - .setCookieSpec(CookieSpecs.IGNORE_COOKIES) - .build(); - - this.httpClient = HttpClients.custom() - .setDefaultRequestConfig(requestConfig) - .setConnectionManager(cm) - .build();*/ - this.httpClient = new DefaultHttpClient(); - this.allSolrServers = initSolrServers(this.storeConfig, httpClient); - this.solrSchema = new SolrSchemaImpl(new File(storeConfig.getMasterSolrConfig().getHome() + "/" + name - + "/conf/managed-schema")); - int size = (masterCommitSolrServerCount + 1) * 2; - this.solrAddExecutorService = new BlockingThreadPoolExecutor(name + "-SolrAdd", 1, size, size, 60000); - } - - private CommonsHttpSolrServer createSolrServer(String coreName, SolrConfig solronfig) throws ConfigurationException { - try { - CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(solronfig.getHost(), coreName, httpClient); - return solrServer; - } catch (MalformedURLException e) { - throw new ConfigurationException("[" + this.name + "] error create solr server: ", e); - } - } - - private HttpSolrServer[] initSolrServer(Map allSolrServers, SolrConfig solrConfig) - throws ConfigurationException { - SolrConfig[] shardsConfigs = solrConfig.getShardsConfigs(); - if (shardsConfigs.length == 0) { - shardsConfigs = new SolrConfig[] { solrConfig }; - } - HttpSolrServer[] shards = new HttpSolrServer[shardsConfigs.length]; - for (int i = 0; i < shardsConfigs.length; i++) { - SolrConfig shardConfig = shardsConfigs[i]; - if (!allSolrServers.containsKey(shardConfig.getId())) { - SolrConfig[] slaveConfigs = shardConfig.getSlaveConfigs(); - if (slaveConfigs.length == 0) { - allSolrServers.put(shardConfig.getId(), createSolrServer(name, shardConfig)); - } else { - CommonsHttpSolrServer[] slaveServers = new CommonsHttpSolrServer[slaveConfigs.length]; - for (int j = 0; j < slaveConfigs.length; j++) { - SolrConfig slaveConfig = slaveConfigs[j]; - if (!allSolrServers.containsKey(slaveConfig.getId())) { - allSolrServers.put(slaveConfig.getId(), createSolrServer(name, slaveConfig)); - } - slaveServers[j] = (CommonsHttpSolrServer) allSolrServers.get(slaveConfig.getId()); - } - allSolrServers.put(shardConfig.getId(), new LBHttpSolrServer(slaveServers)); - } - } - shards[i] = allSolrServers.get(shardConfig.getId()); - } - return shards; - } - - private synchronized Map initSolrServers(StoreConfig storeConfig, HttpClient httpClient) - throws ConfigurationException { - // http connections - //httpClient.getHttpConnectionManager().getParams().setDefaultMaxConnectionsPerHost( - //storeConfig.getSolrMaxConnectionsPerHost()); - //httpClient.getHttpConnectionManager().getParams().setMaxTotalConnections(storeConfig.getSolrMaxTotalConnections()); - - Map solrServers = new HashMap(); - SolrConfig[] masterConfigs = storeConfig.getMasterSolrConfigs(); - masterCommitSolrServers = new HttpSolrServer[masterConfigs.length][]; - masterCommitFilters = new String[masterConfigs.length]; - solrSchemas = new SolrSchema[masterConfigs.length]; - masterCommitSolrServerCount = 0; - for (int i = 0; i < masterConfigs.length; i++) { - masterCommitSolrServers[i] = initSolrServer(solrServers, masterConfigs[i]); - masterCommitFilters[i] = masterConfigs[i].getFilter(); - solrSchemas[i] = new SolrSchemaImpl(new File(masterConfigs[i].getHome() + "/" + name + "/conf/managed-schema")); - masterCommitSolrServerCount += masterCommitSolrServers[i].length; - } - masterSolrServers = initSolrServer(solrServers, storeConfig.getMasterSolrConfig()); - slaveSolrServers = initSolrServer(solrServers, storeConfig.getSlaveSolrConfig()); - - return solrServers; - } - - private void addToSolr0(HttpSolrServer solrServer, List documents, int errorCount) { - try { - if (null != documents && !documents.isEmpty()) { - solrServer.add(documents); - solrServer.commit(); - } - } catch (Exception e) { - if (errorCount <= SOLR_COMMIT_MAX_RETRY_COUNT) { - errorCount++; - long sleepTime = errorCount * errorCount; - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] error commit to solr[" + solrServer.getHost() + "], sleep " + sleepTime - + "s and retry(" + e.getMessage() + ")"); - } - try { - Thread.sleep(1000 * sleepTime); - } catch (InterruptedException e1) { - LOGGER.error("[" + this.name + "] error save to solr: ", e1); - } - addToSolr0(solrServer, documents, errorCount); - } else { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error save to solr[" + solrServer.getHost() + "] after " - + SOLR_COMMIT_MAX_RETRY_COUNT + " times retry: ", e); - } - } - } - } - - public void add(List dbObjects) throws SolrException { - if (null == dbObjects || dbObjects.size() == 0) { - return; - } - try { - long time = System.currentTimeMillis(); - checkFilter(); - final CountDownLatch countDownLatch = new CountDownLatch(masterCommitSolrServerCount); - for (int j = 0; j < this.masterCommitSolrServers.length; j++) { - final int jndex = j; - final Map> shardedObjectMap = shard(dbObjects, this.masterCommitSolrServers[j].length); - for (int i = 0; i < this.masterCommitSolrServers[j].length; i++) { - final int index = i; - this.solrAddExecutorService.execute(new Runnable() { - public void run() { - try { - List sharedObjects = shardedObjectMap.get(index); - if (null == sharedObjects || sharedObjects.size() == 0) { - return; - } - final List documents = new ArrayList(sharedObjects.size()); - for (BSONObject obj : sharedObjects) { - SolrInputDocument document = toSolrInputDocument(obj, solrSchemas[jndex]); - if (!document.isEmpty()) { - document.addField(SolrSchema.VERTION, Core.VERSION_DATE_FORMAT.format(new Date())); - System.out.println("document:" + document.toString()); - documents.add(document); - } else if (LOGGER.isDebugEnabled()) { - LOGGER.debug("solr input document is empty:" + document); - } - } - addToSolr0(masterCommitSolrServers[jndex][index], documents, 0); - } catch (Exception e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + name + "] error save to solr: ", e); - } - } finally { - countDownLatch.countDown(); - } - } - }); - } - } - countDownLatch.await(); - time = System.currentTimeMillis() - time; - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] add {} documents to solr, {}ms, {}/s.", new Object[] { dbObjects.size(), time, - dbObjects.size() * 1000 / (time) }); - } - } catch (Exception e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error save to solr: ", e); - } - } - } - - @SuppressWarnings("unchecked") - private SolrInputDocument toSolrInputDocument(BSONObject doc, SolrSchema solrSchema) { - Collection indexes = solrSchema.getIndexes(); - SolrInputDocument document = new SolrInputDocument(); - Map boostsMap = null; - if (doc.get("$boosts") != null) { - if (doc.get("$boosts") instanceof Map) { - boostsMap = (Map) doc.get("$boosts"); - if (boostsMap.containsKey("$doc_boost")) { - document.setDocumentBoost(((Number) boostsMap.get("$doc_boost")).floatValue()); - } - } - } - - for (Index index : indexes) { - String key = index.getName(); - if (doc.containsField(key)) { - Object value = doc.get(key); - SolrInputField field = new SolrInputField(key); - if (value instanceof ObjectId) { - field.setValue(value.toString(), 1.0f); - } else { - if (boostsMap != null && boostsMap.containsKey(key)) { - field.setValue(value, ((Number) boostsMap.get(key)).floatValue()); - } else { - field.setValue(value, 1.0f); - } - } - document.put(key, field); - } - } - return document; - } - - private Map> shard(final List dbObjects, int shards) { - if (CollectionUtils.isEmpty(dbObjects)) { - return Collections.emptyMap(); - } - Map> ret = new HashMap>(); - for (BSONObject object : dbObjects) { - int hashMod = Math.abs(object.get("_id").hashCode()) % shards; - List modObjects = ret.get(hashMod); - if (null == modObjects) { - modObjects = new ArrayList(); - ret.put(hashMod, modObjects); - } - modObjects.add(object); - } - return ret; - } - - private void checkFilter() { - // 5分钟一次 - if (System.currentTimeMillis() - lastFilterTime < 300000) { - return; - } - try { - if (this.filtering.compareAndSet(false, true)) { - final long start = System.currentTimeMillis(); - for (int i = 0; i < masterCommitFilters.length; i++) { - if (StringUtils.isEmpty(masterCommitFilters[i])) { - continue; - } - for (HttpSolrServer server : masterCommitSolrServers[i]) { - try { - server.deleteByQuery(masterCommitFilters[i]); - } catch (SolrServerException e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error execute filter delete: ", e); - } - } catch (IOException e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error execute filter delete: ", e); - } - } - } - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] execute filter delete, " + (System.currentTimeMillis() - start) / 1000 - + "s."); - } - lastFilterTime = System.currentTimeMillis(); - } - } catch (Exception e) { - if (LOGGER.isErrorEnabled()) { - LOGGER.error("[" + this.name + "] error execute filter delete: ", e); - } - } finally { - this.filtering.compareAndSet(true, false); - } - } - - public void deleteByQuery(String query) throws SolrException { - try { - for (HttpSolrServer[] servers : this.masterCommitSolrServers) { - for (HttpSolrServer server : servers) { - server.deleteByQuery(query); - } - } - } catch (SolrServerException e) { - throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e); - } catch (IOException e) { - throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e); - } - return; - } - - public synchronized void reload() throws ConfigurationException { - this.allSolrServers.clear(); - this.allSolrServers.putAll(initSolrServers(this.storeConfig, httpClient)); - String schema = this.storeConfig.getMasterSolrConfig().getHome() + "/" + name + "/conf/managed-schema"; - this.solrSchema.setSchemaFile(new File(schema)); - this.solrSchema.reload(); - } - - public void reloadCores() throws SolrException { - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] reload core..."); - } - for (String id : this.allSolrServers.keySet()) { - HttpSolrServer solrServer = this.allSolrServers.get(id); - solrServer.reloadCore(); - } - if (LOGGER.isInfoEnabled()) { - LOGGER.info("[" + this.name + "] core reload success"); - } - } - - private boolean isShardedServer() { - return this.slaveSolrServers.length > 1; - } - - private HttpSolrServer selectRandomServer(HttpSolrServer[] shards) { - if (null == shards || shards.length == 0) { - throw new IllegalArgumentException("no shards"); - } - if (shards.length == 1) { - LOGGER.info("Choice:" + shards[0].getHost()); - return shards[0]; - } - HttpSolrServer server = shards[RandomUtils.nextInt(shards.length)]; - LOGGER.info("Choice:" + server.getHost()); - return server; - } - - private String buildShardsParam(HttpSolrServer[] solrServers) { - StringBuilder ret = new StringBuilder(); - for (int i = 0; i < solrServers.length; i++) { - ret.append(solrServers[i].getHost()).append('/').append(this.name); - if (i < solrServers.length - 1) { - ret.append(','); - } - } - LOGGER.info("slaveAddress:" + ret.toString()); - return ret.toString(); - } - - public QueryResponse query(SolrQuery solrQuery, boolean fromMaster, boolean bulk) throws SolrException { - if (isShardedServer() && !bulk && solrQuery.getStart() > MAX_NONE_BULK_SKIP) { - throw new SolrException("[" + this.name + "] skip of none bulk query must not greater than " + MAX_NONE_BULK_SKIP - + ", try set $bulk to true in the query !"); - } - return queryWithRetry(solrQuery, fromMaster, bulk, 0); - } - - private QueryResponse queryWithRetry(SolrQuery solrQuery, boolean fromMaster, boolean bulk, int retryCount) - throws SolrException { - // shards params - LOGGER.info("isShardedServer:"+isShardedServer()); - LOGGER.info("fromMaster:"+fromMaster); - if (isShardedServer()) { - if (fromMaster) { - solrQuery.set(ShardParams.SHARDS, buildShardsParam(this.masterSolrServers)); - } else { - solrQuery.set(ShardParams.SHARDS, buildShardsParam(this.slaveSolrServers)); - } - // bulk query - if (bulk) { - int numberShards = this.masterSolrServers.length; - solrQuery.set(ShardParams.SHARDS_START, solrQuery.getStart() / numberShards); - solrQuery.set(ShardParams.SHARDS_ROWS, Math.max(solrQuery.getRows() / numberShards, 1)); - solrQuery.setStart(0); - } - } - LOGGER.info("solrQuery:"+solrQuery.toString()); - QueryResponse queryResponse = null; - HttpSolrServer httpSolrServer = null; - try { - if (fromMaster) { - httpSolrServer = selectRandomServer(this.masterSolrServers); - try { - queryResponse = httpSolrServer.query(solrQuery); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } else { - httpSolrServer = selectRandomServer(this.slaveSolrServers); - try { - queryResponse = httpSolrServer.query(solrQuery); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - } catch (SolrServerException ex) { - if (ex.getCause() instanceof org.apache.solr.common.SolrException) { - org.apache.solr.common.SolrException e = (org.apache.solr.common.SolrException) ex.getCause(); - if (retryCount < SOLR_QUERY_MAX_RETRY_COUNT && ErrorCode.getErrorCode(e.code()) == ErrorCode.SERVER_ERROR) { - retryCount++; - if (LOGGER.isErrorEnabled()) { - LOGGER.error("#[" + this.name + "]# error query solr[" + httpSolrServer.getHost() + "], code:" - + ErrorCode.SERVER_ERROR + "(" + ErrorCode.SERVER_ERROR.code + "), retry for the " + retryCount - + "th time..."); - } - return queryWithRetry(solrQuery, fromMaster, bulk, retryCount); - } else { - throw new SolrException("=[" + this.name + "]= error query solr[" + httpSolrServer.getHost() + "]:", e); - } - } - throw new SolrException("*[" + this.name + "]* error query solr:", ex); - } - return queryResponse; - } - - public SolrSchema getSolrSchema() { - return solrSchema; - } - +public class SolrServerImpl implements SolrServer +{ + private static final Logger LOGGER; + private static final int SOLR_COMMIT_MAX_RETRY_COUNT = 3; + private static final int SOLR_QUERY_MAX_RETRY_COUNT = 3; + private static final int MAX_NONE_BULK_SKIP = 100000; + private final String name; + private final StoreConfig storeConfig; + private final HttpClient httpClient; + private int masterCommitSolrServerCount; + private final Map allSolrServers; + private String[] masterCommitFilters; + private HttpSolrServer[][] masterCommitSolrServers; + private HttpSolrServer[] masterSolrServers; + private HttpSolrServer[] slaveSolrServers; + private SolrSchema[] solrSchemas; + private final SolrSchema solrSchema; + private final Executor solrAddExecutorService; + private final AtomicBoolean filtering; + private long lastFilterTime; + + public SolrServerImpl(final StoreConfig storeConfig) throws ConfigurationException { + this.filtering = new AtomicBoolean(false); + this.lastFilterTime = 0L; + this.storeConfig = storeConfig; + this.name = storeConfig.getName(); + final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); + cm.setMaxTotal(50); + this.httpClient = (HttpClient)new DefaultHttpClient(); + this.allSolrServers = this.initSolrServers(this.storeConfig, this.httpClient); + this.solrSchema = new SolrSchemaImpl(new File(storeConfig.getMasterSolrConfig().getHome() + "/" + this.name + "/conf/managed-schema")); + final int size = (this.masterCommitSolrServerCount + 1) * 2; + this.solrAddExecutorService = (Executor)new BlockingThreadPoolExecutor(this.name + "-SolrAdd", 1, size, size, 60000L); + } + + private CommonsHttpSolrServer createSolrServer(final String coreName, final SolrConfig solronfig) throws ConfigurationException { + try { + final CommonsHttpSolrServer solrServer = new CommonsHttpSolrServer(solronfig.getHost(), coreName, this.httpClient); + return solrServer; + } + catch (MalformedURLException e) { + throw new ConfigurationException("[" + this.name + "] error create solr server: ", e); + } + } + + private HttpSolrServer[] initSolrServer(final Map allSolrServers, final SolrConfig solrConfig) throws ConfigurationException { + SolrConfig[] shardsConfigs = solrConfig.getShardsConfigs(); + if (shardsConfigs.length == 0) { + shardsConfigs = new SolrConfig[] { solrConfig }; + } + final HttpSolrServer[] shards = new HttpSolrServer[shardsConfigs.length]; + for (int i = 0; i < shardsConfigs.length; ++i) { + final SolrConfig shardConfig = shardsConfigs[i]; + if (!allSolrServers.containsKey(shardConfig.getId())) { + final SolrConfig[] slaveConfigs = shardConfig.getSlaveConfigs(); + if (slaveConfigs.length == 0) { + allSolrServers.put(shardConfig.getId(), this.createSolrServer(this.name, shardConfig)); + } + else { + final CommonsHttpSolrServer[] slaveServers = new CommonsHttpSolrServer[slaveConfigs.length]; + for (int j = 0; j < slaveConfigs.length; ++j) { + final SolrConfig slaveConfig = slaveConfigs[j]; + if (!allSolrServers.containsKey(slaveConfig.getId())) { + allSolrServers.put(slaveConfig.getId(), this.createSolrServer(this.name, slaveConfig)); + } + slaveServers[j] = (CommonsHttpSolrServer)allSolrServers.get(slaveConfig.getId()); + } + allSolrServers.put(shardConfig.getId(), new LBHttpSolrServer((HttpSolrServer[])slaveServers)); + } + } + shards[i] = allSolrServers.get(shardConfig.getId()); + } + return shards; + } + + private synchronized Map initSolrServers(final StoreConfig storeConfig, final HttpClient httpClient) throws ConfigurationException { + final Map solrServers = new HashMap(); + final SolrConfig[] masterConfigs = storeConfig.getMasterSolrConfigs(); + this.masterCommitSolrServers = new HttpSolrServer[masterConfigs.length][]; + this.masterCommitFilters = new String[masterConfigs.length]; + this.solrSchemas = new SolrSchema[masterConfigs.length]; + this.masterCommitSolrServerCount = 0; + for (int i = 0; i < masterConfigs.length; ++i) { + this.masterCommitSolrServers[i] = this.initSolrServer(solrServers, masterConfigs[i]); + this.masterCommitFilters[i] = masterConfigs[i].getFilter(); + this.solrSchemas[i] = new SolrSchemaImpl(new File(masterConfigs[i].getHome() + "/" + this.name + "/conf/managed-schema")); + this.masterCommitSolrServerCount += this.masterCommitSolrServers[i].length; + } + this.masterSolrServers = this.initSolrServer(solrServers, storeConfig.getMasterSolrConfig()); + this.slaveSolrServers = this.initSolrServer(solrServers, storeConfig.getSlaveSolrConfig()); + return solrServers; + } + + private void addToSolr0(final HttpSolrServer solrServer, final List documents, int errorCount) { + try { + if (null != documents && !documents.isEmpty()) { + solrServer.add(documents); + solrServer.commit(); + } + } + catch (Exception e) { + if (errorCount <= 3) { + final long sleepTime = ++errorCount * errorCount; + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] error commit to solr[" + solrServer.getHost() + "], sleep " + sleepTime + "s and retry(" + e.getMessage() + ")"); + } + try { + Thread.sleep(1000L * sleepTime); + } + catch (InterruptedException e2) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr: ", (Throwable)e2); + } + this.addToSolr0(solrServer, documents, errorCount); + } + else if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr[" + solrServer.getHost() + "] after " + 3 + " times retry: ", (Throwable)e); + } + } + } + + @Override + public void add(final List dbObjects) throws SolrException { + if (null == dbObjects || dbObjects.size() == 0) { + return; + } + try { + long time = System.currentTimeMillis(); + this.checkFilter(); + final CountDownLatch countDownLatch = new CountDownLatch(this.masterCommitSolrServerCount); + for (int j = 0; j < this.masterCommitSolrServers.length; ++j) { + final int jndex = j; + final Map> shardedObjectMap = this.shard(dbObjects, this.masterCommitSolrServers[j].length); + for (int i = 0; i < this.masterCommitSolrServers[j].length; ++i) { + final int index = i; + this.solrAddExecutorService.execute(new Runnable() { + @Override + public void run() { + try { + final List sharedObjects = shardedObjectMap.get(index); + if (null == sharedObjects || sharedObjects.size() == 0) { + return; + } + final List documents = new ArrayList(sharedObjects.size()); + for (final BSONObject obj : sharedObjects) { + final SolrInputDocument document = SolrServerImpl.this.toSolrInputDocument(obj, SolrServerImpl.this.solrSchemas[jndex]); + if (!document.isEmpty()) { + document.addField("_version", (Object)Core.VERSION_DATE_FORMAT.format(new Date())); + System.out.println("document:" + document.toString()); + documents.add(document); + } + else { + if (!SolrServerImpl.LOGGER.isDebugEnabled()) { + continue; + } + SolrServerImpl.LOGGER.debug("solr input document is empty:" + document); + } + } + SolrServerImpl.this.addToSolr0(SolrServerImpl.this.masterCommitSolrServers[jndex][index], documents, 0); + } + catch (Exception e) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + SolrServerImpl.this.name + "] error save to solr: ", (Throwable)e); + } + } + finally { + countDownLatch.countDown(); + } + } + }); + } + } + countDownLatch.await(); + time = System.currentTimeMillis() - time; + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] add {} documents to solr, {}ms, {}/s.", new Object[] { dbObjects.size(), time, dbObjects.size() * 1000 / time }); + } + } + catch (Exception e) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error save to solr: ", (Throwable)e); + } + } + } + + private SolrInputDocument toSolrInputDocument(final BSONObject doc, final SolrSchema solrSchema) { + final Collection indexes = solrSchema.getIndexes(); + final SolrInputDocument document = new SolrInputDocument(); + Map boostsMap = null; + if (doc.get("$boosts") != null && doc.get("$boosts") instanceof Map) { + boostsMap = (Map)doc.get("$boosts"); + if (boostsMap.containsKey("$doc_boost")) { + document.setDocumentBoost(((Number) boostsMap.get("$doc_boost")).floatValue()); + } + } + for (final Index index : indexes) { + final String key = index.getName(); + if (doc.containsField(key)) { + final Object value = doc.get(key); + final SolrInputField field = new SolrInputField(key); + if (value instanceof ObjectId) { + field.setValue(value.toString(), 1.0f); + } + else if (boostsMap != null && boostsMap.containsKey(key)) { + field.setValue(value, ((Number) boostsMap.get(key)).floatValue()); + } + else { + field.setValue(value, 1.0f); + } + document.put(key, field); + } + } + return document; + } + + private Map> shard(final List dbObjects, final int shards) { + if (CollectionUtils.isEmpty((Collection)dbObjects)) { + return Collections.>emptyMap(); + } + final Map> ret = new HashMap>(); + for (final BSONObject object : dbObjects) { + final int hashMod = Math.abs(object.get("_id").hashCode()) % shards; + List modObjects = ret.get(hashMod); + if (null == modObjects) { + modObjects = new ArrayList(); + ret.put(hashMod, modObjects); + } + modObjects.add(object); + } + return ret; + } + + private void checkFilter() { + if (System.currentTimeMillis() - this.lastFilterTime < 300000L) { + return; + } + try { + if (this.filtering.compareAndSet(false, true)) { + final long start = System.currentTimeMillis(); + for (int i = 0; i < this.masterCommitFilters.length; ++i) { + if (!StringUtils.isEmpty(this.masterCommitFilters[i])) { + for (final HttpSolrServer server : this.masterCommitSolrServers[i]) { + try { + server.deleteByQuery(this.masterCommitFilters[i]); + } + catch (SolrServerException e) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e); + } + } + catch (IOException e2) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e2); + } + } + } + } + } + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] execute filter delete, " + (System.currentTimeMillis() - start) / 1000L + "s."); + } + this.lastFilterTime = System.currentTimeMillis(); + } + } + catch (Exception e3) { + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("[" + this.name + "] error execute filter delete: ", (Throwable)e3); + } + } + finally { + this.filtering.compareAndSet(true, false); + } + } + + @Override + public void deleteByQuery(final String query) throws SolrException { + try { + for (final HttpSolrServer[] array : this.masterCommitSolrServers) { + final HttpSolrServer[] servers = array; + for (final HttpSolrServer server : array) { + server.deleteByQuery(query); + } + } + } + catch (SolrServerException e) { + throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", (Throwable)e); + } + catch (IOException e2) { + throw new SolrException("[" + this.name + "] error execute delete[" + query + "]: ", e2); + } + } + + @Override + public synchronized void reload() throws ConfigurationException { + this.allSolrServers.clear(); + this.allSolrServers.putAll(this.initSolrServers(this.storeConfig, this.httpClient)); + final String schema = this.storeConfig.getMasterSolrConfig().getHome() + "/" + this.name + "/conf/managed-schema"; + this.solrSchema.setSchemaFile(new File(schema)); + this.solrSchema.reload(); + } + + @Override + public void reloadCores() throws SolrException { + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] reload core..."); + } + for (final String id : this.allSolrServers.keySet()) { + final HttpSolrServer solrServer = this.allSolrServers.get(id); + solrServer.reloadCore(); + } + if (SolrServerImpl.LOGGER.isInfoEnabled()) { + SolrServerImpl.LOGGER.info("[" + this.name + "] core reload success"); + } + } + + private boolean isShardedServer() { + return this.slaveSolrServers.length > 1; + } + + private HttpSolrServer selectRandomServer(final HttpSolrServer[] shards) { + if (null == shards || shards.length == 0) { + throw new IllegalArgumentException("no shards"); + } + if (shards.length == 1) { + SolrServerImpl.LOGGER.info("Choice:" + shards[0].getHost()); + return shards[0]; + } + final HttpSolrServer server = shards[RandomUtils.nextInt(shards.length)]; + SolrServerImpl.LOGGER.info("Choice:" + server.getHost()); + return server; + } + + private String buildShardsParam(final HttpSolrServer[] solrServers) { + final StringBuilder ret = new StringBuilder(); + for (int i = 0; i < solrServers.length; ++i) { + ret.append(solrServers[i].getHost()).append('/').append(this.name); + if (i < solrServers.length - 1) { + ret.append(','); + } + } + SolrServerImpl.LOGGER.info("slaveAddress:" + ret.toString()); + return ret.toString(); + } + + @Override + public QueryResponse query(final SolrQuery solrQuery, final boolean fromMaster, final boolean bulk) throws SolrException { + if (this.isShardedServer() && !bulk && solrQuery.getStart() > 100000) { + throw new SolrException("[" + this.name + "] skip of none bulk query must not greater than " + 100000 + ", try set $bulk to true in the query !"); + } + return this.queryWithRetry(solrQuery, fromMaster, bulk, 0); + } + + private QueryResponse queryWithRetry(final SolrQuery solrQuery, final boolean fromMaster, final boolean bulk, int retryCount) throws SolrException { + SolrServerImpl.LOGGER.info("isShardedServer:" + this.isShardedServer()); + SolrServerImpl.LOGGER.info("fromMaster:" + fromMaster); + if (this.isShardedServer()) { + if (fromMaster) { + solrQuery.set("shards", this.buildShardsParam(this.masterSolrServers)); + } + else { + solrQuery.set("shards", this.buildShardsParam(this.slaveSolrServers)); + } + if (bulk) { + final int numberShards = this.masterSolrServers.length; + solrQuery.set("shards.start", solrQuery.getStart() / numberShards); + solrQuery.set("shards.rows", Math.max(solrQuery.getRows() / numberShards, 1)); + solrQuery.setStart(0); + } + } + SolrServerImpl.LOGGER.info("solrQuery:" + solrQuery.toString()); + QueryResponse queryResponse = null; + HttpSolrServer httpSolrServer = null; + try { + if (fromMaster) { + httpSolrServer = this.selectRandomServer(this.masterSolrServers); + try { + queryResponse = httpSolrServer.query(solrQuery); + } + catch (IOException e) { + e.printStackTrace(); + } + } + else { + httpSolrServer = this.selectRandomServer(this.slaveSolrServers); + try { + queryResponse = httpSolrServer.query(solrQuery); + } + catch (IOException e) { + e.printStackTrace(); + } + } + solrQuery.add("__host", httpSolrServer.getHost()); + } + catch (SolrServerException ex) { + if (!(ex.getCause() instanceof org.apache.solr.common.SolrException)) { + throw new SolrException("*[" + this.name + "]* error query solr:", (Throwable)ex); + } + final org.apache.solr.common.SolrException e2 = (org.apache.solr.common.SolrException)ex.getCause(); + if (retryCount < 3 && org.apache.solr.common.SolrException.ErrorCode.getErrorCode(e2.code()) == org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR) { + ++retryCount; + if (SolrServerImpl.LOGGER.isErrorEnabled()) { + SolrServerImpl.LOGGER.error("#[" + this.name + "]# error query solr[" + httpSolrServer.getHost() + "], code:" + org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR + "(" + org.apache.solr.common.SolrException.ErrorCode.SERVER_ERROR.code + "), retry for the " + retryCount + "th time..."); + } + return this.queryWithRetry(solrQuery, fromMaster, bulk, retryCount); + } + throw new SolrException("=[" + this.name + "]= error query solr[" + httpSolrServer.getHost() + "]:", (Throwable)e2); + } + return queryResponse; + } + + @Override + public SolrSchema getSolrSchema() { + return this.solrSchema; + } + + static { + LOGGER = LoggerFactory.getLogger((Class)SolrServerImpl.class); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/BSONUtils.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/BSONUtils.java index 16e064815..5a1627761 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/BSONUtils.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/BSONUtils.java @@ -1,53 +1,34 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.utils; import java.io.IOException; -public class BSONUtils { - - public static long bitSet(long holder, int i, boolean set) { - if (i < 0 || i > 63) { - throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive"); - } - if (set) { - return holder | (1 << i); - } else { - int ii = 1 << i; - return (holder | ii) ^ ii; - } - } - - public static boolean isBitSet(long holder, int i) { - if (i < 0 || i > 63) { - throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive"); - } - return (holder & (1 << i)) > 0; - } - - public static int stringByteLength(String s) throws IOException { - byte[] bs = s.getBytes("UTF-8"); - return bs.length + 5; - } - - public static int cstringByteLength(String s) throws IOException { - byte[] bs = s.getBytes("UTF-8"); - return bs.length + 1; - } - +public class BSONUtils +{ + public static long bitSet(final long holder, final int i, final boolean set) { + if (i < 0 || i > 63) { + throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive"); + } + if (set) { + return holder | (long)(1 << i); + } + final int ii = 1 << i; + return (holder | (long)ii) ^ (long)ii; + } + + public static boolean isBitSet(final long holder, final int i) { + if (i < 0 || i > 63) { + throw new IllegalArgumentException("i should from 0 inclusive and 64 exclusive"); + } + return (holder & (long)(1 << i)) > 0L; + } + + public static int stringByteLength(final String s) throws IOException { + final byte[] bs = s.getBytes("UTF-8"); + return bs.length + 5; + } + + public static int cstringByteLength(final String s) throws IOException { + final byte[] bs = s.getBytes("UTF-8"); + return bs.length + 1; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/HexDump.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/HexDump.java index 3144e1ea1..dcf4e1dd0 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/HexDump.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/HexDump.java @@ -1,275 +1,106 @@ -/** - * JMongo is a mongodb driver writtern in java. - * Copyright (C) 2010 Xiaohu Huang - * - * JMongo is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * JMongo is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with JMongo. If not, see . - */ package com.fangjia.datastore.utils; import java.io.PrintStream; -/** - * Hex dump class. - */ -public final class HexDump { - - public static final void dump(byte[] byt) { - dump(byt, byt.length, 0, System.out); - } - - /** - * Hex dump a byte array - * - * @param byt - * Byte array to dump - * @param len - * Length of data to dump - * @param offset - * Offset to start data dump - */ - public static final void dump(byte[] byt, int len, int offset) { - dump(byt, len, offset, System.out); - } - - /** - * Hex dump a byte array - * - * @param byt - * Byte array to dump - * @param len - * Length of data to dump - * @param offset - * Offset to start data dump - * @param stream - * Output stream to dump the output to. - */ - - public static final void dump(byte[] byt, int len, int offset, PrintStream stream) { - - // Create buffers for the ASCII and Hex output - - StringBuffer ascBuf = new StringBuffer(); - StringBuffer hexBuf = new StringBuffer(); - - // dump 16 byte blocks from the array until the length has been - // reached - - int dlen = 0; - int doff = offset; - String posStr = null; - - while (dlen < len) { - - // Reset the ASCII/Hex buffers - - ascBuf.setLength(0); - hexBuf.setLength(0); - - posStr = generatePositionString(doff); - - // dump a block of data, update the data offset - - doff = generateLine(byt, doff, ascBuf, hexBuf); - - // Output the current record - - stream.print(posStr); - stream.print(hexBuf.toString()); - stream.println(ascBuf.toString()); - - // Update the dump length - - dlen += 16; - } - } - - /** - * Generate a hex string for the specified string - * - * @param str - * String - * @return String - */ - public static final String hexString(String str) { - if (str != null) - return hexString(str.getBytes()); - return ""; - } - - /** - * Generate a hex string for the specified string - * - * @param str - * String - * @param gap - * String - * @return String - */ - public static final String hexString(String str, String gap) { - if (str != null) - return hexString(str.getBytes(), gap); - return ""; - } - - /** - * Generate a hex string for the specified bytes - * - * @param buf - * byte[] - * @return String - */ - public static final String hexString(byte[] buf) { - return hexString(buf, buf.length, null); - } - - /** - * Generate a hex string for the specified bytes - * - * @param buf - * byte[] - * @param gap - * String - * @return String - */ - public static final String hexString(byte[] buf, String gap) { - return hexString(buf, buf.length, gap); - } - - /** - * Generate a hex string for the specified bytes - * - * @param buf - * byte[] - * @param len - * int - * @param gap - * String - * @return String - */ - public static final String hexString(byte[] buf, int len, String gap) { - - // Check if the buffer is valid - - if (buf == null) - return ""; - - // Create a string buffer for the hex string - - int buflen = buf.length * 2; - if (gap != null) - buflen += buf.length * gap.length(); - - StringBuffer hex = new StringBuffer(buflen); - - // Convert the bytes to hex-ASCII - - for (int i = 0; i < len; i++) { - - // Get the current byte - - int curbyt = (int) (buf[i] & 0x00FF); - - // Output the hex string - - hex.append(Integer.toHexString((curbyt & 0xF0) >> 4)); - hex.append(Integer.toHexString(curbyt & 0x0F)); - - // Add the gap string, if specified - - if (gap != null && i < (len - 1)) - hex.append(gap); - } - - // Return the hex-ASCII string - - return hex.toString(); - } - - /** - * Generate a buffer position string - * - * @param off - * int - * @return String - */ - private static final String generatePositionString(int off) { - - // Create a buffer position string - - StringBuffer posStr = new StringBuffer("" + off + ": "); - while (posStr.length() < 10) - posStr.insert(0, " "); - - // Return the string - - return posStr.toString(); - } - - /** - * Output a single line of the hex dump to a debug device - * - * @param byt - * Byte array to dump - * @param off - * Offset to start data dump - * @param ascBuf - * Buffer for ASCII output - * @param hexBuf - * Buffer for Hex output - * @return New offset value - */ - - private static final int generateLine(byte[] byt, int off, StringBuffer ascBuf, StringBuffer hexBuf) { - - // Check if there is enough buffer space to dump 16 bytes - - int dumplen = byt.length - off; - if (dumplen > 16) - dumplen = 16; - - // dump a 16 byte block of data - - for (int i = 0; i < dumplen; i++) { - - // Get the current byte - - int curbyt = (int) (byt[off++] & 0x00FF); - - // Output the hex string - - hexBuf.append(Integer.toHexString((curbyt & 0xF0) >> 4)); - hexBuf.append(Integer.toHexString(curbyt & 0x0F)); - hexBuf.append(" "); - - // Output the character equivalent, if printable - - if (Character.isLetterOrDigit((char) curbyt) || Character.getType((char) curbyt) != Character.CONTROL) - ascBuf.append((char) curbyt); - else - ascBuf.append("."); - } - - for (int i = dumplen; i < 16; i++) { - hexBuf.append(" "); - } - - // Output the hex dump line - - hexBuf.append(" "); - - // Return the new data offset - - return off; - } +public final class HexDump +{ + public static final void dump(final byte[] byt) { + dump(byt, byt.length, 0, System.out); + } + + public static final void dump(final byte[] byt, final int len, final int offset) { + dump(byt, len, offset, System.out); + } + + public static final void dump(final byte[] byt, final int len, final int offset, final PrintStream stream) { + final StringBuffer ascBuf = new StringBuffer(); + final StringBuffer hexBuf = new StringBuffer(); + int dlen = 0; + int doff = offset; + String posStr = null; + while (dlen < len) { + ascBuf.setLength(0); + hexBuf.setLength(0); + posStr = generatePositionString(doff); + doff = generateLine(byt, doff, ascBuf, hexBuf); + stream.print(posStr); + stream.print(hexBuf.toString()); + stream.println(ascBuf.toString()); + dlen += 16; + } + } + + public static final String hexString(final String str) { + if (str != null) { + return hexString(str.getBytes()); + } + return ""; + } + + public static final String hexString(final String str, final String gap) { + if (str != null) { + return hexString(str.getBytes(), gap); + } + return ""; + } + + public static final String hexString(final byte[] buf) { + return hexString(buf, buf.length, null); + } + + public static final String hexString(final byte[] buf, final String gap) { + return hexString(buf, buf.length, gap); + } + + public static final String hexString(final byte[] buf, final int len, final String gap) { + if (buf == null) { + return ""; + } + int buflen = buf.length * 2; + if (gap != null) { + buflen += buf.length * gap.length(); + } + final StringBuffer hex = new StringBuffer(buflen); + for (int i = 0; i < len; ++i) { + final int curbyt = buf[i] & 0xFF; + hex.append(Integer.toHexString((curbyt & 0xF0) >> 4)); + hex.append(Integer.toHexString(curbyt & 0xF)); + if (gap != null && i < len - 1) { + hex.append(gap); + } + } + return hex.toString(); + } + + private static final String generatePositionString(final int off) { + final StringBuffer posStr = new StringBuffer("" + off + ": "); + while (posStr.length() < 10) { + posStr.insert(0, " "); + } + return posStr.toString(); + } + + private static final int generateLine(final byte[] byt, int off, final StringBuffer ascBuf, final StringBuffer hexBuf) { + int dumplen = byt.length - off; + if (dumplen > 16) { + dumplen = 16; + } + for (int i = 0; i < dumplen; ++i) { + final int curbyt = byt[off++] & 0xFF; + hexBuf.append(Integer.toHexString((curbyt & 0xF0) >> 4)); + hexBuf.append(Integer.toHexString(curbyt & 0xF)); + hexBuf.append(" "); + if (Character.isLetterOrDigit((char)curbyt) || Character.getType((char)curbyt) != 15) { + ascBuf.append((char)curbyt); + } + else { + ascBuf.append("."); + } + } + for (int i = dumplen; i < 16; ++i) { + hexBuf.append(" "); + } + hexBuf.append(" "); + return off; + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/LoggerFactory.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/LoggerFactory.java index 8031389b2..41f60064f 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/LoggerFactory.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/LoggerFactory.java @@ -2,17 +2,17 @@ package com.fangjia.datastore.utils; import org.slf4j.Logger; -public class LoggerFactory { - - public static Logger getLogger(Class clazz) { - return org.slf4j.LoggerFactory.getLogger(clazz); - } - - public static Logger getLogger(String name) { - return org.slf4j.LoggerFactory.getLogger(name); - } - - public static Logger getPerformanceLogger(Class clazz) { - return org.slf4j.LoggerFactory.getLogger("performance." + clazz.getName()); - } +public class LoggerFactory +{ + public static Logger getLogger(final Class clazz) { + return org.slf4j.LoggerFactory.getLogger((Class)clazz); + } + + public static Logger getLogger(final String name) { + return org.slf4j.LoggerFactory.getLogger(name); + } + + public static Logger getPerformanceLogger(final Class clazz) { + return org.slf4j.LoggerFactory.getLogger("performance." + clazz.getName()); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/MongoUtils.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/MongoUtils.java index 1bc84fd61..e90fda2a0 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/MongoUtils.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/MongoUtils.java @@ -2,22 +2,23 @@ package com.fangjia.datastore.utils; import org.apache.commons.lang.StringUtils; -public class MongoUtils { - private MongoUtils() { - } - - public static String[] parseFullCollectionName(String fullCollectionName) { - if (StringUtils.isEmpty(fullCollectionName)) { - return null; - } - int i = fullCollectionName.indexOf('.'); - if (i <= 0 || i >= fullCollectionName.length() - 1) { - return null; - } - return new String[] { fullCollectionName.substring(0, i), fullCollectionName.substring(i + 1) }; - } - - public static void main(String[] args) { - System.out.println(parseFullCollectionName(".q")); - } +public class MongoUtils +{ + private MongoUtils() { + } + + public static String[] parseFullCollectionName(final String fullCollectionName) { + if (StringUtils.isEmpty(fullCollectionName)) { + return null; + } + final int i = fullCollectionName.indexOf(46); + if (i <= 0 || i >= fullCollectionName.length() - 1) { + return null; + } + return new String[] { fullCollectionName.substring(0, i), fullCollectionName.substring(i + 1) }; + } + + public static void main(final String[] args) { + System.out.println(parseFullCollectionName(".q")); + } } diff --git a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/XmlUtils.java b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/XmlUtils.java index 26246685b..72363e986 100644 --- a/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/XmlUtils.java +++ b/projects/fangjia-datastore/src/main/java/com/fangjia/datastore/utils/XmlUtils.java @@ -2,45 +2,42 @@ package com.fangjia.datastore.utils; import org.dom4j.Node; -public class XmlUtils { - public static String getNodeValue(Node parentNode, String nodeName, String defaultValue) { - Node node = parentNode.selectSingleNode(nodeName); - String ret = defaultValue; - if (null == node) { - return ret; - } else { - ret = node.getText(); - if (null == ret) { - ret = defaultValue; - } - } - return ret; - } - - public static int getNodeValueAsInt(Node parentNode, String nodeName, int defaultValue) { - String value = getNodeValue(parentNode, nodeName, null); - if (null == value) { - return defaultValue; - } else { - return Integer.parseInt(value); - } - } - - public static long getNodeValueAsLong(Node parentNode, String nodeName, long defaultValue) { - String value = getNodeValue(parentNode, nodeName, null); - if (null == value) { - return defaultValue; - } else { - return Long.parseLong(value); - } - } - - public static boolean getNodeValueAsBoolean(Node parentNode, String nodeName, boolean defaultValue) { - String value = getNodeValue(parentNode, nodeName, null); - if (null == value) { - return defaultValue; - } else { - return Boolean.parseBoolean(value); - } - } +public class XmlUtils +{ + public static String getNodeValue(final Node parentNode, final String nodeName, final String defaultValue) { + final Node node = parentNode.selectSingleNode(nodeName); + String ret = defaultValue; + if (null == node) { + return ret; + } + ret = node.getText(); + if (null == ret) { + ret = defaultValue; + } + return ret; + } + + public static int getNodeValueAsInt(final Node parentNode, final String nodeName, final int defaultValue) { + final String value = getNodeValue(parentNode, nodeName, null); + if (null == value) { + return defaultValue; + } + return Integer.parseInt(value); + } + + public static long getNodeValueAsLong(final Node parentNode, final String nodeName, final long defaultValue) { + final String value = getNodeValue(parentNode, nodeName, null); + if (null == value) { + return defaultValue; + } + return Long.parseLong(value); + } + + public static boolean getNodeValueAsBoolean(final Node parentNode, final String nodeName, final boolean defaultValue) { + final String value = getNodeValue(parentNode, nodeName, null); + if (null == value) { + return defaultValue; + } + return Boolean.parseBoolean(value); + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java index b515c4593..3668027a7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java @@ -32,7 +32,7 @@ public class AuctionController { private PaiMaiService paiMaiService; private DistrictNewPriceService districtNewPriceService; private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", - "matchEstateType","type","count", "tradeAvgPrice", "evaluationPrice"}; + "matchEstateType","type","count", "tradeAvgPrice", "evaluationPrice","houseNum","roomNum","unitNum","estateType"}; public Object housesAction(AuctionModel model){ RestfulModelAndView mav = new RestfulModelAndView(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index 50b2700a2..5f2fddf88 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -452,6 +452,62 @@ public class HouseController { return mav; } + + + public Object arroundlistAction(final HouseSearchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) { + mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项")); + return mav; + } + District district = null; + DistrictKey districtKey = null; + if(StringUtils.isNotBlank(model.getName())){ + districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName()); + district = districtService.findDistrict(districtKey, "district"); + }else{ + district = districtService.findDistrictById(model.getId()); + if(district!=null){ + districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + } + } + + if(district == null){ + mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mav; + } + + //周边小区,默认10 + List districts = districtService.findNearestDistrict(district, 1000, model.getDistCount()); + List houses = new ArrayList(); + for(District dist : districts){ + districtKey = new DistrictKey(dist.getCity(), dist.getRegion(), dist.getName()); + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.9); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.1); + } + List names = new ArrayList(); + names.add(district.getName()); + if(district.getAlias() != null){ + for(String alia : district.getAlias()){ + if(StringUtils.isNotBlank(alia) && !names.contains(alia)){ + names.add(alia); + } + } + } + List arroundhouses = seawaterDao.findSeawatersFromEs("sell", dist.getCity(), dist.getRegion(), dist.getName(), dist.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater"); + if(arroundhouses!=null && arroundhouses.size()>0){ + houses.addAll(arroundhouses); + } + } + mav.setModel(BasicModel.buildOKModel().setResult(houses)); + return mav; + } + String getPicUrl(String url, List urls) { if (!urls.contains(url)) { urls.add(url); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PdsController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PdsController.java index 26d60ec4e..8ebc0e92f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PdsController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PdsController.java @@ -2259,11 +2259,11 @@ public class PdsController { averageStatisticsMap.put("安居客", beforeAvgSta.getAnjukeEsfPrice()); } } - if(averageStatistics.getCentanetEsfPrice()!=null){ - averageStatisticsMap.put("中原",averageStatistics.getCentanetEsfPrice()); + if(averageStatistics.getZhongyuanEsfPrice()!=null){ + averageStatisticsMap.put("中原",averageStatistics.getZhongyuanEsfPrice()); }else{ if(beforeAvgSta!=null){ - averageStatisticsMap.put("中原",beforeAvgSta.getCentanetEsfPrice()); + averageStatisticsMap.put("中原",beforeAvgSta.getZhongyuanEsfPrice()); } } if(averageStatistics.getGanjiEsfPrice()!=null){ @@ -2284,7 +2284,7 @@ public class PdsController { if(beforeAvgSta!=null){ averageStatisticsMap.put("房天下", beforeAvgSta.getSoufangEsfPrice()); averageStatisticsMap.put("安居客", beforeAvgSta.getAnjukeEsfPrice()); - averageStatisticsMap.put("中原",beforeAvgSta.getCentanetEsfPrice()); + averageStatisticsMap.put("中原",beforeAvgSta.getZhongyuanEsfPrice()); averageStatisticsMap.put("赶集",beforeAvgSta.getGanjiEsfPrice()); averageStatisticsMap.put("链家", beforeAvgSta.getLianjiaEsfPrice()); } @@ -3092,7 +3092,8 @@ public class PdsController { * 涨跌幅在城市排名 * @author yinjihuan * @param dto - * @param key + * @param model + * @param districtAnalyData */ private void buildUpDownRangecityLevel(EvaluationBeforeLoanDto dto, CollateralEvaluationInfoModel model, DistrictAnalyData districtAnalyData) { Double increaseRate = 0d; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PdsTotalScoreController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PdsTotalScoreController.java index 80cbfc213..a5e9a48a2 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PdsTotalScoreController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PdsTotalScoreController.java @@ -1264,11 +1264,11 @@ public class PdsTotalScoreController { averageStatisticsMap.put("安居客", beforeAvgSta.getAnjukeEsfPrice()); } } - if(averageStatistics.getCentanetEsfPrice()!=null){ - averageStatisticsMap.put("中原",averageStatistics.getCentanetEsfPrice()); + if(averageStatistics.getZhongyuanEsfPrice()!=null){ + averageStatisticsMap.put("中原",averageStatistics.getZhongyuanEsfPrice()); }else{ if(beforeAvgSta!=null){ - averageStatisticsMap.put("中原",beforeAvgSta.getCentanetEsfPrice()); + averageStatisticsMap.put("中原",beforeAvgSta.getZhongyuanEsfPrice()); } } if(averageStatistics.getGanjiEsfPrice()!=null){ @@ -1289,7 +1289,7 @@ public class PdsTotalScoreController { if(beforeAvgSta!=null){ averageStatisticsMap.put("房天下", beforeAvgSta.getSoufangEsfPrice()); averageStatisticsMap.put("安居客", beforeAvgSta.getAnjukeEsfPrice()); - averageStatisticsMap.put("中原",beforeAvgSta.getCentanetEsfPrice()); + averageStatisticsMap.put("中原",beforeAvgSta.getZhongyuanEsfPrice()); averageStatisticsMap.put("赶集",beforeAvgSta.getGanjiEsfPrice()); averageStatisticsMap.put("链家", beforeAvgSta.getLianjiaEsfPrice()); } @@ -2011,7 +2011,8 @@ public class PdsTotalScoreController { * 涨跌幅在城市排名 * @author yinjihuan * @param dto - * @param key + * @param model + * @param districtAnalyData */ private void buildUpDownRangecityLevel(EvaluationBeforeLoanDto dto, CollateralEvaluationInfoModel model, DistrictAnalyData districtAnalyData) { Double increaseRate = 0d; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 1874f3500..3080eefd4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -33,13 +33,13 @@ import com.fangjia.dal.*; import com.fangjia.dal.mongo.model.*; import com.fangjia.dal.mongo.open.model.CommunityBasePrice; import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.dal.mongo.open.model.EvaluateRatio; import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.open.dto.*; import com.fangjia.service.district.*; import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.open.*; -import com.fangjia.service.open.impl.CommunityPoiServiceImpl; import com.fangjia.service.util.DistrictScoreUtil; import net.sf.json.JSONArray; import net.sf.json.JSONObject; @@ -193,6 +193,7 @@ public class PropertyController { private DistrictRoomSqmetreRentService districtRoomSqmetreRentService; private RentDealPriceService rentDealPriceService; + private EvaluateRatioService evaluateRatioService; private static final int EVALUATE_SHOW_DETAIL = 1; private static final Map PROPERTY_TYPES = new HashMap() { @@ -1017,6 +1018,8 @@ public class PropertyController { } catch (IllegalAccessException e) { e.printStackTrace(); } + + this.adjustPriceByRatio(district, evaluation); } // 调整面积不同面积价格 @@ -1042,7 +1045,7 @@ public class PropertyController { } } } catch (Exception ex) { - + LOGGER.error("调整面积不同面积价格异常{}", ex.getMessage()); } if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0 @@ -1101,16 +1104,7 @@ public class PropertyController { districtPriceMap.put("credit", evaluation.getCredit()); districtPriceMap.put("id", evaluation.getId()); } - - ApiUserCfg cfg = apiUserCfgService.findByUserId(Long.valueOf(evaluation.getFromUserId()), ApiUserCfg.ApiUserCfgKey.PRICE_TYPE); - if(cfg != null &&cfg.getCfgValue() != null && StringUtils.equalsIgnoreCase("2B", cfg.getCfgValue().toString())){ - if(evaluation != null && evaluation.getAvgPrice2b() != 0){ - districtPriceMap.put("avgPrice", evaluation.getAvgPrice2b()); - } - } - mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap)); - try{ if(StringUtils.isNotBlank(evaluation.getFromUserId()) && !evaluation.getFromUserId().equals("1007497")){ openApiLogService.saveLog(CommonUtils.convertToMap(model), districtPriceMap, evaluation, "/property/evaluate"); @@ -1128,6 +1122,41 @@ public class PropertyController { } } + /** + * 系数下调 + * + * @param district + * @param evaluation + */ + private void adjustPriceByRatio(District district, Evaluation evaluation) { + + try { + ApiUserCfg apiUserCfg_pt = apiUserCfgService.findByUserId(Long.valueOf(evaluation.getFromUserId()), ApiUserCfg.ApiUserCfgKey.PRICE_TYPE); + if (apiUserCfg_pt != null && apiUserCfg_pt.getCfgValue() != null) { + String priceType = apiUserCfg_pt.getCfgValue().toString(); + // 评估价则系数调整 + if (StringUtils.isNotBlank(priceType) && "evaluate".equals(priceType)) { + EvaluateRatio evaluateRatio = null; + if (StringUtils.isNotBlank(district.getRegion())) { + evaluateRatio = evaluateRatioService.findByCityAndRegion(district.getCity(), district.getRegion()); + } + if (evaluateRatio == null) { + evaluateRatio = evaluateRatioService.findByCity(district.getCity()); + } + if (evaluateRatio != null) { + // 系数添加 + long totalPrice = evaluation.getBigTotalPrice(); + totalPrice = (long) (totalPrice * (1 + evaluateRatio.getRatio())); + evaluation.setBigTotalPrice(totalPrice); + evaluation.setAvgPrice((int) (totalPrice / evaluation.getArea())); + } + } + } + } catch (Exception ex) { + LOGGER.error("系数下调异常{}", ex.getMessage()); + } + } + static final Map CITY_CORRECTION_FACTOR = new HashMap(); static { CITY_CORRECTION_FACTOR.put("上海", 0.02); @@ -1216,6 +1245,9 @@ public class PropertyController { mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); return mav; } + + this.adjustPriceByRatio(district, evaluation); + // 调整面积不同面积价格 try { DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); @@ -1245,13 +1277,7 @@ public class PropertyController { districtPriceMap.put("city", district.getCity()); districtPriceMap.put("district", district.getRegion()); districtPriceMap.put("name", district.getName()); - long totalPrice = evaluation.getBigTotalPrice(); - // 取消系数下调 -// if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){ -// totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity()))); -// evaluation.setBigTotalPrice(totalPrice); -// evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea())); -// } + districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); districtPriceMap.put("unitPrice", evaluation.getAvgPrice()); // 新增置信度 @@ -5757,6 +5783,10 @@ public class PropertyController { this.rentDealPriceService = rentDealPriceService; } + public void setEvaluateRatioService(EvaluateRatioService evaluateRatioService) { + this.evaluateRatioService = evaluateRatioService; + } + public void setCommunityPoiService(CommunityPoiService communityPoiService) { this.communityPoiService = communityPoiService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 272979fca..82bf9b1ce 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -670,6 +670,7 @@ public class WeiXinEvaluationController { model.setUserTrades(userTrade); if(fjtUser!=null && fjtUser.getRole()==17 && district.getCity().equals("上海")){ + System.out.println("渤海用户:"+fjtUser.getEmail()); Map retMap = getLoanInfo(district, evaluation.getHouseArea()); if(retMap!=null && retMap.size()>0){ model.setLoanInfoMap(retMap); @@ -918,6 +919,19 @@ public class WeiXinEvaluationController { return jsonMmodel; } + public Object saveBhLoanAction(EvaluationModel model){ + JsonModel jsonMmodel = new JsonModel(); + if(StringUtils.isNotBlank(model.getEvaluationId())){ + FjtEvaluation fjtEvaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId())); + if(fjtEvaluation != null && fjtEvaluation.getLoan()!=model.getLoan()){ + fjtEvaluation.setLoan(model.getLoan()); + fjtEvaluation.setLastUpdated(new Date()); + fjtEvaluationService.updateEvaluation(fjtEvaluation); + } + } + return jsonMmodel; + } + //刷新审核价 public Object refreshAuditPriceAction(EvaluationModel model){ JsonModel jsonMmodel = new JsonModel(); @@ -1106,9 +1120,15 @@ public class WeiXinEvaluationController { public Object checkImageAction(EvaluationModel model){ JsonModel jsonModel = new JsonModel(); if (StringUtils.isNotBlank(model.getEvaluationId())){ - List datas = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 3); + List datas3 = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 3); Map data = new HashMap(); - data.put("count", datas.size()); + data.put("count_3", datas3.size()); + + FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId())); + if(evaluation!=null && evaluation.getLoan()!=null && evaluation.getLoan()==1){ + List datas5 = reportFileAttachService.findHouseImage(Integer.valueOf(model.getEvaluationId()), 5); + data.put("count_5", datas5.size()); + } jsonModel.setJsonModel(data); } return jsonModel; @@ -1226,6 +1246,9 @@ public class WeiXinEvaluationController { } } Integer yearAge = getBuildYear(district.getFinishBuildingDate()); + if(pbObj!=null){ + System.out.println("渤海银行:"+district.getName() +" 年限:"+yearAge+" Level:"+pbObj.getLevel()+" 所属板块:"+pbObj.getName()); + } Map retsMap = new HashMap(); if(pbObj!=null && pbObj.getLevel()!=null && pbObj.getLevel()==1 && yearAge!=null && yearAge<=25 && area>=60){ retsMap.put("type", "A"); @@ -1243,6 +1266,7 @@ public class WeiXinEvaluationController { retsMap.put("type", "D"); retsMap.put("loan_ratio", 5); } + return retsMap; } @@ -1287,7 +1311,7 @@ public class WeiXinEvaluationController { FjtUsers fjtUser = fjtUsersService.findById(Long.valueOf(userId)); UsersDetail usersDetail = usersDetailService.findById(fjtUser.getId()); try { - String id = new ObjectId().toString(); + List uploadfiles = new ArrayList(); for (CommonsMultipartFile file : files) { if (file.getSize() == 0 ) { @@ -1319,6 +1343,8 @@ public class WeiXinEvaluationController { } catch (IOException e) { //处理异常信息 } + String id = new ObjectId().toString(); + reportFileAttach = new ReportFileAttach(); reportFileAttach.setEvaluationId(Integer.valueOf(model.getEvaluationId())); reportFileAttach.setQueryId(model.getQueryId()); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java index 8f1311061..64fb38772 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java @@ -48,6 +48,7 @@ public abstract class SearchModel extends FangjiaApiModel { private Double area; protected Integer dataPeriod;//查询天数 + private Integer distCount; /** * 获取房源数量类型 sell:只取得二手房数量 rent:只取得出租房数量 其他:不获取数量 @@ -376,4 +377,12 @@ public abstract class SearchModel extends FangjiaApiModel { public void setEndTime(Integer endTime) { this.endTime = endTime; } + + public Integer getDistCount() { + return (null != distCount && 0 != distCount) ? distCount : 20; + } + + public void setDistCount(Integer distCount) { + this.distCount = distCount; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java index 8de0977f8..20f093053 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/weixin/EvaluationModel.java @@ -45,6 +45,7 @@ public class EvaluationModel extends BaseModel{ private Map loanInfoMap; private Integer hopePrice; private String remark; + private Integer loan; // private String preCode;//渤海接口--楼盘编码 // private String preName;//渤海接口--楼盘名称 // private String countryCode;//渤海接口--区域编码 @@ -317,4 +318,12 @@ public class EvaluationModel extends BaseModel{ public void setAddress(String address) { this.address = address; } + + public Integer getLoan() { + return loan; + } + + public void setLoan(Integer loan) { + this.loan = loan; + } } diff --git a/projects/fangjia-open/src/main/webapp/404.json b/projects/fangjia-open/src/main/webapp/404.json new file mode 100644 index 000000000..79ec33f0d --- /dev/null +++ b/projects/fangjia-open/src/main/webapp/404.json @@ -0,0 +1,4 @@ +{ + "code": 400, + "msg": "Interface Not Found" +} \ No newline at end of file diff --git a/projects/fangjia-open/src/main/webapp/500.json b/projects/fangjia-open/src/main/webapp/500.json new file mode 100644 index 000000000..e17a904c0 --- /dev/null +++ b/projects/fangjia-open/src/main/webapp/500.json @@ -0,0 +1,4 @@ +{ + "code": 500, + "msg": "Server Inner Error" +} \ No newline at end of file diff --git a/projects/fangjia-open/src/main/webapp/WEB-INF/web.xml b/projects/fangjia-open/src/main/webapp/WEB-INF/web.xml index 47a7251fb..0fd12bdf7 100644 --- a/projects/fangjia-open/src/main/webapp/WEB-INF/web.xml +++ b/projects/fangjia-open/src/main/webapp/WEB-INF/web.xml @@ -90,4 +90,12 @@ SafeCode /randomCode + + 500 + /500.json + + + 404 + /404.json + diff --git a/projects/fangjia-open/src/main/webapp/static/js/report.js b/projects/fangjia-open/src/main/webapp/static/js/report.js index 92ef631b5..b61a6e2d1 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/report.js +++ b/projects/fangjia-open/src/main/webapp/static/js/report.js @@ -190,6 +190,36 @@ $('#submitRemark').on('click', function () { } }); +$('#submitLoan').on('click', function () { + var opt = $("#submitLoan").text(); + if(opt == "淇敼"){ + $(this).parent().find("span").eq(1).hide() + $("input[name=loan]").removeClass("dn"); + $("#submitLoan").text("淇濆瓨"); + }else{ + var loan = $("input[name=loan]").val(); + if(loan == ""){ + $.toast("璇疯緭鍏ユ槸鍚︾画璐�"); + }else{ + $.ajax({ + url:"saveBhLoan", + data:{ + evaluationId:eId, + loan:loan + }, + dataType:"json", + success:function(data){ + $("#submitLoan").remove(); + $("input[name=loan]").remove(); + $("#refreshPrice0").show(); + location.href ="bhreport?evaluationId="+eId; + } + }) + } + } +}); + + $('#refreshPrice,#refreshPrice0').on('click', function () { $.ajax({ url:"refreshAuditPrice", @@ -301,21 +331,34 @@ $('.download_btn').on('click', function () { /* 涓嬭浇璇勪及鎶ュ憡 */ $('.formal_btn').on('click', function () { - $.ajax({ - url:"checkImage", - data:{ - evaluationId:eId - }, - dataType:"json", - success:function(data){ - if(data && data.count>0){ - clickFormalReport() - }else{ - $.alert('璇蜂笂浼犳埧灞嬬収鐗囷紒'); + if(isLoan!=null) { + $.ajax({ + url:"checkImage", + data:{ + evaluationId:eId + }, + dataType:"json", + success:function(data){ + if(isLoan==1){ + if(data.count_5==0){ + $.alert('璇蜂笂浼犱骇璋冩姤鍛婃垨鏉冪睄锛�'); + }else if(data.count_3==0){ + $.alert('璇蜂笂浼犳埧灞嬬収鐗囷紒'); + }else{ + clickFormalReport() + } + }else{ + if(data.count_3==0){ + $.alert('璇蜂笂浼犳埧灞嬬収鐗囷紒'); + }else{ + clickFormalReport() + } + } } - } - }) - + }) + }else{ + $.alert('璇烽�夋嫨鏄惁缁捶锛�'); + } }); function clickFormalReport(){ diff --git a/projects/fangjia-open/src/main/webapp/static/style/wx.css b/projects/fangjia-open/src/main/webapp/static/style/wx.css index 269a1a00e..0e85e8a93 100644 --- a/projects/fangjia-open/src/main/webapp/static/style/wx.css +++ b/projects/fangjia-open/src/main/webapp/static/style/wx.css @@ -134,7 +134,7 @@ input::-webkit-input-placeholder{ .inputTopic01{margin-bottom:-1px;} .inputTopic01 .topicTitle,.inputTopic02 .topicTitle{margin-right:0;border:none;line-height:2.5rem;} .inputTopic01 .topicTitle .input_price{width:20%;padding-left:5px;} -.inputTopic02 .topicTitle .input_lender{width:40%;padding-left:5px;} +.inputTopic02 .topicTitle .input_lender{width:39%;padding-left:5px;line-height: 28px;} .inputTopic02 .topicTitle .input_cardId{width:35%;padding-left:5px;} .submit_btn{width:90px;border-left:solid 1px #e7e7e7;float:right;padding-right:0;text-align:center;cursor:pointer;} .left01{float:left;margin-right:8%;} diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index 165012ff6..f16637105 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -132,6 +132,18 @@ +
    + +
    + 娉細濡傚嚭鍏锋寮忔姤鍛婃椂锛岄渶涓婁紶浜ц皟鎶ュ憡鎴栨潈绫� +
    +
    +
    @@ -139,7 +151,7 @@ <#if reportFileMap??>
    宸蹭笂浼犳枃浠讹細 - <#if reportFileMap["1"]??>璇环鍗�(${reportFileMap["1"]?size}) + <#if reportFileMap["5"]??>浜ц皟鎶ュ憡鎴栨潈绫�(${reportFileMap["5"]?size}) <#if reportFileMap["2"]??> <#if reportFileMap["3"]??> <#if reportFileMap["4"]??> @@ -456,6 +468,10 @@ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy index d8437413e..dae53f491 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/averager/InsertMonitorSquareRentCalcCommandBatch.groovy @@ -16,6 +16,7 @@ import com.fangjia.script.util.ServerAreaMap import com.google.common.collect.MapDifference import com.google.common.collect.Maps import com.mongodb.* +import org.apache.commons.lang.StringUtils import org.bson.types.ObjectId /** @@ -47,9 +48,19 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { } void process(Object args) { - def cities = ServerAreaMap.findAllCities() + String cityParam = System.getProperty("city"); + String calcDate = System.getProperty("calcDate"); + def cities; + if(StringUtils.isNotBlank(cityParam)){ + cities = new ArrayList() + for(String c in cityParam.split(",")){ + cities.add(c); + } + }else{ + cities = ServerAreaMap.findAllCities() + } println cities - Date weekend = getLastDayOfMonth() + Date weekend = getLastDayOfMonth(calcDate) println weekend int cityCount = 0 int regionCount = 0 @@ -58,31 +69,22 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { //开始时间 final Date startDate = new Date() for (String city : cities) { - List idList = getAllDistrictIds(city) - logInfo(city + ":" + idList.size()) - + List districts = getAllDistricts(city) + logInfo(city + ":" + districts.size()) def buffer = [] //生成计算小区队列 - for (String id : idList) { - DBObject fields = new BasicDBObject() - fields.put("city", 1) - fields.put("region", 1) - fields.put("name", 1) - fields.put("estate_type2", 1) - DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields) - if (district != null) { - String region = district.get('region') - String name = district.get('name') - String estateType = district.get('estate_type2') - districtCount++ - String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()] - BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip) - logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())) - buffer << cmd - if(buffer.size() == 1000){ - COMMAND_COLL.insert(buffer) - buffer.clear() - } + for (BasicDBObject district : districts) { + String region = district.get('region') + String name = district.get('name') + String estateType = district.get('estate_type2') + districtCount++ + String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()] + BasicDBObject cmd = saveDistrictCalcCommand(city, region, name, estateType, weekend, ip) + logInfo("district:" + city + "-" + region + "-" + name + "-" + estateType + ",weekend:" + DateUtil.formatToYYYYMMDD(weekend) + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())) + buffer << cmd + if(buffer.size() == 1000){ + COMMAND_COLL.insert(buffer) + buffer.clear() } } if(!buffer.isEmpty()){ @@ -148,7 +150,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { query.append("weekend", weekend) query.append("cat", "block") blockCount = SQMETRE_RENT_COLL.count(query) - def msg = "本月小区平米租金计算量:" + districtCount + ";本月城市平米租金计算量:" + cityCount + ";本月区域平米租金计算量:" + regionCount + ";本月版块平米租金计算量:" + blockCount + ";运行时间:" + secondToTime(time) + "." + def msg = "本月(${weekend.format("yyyyMM")})小区平米租金计算量:".toString() + districtCount + ";本月城市平米租金计算量:" + cityCount + ";本月区域平米租金计算量:" + regionCount + ";本月版块平米租金计算量:" + blockCount + ";运行时间:" + secondToTime(time) + "." sendDingDingMsg(TOKEN, msg) } /** @@ -209,31 +211,34 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { } - private List getAllDistrictIds(String city) { - DBObject q = new BasicDBObject() - q.put("city", city) - q.put("cat", "district") - q.put("visible", 0) + private List getAllDistricts(String city) { + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) - int trytimes = 5 - Set uniqueIdList = new LinkedHashSet() + DBObject fields = new BasicDBObject() + fields.put("city", 1) + fields.put("region", 1) + fields.put("name", 1) + fields.put("estate_type2", 1) + fields.put("_id", 1) - for(int i=0; i districts = new ArrayList() + for(int i=0; i < tryTimes; i++) { + DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000) int totalCount = cur.count() Iterator iter = cur.iterator() while(iter.hasNext()) { - DBObject dbo = iter.next() - String id = dbo.get('_id') - uniqueIdList.add(id) + BasicDBObject dbo = (BasicDBObject)iter.next() + districts.add(dbo) } - - if(uniqueIdList.size() == totalCount) { + if(districts.size() == totalCount) { break } } - - return new ArrayList(uniqueIdList) + return districts } private List getRegions(String city) { @@ -316,7 +321,13 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch { } - private Date getLastDayOfMonth() { + private Date getLastDayOfMonth(String calcDateStr) { + if(StringUtils.isNotBlank(calcDateStr)){ + Date calcDate = DateUtil.parseDate(calcDateStr, "yyyyMM") + if(calcDate != null){ + return DateUtil.truncateDay(DateUtil.getLastDayOfMonth(calcDate, 0)) + } + } return DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/DistrictRegionTransfer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/DistrictRegionTransfer.groovy index 9d0abb578..12717114f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/DistrictRegionTransfer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/DistrictRegionTransfer.groovy @@ -15,7 +15,8 @@ import org.apache.commons.lang.StringUtils public class DistrictRegionTransfer { - static final DBCollection TASK_COLL = MongoUtils.getCollectionByName("fangjia", "district_update_task", "mongo.db.fangjia.com") + static final DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection SQMETRE_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com") static final DBCollection REGION_MIGRATE_COLL = MongoUtils.getCollectionByName("fangjia_log", "region_migrate", "mongo.db.fangjia.com") static final List IGNORE_FIELDS = ["_id", "region", "city", "flag", "estate_type2", "c_date", "m_date", "other_info", "mark_map", "addr_alias", "alias", "name", "address", "status"] static List ALL_CITY = CommonUtils.findAllCity() @@ -31,15 +32,24 @@ public class DistrictRegionTransfer { copyDistrictInfo(orgDistrict, destDistrict) DbUtils.seaweed.save(destDistrict) DbUtils.deleteOnSeaweed(orgDistrict._id.toString()) - sendRelatedInfoMigrateCommand(orgDistrict, destDistrict) saveMigrateLog(orgDistrict, destDistrict) } else { destDistrict = (BasicDBObject) orgDistrict.clone() destDistrict.city = newCity destDistrict.region = newRegion DbUtils.seaweed.save(destDistrict) - sendRelatedInfoMigrateCommand(orgDistrict, destDistrict) saveMigrateLog(orgDistrict, destDistrict) + DISTRICT_STAT.remove(new BasicDBObject("city", newCity).append("region", newRegion).append("district_name", orgDistrict.name)) + DISTRICT_STAT.find(new BasicDBObject("city", newCity).append("region", orgDistrict.region).append("district_name", orgDistrict.name)).each {BasicDBObject stat-> + stat.region = newRegion + DISTRICT_STAT.save(stat) + } + SQMETRE_RENT.remove(new BasicDBObject("city", newCity).append("region", newRegion).append("district_name", orgDistrict.name)) + SQMETRE_RENT.find(new BasicDBObject("city", newCity).append("region", orgDistrict.region).append("district_name", orgDistrict.name)).each {BasicDBObject stat-> + stat.region = newRegion + SQMETRE_RENT.save(stat) + } + } } @@ -60,72 +70,72 @@ public class DistrictRegionTransfer { } } - def sendRelatedInfoMigrateCommand(BasicDBObject orgDistrict, BasicDBObject destDistrict) { - - BasicDBObject task = new BasicDBObject() - task.append("city", orgDistrict.city) - task.append("new_city", destDistrict.city) - task.append("region", orgDistrict.region) - task.append("new_region", destDistrict.region) - task.append("new_name", orgDistrict.name) - task.append("name", destDistrict.name) - task.append("type", 2) - task.append("c_date", new Date()) - task.append("status", 1) - String estateType; - if ("district" == orgDistrict.cat) { - if (StringUtils.isNotBlank(destDistrict.estate_type2)) { - if (destDistrict.getString("estate_type2").contains("别墅")) { - estateType = "villa" - } else { - estateType = "district" - } - } else { - if (StringUtils.isNotBlank(destDistrict.estate_type2)) { - if (destDistrict.getString("estate_type").contains("别墅")) { - estateType = "villa" - } else { - estateType = "district" - } - } else { - estateType = "district" - } - } - } else { - estateType = destDistrict.getString("cat") - } - task.append("estate_type", estateType) - def query = new BasicDBObject() - query.append("city", task.city) - query.append("region", task.region) - query.append("name", task.name) - query.append("type", task.type) - TASK_COLL.update(query, new BasicDBObject('$set', task), true, false) - - def params = [:] - params.city = orgDistrict.city - params.region = orgDistrict.region - params.name = orgDistrict.name - params.estateType = orgDistrict.estate_type2 - - params.newCity = destDistrict.city - params.newRegion = destDistrict.region - params.newName = destDistrict.name - params.newEstateType = destDistrict.estate_type2 - int count = 3 - while (count-- > 0) { - try { - JSONObject json = new HttpRequest().getJsonContent("http://192.168.0.24:9080/command/transfer", params, "UTF-8") - if (json && json.getInt("code") == 200) { - println("均价重算命令发从成功:" + params) - break - } - } catch (Exception e) { - - } - - } - } +// def sendRelatedInfoMigrateCommand(BasicDBObject orgDistrict, BasicDBObject destDistrict) { +// +// BasicDBObject task = new BasicDBObject() +// task.append("city", orgDistrict.city) +// task.append("new_city", destDistrict.city) +// task.append("region", orgDistrict.region) +// task.append("new_region", destDistrict.region) +// task.append("new_name", orgDistrict.name) +// task.append("name", destDistrict.name) +// task.append("type", 2) +// task.append("c_date", new Date()) +// task.append("status", 1) +// String estateType; +// if ("district" == orgDistrict.cat) { +// if (StringUtils.isNotBlank(destDistrict.estate_type2)) { +// if (destDistrict.getString("estate_type2").contains("别墅")) { +// estateType = "villa" +// } else { +// estateType = "district" +// } +// } else { +// if (StringUtils.isNotBlank(destDistrict.estate_type2)) { +// if (destDistrict.getString("estate_type").contains("别墅")) { +// estateType = "villa" +// } else { +// estateType = "district" +// } +// } else { +// estateType = "district" +// } +// } +// } else { +// estateType = destDistrict.getString("cat") +// } +// task.append("estate_type", estateType) +// def query = new BasicDBObject() +// query.append("city", task.city) +// query.append("region", task.region) +// query.append("name", task.name) +// query.append("type", task.type) +// TASK_COLL.update(query, new BasicDBObject('$set', task), true, false) +// +// def params = [:] +// params.city = orgDistrict.city +// params.region = orgDistrict.region +// params.name = orgDistrict.name +// params.estateType = orgDistrict.estate_type2 +// +// params.newCity = destDistrict.city +// params.newRegion = destDistrict.region +// params.newName = destDistrict.name +// params.newEstateType = destDistrict.estate_type2 +// int count = 3 +// while (count-- > 0) { +// try { +// JSONObject json = new HttpRequest().getJsonContent("http://192.168.0.24:9080/command/transfer", params, "UTF-8") +// if (json && json.getInt("code") == 200) { +// println("均价重算命令发从成功:" + params) +// break +// } +// } catch (Exception e) { +// +// } +// +// } +// } def saveMigrateLog(BasicDBObject orgDistrict, BasicDBObject destDistrict) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/RegionTransfer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/RegionTransfer.groovy index b4d6580d9..397ae2c8e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/RegionTransfer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/RegionTransfer.groovy @@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.* import com.fangjia.core.util.Mongo3Utils -def city = "洛阳" -def destCity = "洛阳" -def region = "吉利" -def destRegion = "孟津" +def city = "中山" +def destCity = "中山" +def region = "火炬" +def destRegion = "中山港" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 235253d60..96febc0f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -1,6 +1,39 @@ package com.fangjia.script.wenyan +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\国信达级联价格-上海.xlsx",{List item-> +// if(item["AB"]){ +// return +// } +// String region = null +// String name = item["F"] +// if(name.endsWith("小区")){ +// name = name.replaceAll("小区\$", "") +// } +// region = RegionData.formatRegion(item["B"], item["D"]) +// def results = [] +// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region, name)).each { +// if(!results.contains(it)){ +// results.add(it) +// } +// } +// DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(item["B"], region).append("alias", name)).each { +// if(!results.contains(it)){ +// results.add(it) +// } +// } +// if(results.size() == 1){ +// item["Z"] = results[0]._id.toString() +// item["AA"] = results[0].region +// item["AB"] = results[0].name +// println(item) +// } +//}) + + + + import com.fangjia.core.ProvinceData +import com.fangjia.core.RegionData import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils @@ -12,6 +45,9 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilders + //DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) //def count = 0 //List> data = Collections.synchronizedList([]) @@ -76,7 +112,77 @@ import org.apache.commons.lang.StringUtils // }, new BasicDBObject("township", 1)) //} //println(count) +//def GXD_CCB = [:] +//def CCB_FJW = [:] +//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",sheetIndex = 1).each {List item-> +// if(item["V"] == "级联"){ +// if(item["Y"] == "国信达"){ +// GXD_CCB[item['J']] = item["E"] +// }else if(item["Y"] == "房价网"){ +// CCB_FJW[item['E']] = item["J"] +// } +// } +//} +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\4月级联待优化小区_3个城市.xlsx",{List item-> +// if(item["V"] == "级联" && item["Y"] == "国信达"){ +// def ccbId = GXD_CCB.get(item['J'], null) +// def fjwId = CCB_FJW.get(ccbId, null) +// if(fjwId != null){ +// item["AB"] = fjwId +//// BasicDBObject seaweed = DbUtils.seaweed.findById(fjwId) +//// List results = DbUtils.findAroundDistrict(seaweed) +//// long total = 0 +//// long count = 0 +//// for(BasicDBObject s : results){ +//// def avg = DbUtils.findDistrictAveragePrice(seaweed.city, seaweed.region, seaweed.name, Date.parse('yyyyMMdd', '20220619'), seaweed.estate_type2) +//// if(avg){ +//// total += avg.avg_price +//// count += 1 +//// } +//// } +//// if(count > 0){ +//// item['Z'] = (int)(total/count) +//// println(item) +//// } +// } +// } +//},sheetIndex = 1) -ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{ - println(it) +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\小区字段补充.xlsx",{List item-> + BasicDBObject seaweed = DbUtils.seaweed.findById(item["A"]) + if(StringUtils.isBlank(seaweed.create_year)){ + def names = [] + names.add(seaweed.name) + if(seaweed.alias){ + for(def it in seaweed.alias){ + if(StringUtils.isNotBlank(it) && !names.contains(it)){ + names.add(it) + } + } + } + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery() + boolQueryBuilder.must(QueryBuilders.termQuery("city", seaweed.city)) + boolQueryBuilder.must(QueryBuilders.termQuery("region", seaweed.region)) + boolQueryBuilder.must(QueryBuilders.termsQuery("name", names)); + item["B"] = seaweed.city + item["C"] = seaweed.region + item["D"] = seaweed.name + List result = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", boolQueryBuilder) + def buildYears = [] + result.each { + if(StringUtils.isNotBlank(it.build_year) && !StringUtils.containsAny(it.build_year,"无")){ + String build_year = it.build_year + build_year = build_year.replaceAll("((|\\().*()|\\))", "") + build_year = build_year.replaceAll("[年建]", "") +// property_fee = property_fee.replace("元/㎡·月", "元/平米/月") + buildYears.add(build_year) + } + } + buildYears = buildYears.unique() + if(buildYears.size() > 0){ + DbUtils.seaweed.update(new BasicDBObject("_id", seaweed._id), new BasicDBObject('$set', new BasicDBObject('create_year', buildYears[0]))) + println(item) + println(buildYears) + } + } }) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/TongCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/TongCheng.groovy new file mode 100644 index 000000000..a0e5faf71 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/TongCheng.groovy @@ -0,0 +1,151 @@ +package com.fangjia.script.wenyan + +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import net.sf.json.JSONArray +import net.sf.json.JSONObject +import org.bson.Document + +//def data = ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\TC重新跑数据.xlsx") +//def addMap = [:] +//data.each {def list-> +// if(list["C"] == "有效"){ +// addMap[list["B"]] = list +// } +//} +//ExcelUtils.modify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx",{List item-> +// if(addMap.containsKey(item["B"])){ +// List list = addMap.get(item["B"]) +// item["C"] = list["D"] +// item["D"] = list["E"] +// item["E"] = list["F"] +// item["S"] = "有效" +// println(item) +// } +//}) + +def DISTRICT_ANALY_DATA = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data") +ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\data_gj20220616.xlsx", { List item -> +// if(item["S"] == null){ +// if(item["Q"] != null || item["R"] != null){ +// item["S"] = "有效" +// } +// } +// if(item["S"] == null){ +// def source = null +// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["B"], "property") +// source = "小区库" +// if(resp.credit == 0){ +// resp = OpenApiClient.addressMatch(item["A"], item["B"], null) +// source = "小区库" +// } +// if(resp.credit == 0){ +// resp = OpenApiClient.addressMatch(item["A"], item["B"], null, true) +// source = "地图" +// } +// +// if(resp.credit > 0){ +// println(resp) +// item["C"] = resp.city +// item["D"] = resp.district +// item["E"] = resp.name +// item["F"] = source +// item["G"] = resp.lng +// item["H"] = resp.lat +// item["I"] = resp.propertyType +// } +// if(item["G"] != null && !["普通住宅", "商住", "别墅"].contains(item["I"])){ +// List seaweeds = DbUtils.findAroundDistrict(item["G"], item["H"], 1000 ,1, 2) +// if(seaweeds.size() > 0){ +// BasicDBObject s = seaweeds[0] +// item["C"] = s.city +// item["D"] = s.region +// item["E"] = s.name +// item["F"] = "周边" +// item["G"] = s.lng/10000000000d +// item["H"] = s.lat/10000000000d +// item["I"] = s.estate_type2 +// } +// } +// } + + + +// if(item["S"]|| item["T"]){ +// if(!item["i"]){ +// JSONObject result = GaoDeApiUtils.geo(item["A"], item["B"]) +// if(result != null && result.getInt("status") == 1){ +// JSONArray geocodes = result.getJSONArray("geocodes"); +// if(geocodes != null && geocodes.size() > 0){ +// JSONObject geocode = geocodes.getJSONObject(0); +// String location = geocode.getString("location") +// item["i"] = location.split(",")[0].toDouble() +// item["j"] = location.split(",")[1].toDouble() +// } +// } +// println(item) +// } +// } +// if(item["D"] == null && item["F"] != null){ +// String kw = item["F"] +// if(item["E"]){ +// kw = item["E"] + kw +// } +// JSONObject result = GaoDeApiUtils.geo(item["A"], kw) +// if(result != null && result.getInt("status") == 1){ +// JSONArray geocodes = result.getJSONArray("geocodes"); +// if(geocodes != null && geocodes.size() > 0){ +// JSONObject geocode = geocodes.getJSONObject(0); +// String location = geocode.getString("location") +// item["i"] = location.split(",")[0].toDouble() +// item["j"] = location.split(",")[1].toDouble() +// } +// } +// println(item) +// } +// if(item["E"] != null && ["普通住宅", "商住", "别墅"].contains(item["I"])){ +// Document doc = DISTRICT_ANALY_DATA.find(new Document("city", item["C"]).append("region", item["D"]).append("district_name", item["E"])).first() +// if(doc){ +// item["N"] =doc["rank"] +// } +// } +// if (item["S"] != "有效" || item["F"] != null) { +// return +// } +// JSONObject resp = OpenApiClient.addressMatch(item["A"], item["E"], "property", false, item["D"], null, "strict") +// if(resp.credit > 0){ +// println(resp) +// item["C"] = resp.city +// item["D"] = resp.district +// item["E"] = resp.name +// item["F"] = "小区库" +// item["G"] = resp.lng +// item["H"] = resp.lat +// item["I"] = resp.propertyType +// } +// println(resp) +// if (item["F"] == null) { +// String kw = item["E"] +// if (item["D"]) { +// kw = item["D"] + kw +// } +// JSONObject result = GaoDeApiUtils.geo(item["A"], kw) +// if (result != null && result.getInt("status") == 1) { +// JSONArray geocodes = result.getJSONArray("geocodes"); +// if (geocodes != null && geocodes.size() > 0) { +// JSONObject geocode = geocodes.getJSONObject(0); +// String location = geocode.getString("location") +// item["G"] = location.split(",")[0].toDouble() +// item["H"] = location.split(",")[1].toDouble() +// item["F"] = "反查经纬度" +// } +// } +// println(item) +// } +}) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 15909a4c9..c7c6b08a6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -499,6 +499,7 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setAddressParserGoal(params.getAddressParserGoal()); addr.setProvince(params.getProvince()); addr.setDebugName(params.getDebugName()); + addr.setUseMap(params.isUseMap()); return AddressProcessors.process(addr); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index ed723c1d5..02cee4b25 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -63,6 +63,7 @@ public class Address { private String suggestNamePinYin; private String polygonSeaweedId; private String geoRegion; + private boolean useMap; private List buildingNumbers; private PreciseLevel preciseLevel; private boolean pinYin; @@ -389,6 +390,14 @@ public class Address { isStandardAddress = standardAddress; } + public boolean isUseMap() { + return useMap; + } + + public void setUseMap(boolean useMap) { + this.useMap = useMap; + } + /** * 获得有效地址 * diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AbstractAddressProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AbstractAddressProcessor.java index ad0885831..a7e546f8d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AbstractAddressProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AbstractAddressProcessor.java @@ -133,10 +133,9 @@ public abstract class AbstractAddressProcessor { * @return */ protected String tokenDistrictName(String keyword){ - List tokens = CnTokenizer.tokenizer(keyword, true); - Token token = getTokenByType(tokens, KeywordEnum.district); - if(token != null && "district".equals(token.getKeywordTypes()) && keyword.contains(token.getName())){ - return token.getName(); + List tokens = tokenDistrictNames(keyword); + if(tokens.size() > 0){ + return tokens.get(0); } return null; } @@ -159,7 +158,35 @@ public abstract class AbstractAddressProcessor { protected List tokenDistrictNames(String keyword){ List names = new ArrayList(); List tokens = CnTokenizer.tokenizer(keyword, true); - for (Token token : tokens) { + List mergeTokens = new ArrayList(); + for(int i = 0; i < tokens.size(); i++){ + Token token = tokens.get(i); + mergeTokens.add(token); + if(token.getKeywordTypes().equals("none") && i != tokens.size() - 1){ + for(int j = i +1; j < tokens.size(); j++){ + Token nextToken = tokens.get(j); + if("none".equals(nextToken.getKeywordTypes())){ + token.setEnd(nextToken.getEnd()); + token.setName(token.getName() + nextToken.getName()); + i++; + }else{ + break; + } + } + }else if(token.getKeywordTypes().equals("district") && i != tokens.size() - 1){ + for(int j = i +1; j < tokens.size(); j++){ + Token nextToken = tokens.get(j); + if("partition".equals(nextToken.getKeywordTypes()) || "stage".equals(nextToken.getKeywordTypes()) || "hao_yuan".equals(nextToken.getKeywordTypes())){ + token.setEnd(nextToken.getEnd()); + token.setName(token.getName() + nextToken.getName()); + i++; + }else{ + break; + } + } + } + } + for (Token token : mergeTokens) { if (token.containsType(KeywordEnum.district) && keyword.contains(token.getName())) { names.add(token.getName()); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java index 9097a3a63..d1a15d156 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java @@ -28,8 +28,8 @@ public class AddressProcessors{ addressProcessors.add(new ProvinceProcessor()); addressProcessors.add(new CityProcessor()); addressProcessors.add(new DistrictProcessor()); - addressProcessors.add(new LoopLineProcessor()); addressProcessors.add(new TownshipProcessor()); + addressProcessors.add(new LoopLineProcessor()); addressProcessors.add(new StageAndPartitionProcessor()); addressProcessors.add(new DirectionProcessor()); addressProcessors.add(new OffsetProcessor()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index e3ef9a3da..bb33d7dbd 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -28,6 +28,9 @@ public class GeoProcessor extends AbstractAddressProcessor { if (address.hasCoordinate()) { return RepeatStatus.NEXT; } + if(!address.isUseMap()){ + return RepeatStatus.NEXT; + } String addr = address.getEffectiveAddress(); if(addr == null || addr.length() < 3){ addr = address.getOrgAddress(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TokenProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TokenProcessor.java index a4bd162c2..ef4dfc5af 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TokenProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TokenProcessor.java @@ -19,7 +19,7 @@ public class TokenProcessor extends AbstractAddressProcessor{ for(int i = 0; i < tokens.size(); i++){ Token token = tokens.get(i); mergeTokens.add(token); - if(token.getKeywordTypes().equals("none") && i != tokens.size() - 1){ + if((token.getKeywordTypes().equals("none") || token.getKeywordTypes().equals("stage")|| token.getKeywordTypes().equals("partitions")) && i != tokens.size() - 1){ for(int j = i +1; j < tokens.size(); j++){ Token nextToken = tokens.get(j); if("none".equals(nextToken.getKeywordTypes())){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/KeywordEnum.java b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/KeywordEnum.java index 4b3b0e770..c7fd58315 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/KeywordEnum.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/KeywordEnum.java @@ -18,8 +18,11 @@ public enum KeywordEnum { estate, // 房产类 channel, // 房价网频道 house_number, // 室号 + stage, // 分期 + partition, // 分区 town,//乡镇 village,//居委会 + hao_yuan, number_cn, total_price, room, area, hall, toilet, alpha; public static KeywordEnum parse(String val) { diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/PatternTokenizer.java b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/PatternTokenizer.java index 447ae9cc6..ca4d2176f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/PatternTokenizer.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/PatternTokenizer.java @@ -14,6 +14,9 @@ public class PatternTokenizer { private static Pattern reg_house_number; // 匹配"5号301室" private static Pattern REG_RANGE_PATT; // 匹配"5号301室" private static Pattern REG_NUMBER_CN; // 匹配面积 + private static Pattern STAGE_PATTERN;// 分期 + private static Pattern PARTITION_PATTERN;// 分区 + private static Pattern HAO_YUAN_PATTERN;// 分区 private static final String REG_NUM = "(((,|\\.|\\d| )*\\d)|(一|二|三|四|五|六|七|八|九|十|零|两|个|十|百|千|万|亿| )+)"; private static final String REG_RANGE_SPLITTER = "(-|~|~|-|到|至| )"; @@ -31,6 +34,9 @@ public class PatternTokenizer { REG_RANGE_PATT = Pattern.compile(REG_RANGE_SPLITTER); REG_NUMBER_CN = Pattern.compile(REG_NUM + "(弄|号|栋|厅|卫)"); + STAGE_PATTERN = Pattern.compile("(([零一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([零一二三四五六七八九十百千\\d]+)期"); + PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|零|一|二|三|四|五|六|七|八|九|十)+(区|坊)"); + HAO_YUAN_PATTERN = Pattern.compile("\\d+号((小区)|院)"); } public static void main(String[] args) { @@ -52,6 +58,9 @@ public class PatternTokenizer { } public static String tokenizers(List tokens, String k) { + k = parsePartition(tokens, k); + k = parseStage(tokens, k); + k = parseHaoYuan(tokens, k); // 面积 k = parseArea(tokens, k); // 房型 @@ -77,6 +86,40 @@ public class PatternTokenizer { } return t; } + private static String parseHaoYuan(List ret, String t) { + String value = null; + Matcher m = HAO_YUAN_PATTERN.matcher(t); + while (m.find()) { + value = m.group(); + t = disableValue(t, value); + ret.add(new Token(value, m.start(), m.end(), KeywordEnum.hao_yuan.name())); + m = HAO_YUAN_PATTERN.matcher(t); + } + return t; + } + private static String parseStage(List ret, String t) { + String value = null; + Matcher m = STAGE_PATTERN.matcher(t); + while (m.find()) { + value = m.group(); + t = disableValue(t, value); + ret.add(new Token(value, m.start(), m.end(), KeywordEnum.stage.name())); + m = STAGE_PATTERN.matcher(t); + } + return t; + } + + private static String parsePartition(List ret, String t) { + String value = null; + Matcher m = PARTITION_PATTERN.matcher(t); + while (m.find()) { + value = m.group(); + t = disableValue(t, value); + ret.add(new Token(value, m.start(), m.end(), KeywordEnum.partition.name())); + m = PARTITION_PATTERN.matcher(t); + } + return t; + } private static String parseArea(List ret, String t) { String value = null; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/EvaluationServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/EvaluationServiceImpl.java index e0b0589f2..817db88d3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/EvaluationServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/EvaluationServiceImpl.java @@ -1042,7 +1042,7 @@ public class EvaluationServiceImpl implements EvaluationService { evaluation.setTotalPrice(((Number) (totalPrice)).intValue()); - evaluation.setEvaluateWay(sqmetreRent.getSampleEnum()); + evaluation.setEvaluateWay(sqmetreRent.getValueType()); evaluation.setCredit(sqmetreRent.getCredit()); }else{ evaluation.setTotalPrice(null); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/district/SqmetreRentService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/district/SqmetreRentService.java index 540d00721..ace5f5629 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/district/SqmetreRentService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/district/SqmetreRentService.java @@ -4,6 +4,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.model.SqmetreRent; public interface SqmetreRentService { @@ -28,4 +29,5 @@ public interface SqmetreRentService { SqmetreRent findNearestSqmetreRent(String city, String region, String name, String cat); + List findAllAvgPriceByKeys(DistrictKey districtKey, String cat); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/district/SqmetreRentServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/district/SqmetreRentServiceImpl.java index 60cb7cc97..56630bc77 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/district/SqmetreRentServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/district/SqmetreRentServiceImpl.java @@ -5,6 +5,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import com.fangjia.dal.common.DistrictKey; import org.apache.commons.lang.StringUtils; import org.apache.http.impl.cookie.DateUtils; @@ -216,6 +217,21 @@ public class SqmetreRentServiceImpl implements SqmetreRentService{ } return sqmetreRent; } - - + + @Override + public List findAllAvgPriceByKeys(DistrictKey key, String cat) { + List params = new ArrayList(); + params.add(new QueryParam("city", key.getCity())); + if (StringUtils.isNotBlank(key.getRegion())) { + params.add(new QueryParam("region", key.getRegion())); + } + if (StringUtils.isNotBlank(key.getName())) { + params.add(new QueryParam("districtName", key.getName())); + } + params.add(new QueryParam("cat", cat)); + List sortParams = new ArrayList(); + sortParams.add(new SortParam("weekend", true)); + return sqmetreRentDao.findAllByParams(params, sortParams,0,Integer.MAX_VALUE, + new String[]{"weekend", "credit", "cat", "avgPrice", "calcAvgPrice"}); + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java index efe0684cc..7e88147a0 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/index/SellTriggerServiceImpl.java @@ -46,7 +46,7 @@ public class SellTriggerServiceImpl implements TriggerService { public List processBSONObjects(List bSONDocList) { List ret = new ArrayList(); // Map districtKnowbaseMap = indexHelperService.getFromKnowBase(bSONDocList); - Map> metroDistanceMap = indexHelperService.getMetroDistanceMap(bSONDocList); +// Map> metroDistanceMap = indexHelperService.getMetroDistanceMap(bSONDocList); Map districtMap = indexHelperService.getDistricts(bSONDocList); // Map districtStatMap = // indexHelperService.getDistrictStats(districtKnowbaseMap); @@ -70,12 +70,12 @@ public class SellTriggerServiceImpl implements TriggerService { for (BSONObject object : bSONDocList) { try { ServerArea city = ServerAreaData.getServerAreaByDisplayDefaultNull(String.valueOf(object.get("city"))); - DistrictKey districtKey = new DistrictKey((String) object.get("city"), (String) object.get("region"), - (String) object.get("district_name")); - District district = districtMap.get(districtKey); - if (district != null && district.getName() != null) { - districtKey = new DistrictKey(district.getCity(), district.getRegion(),district.getName()); - } +// DistrictKey districtKey = new DistrictKey((String) object.get("city"), (String) object.get("region"), +// (String) object.get("district_name")); +// District district = districtMap.get(districtKey); +// if (district != null && district.getName() != null) { +// districtKey = new DistrictKey(district.getCity(), district.getRegion(),district.getName()); +// } BSONObject newObj = new BasicBSONObject(); @@ -156,16 +156,16 @@ public class SellTriggerServiceImpl implements TriggerService { newObj.put("fitment", object.get("fitment")); newObj.put("contact_people", object.get("contact_people")); ScoreCalculatorInputData inputDate = new ScoreCalculatorInputData(); - if (district != null) { - newObj.put("region", district.getRegion()); - if (StringUtils.isNotBlank(district.getBlock())) { - newObj.put("block", district.getBlock()); - } - if (district.getName() != null) { - newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); - newObj.put("district_name", district.getName()); - } - } +// if (district != null) { +// newObj.put("region", district.getRegion()); +// if (StringUtils.isNotBlank(district.getBlock())) { +// newObj.put("block", district.getBlock()); +// } +// if (district.getName() != null) { +// newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); +// newObj.put("district_name", district.getName()); +// } +// } // newObj.put("if_school", 0); // if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) { @@ -248,16 +248,16 @@ public class SellTriggerServiceImpl implements TriggerService { newObj.put("visible", 1); } - // 地铁信息 - if (metroDistanceMap != null && metroDistanceMap.get(districtKey) != null) { - newObj.put("metro", metroDistanceMap.get(districtKey).get("metro")); - newObj.put("station", metroDistanceMap.get(districtKey).get("station")); - newObj.put("metro_info", metroDistanceMap.get(districtKey).get("metro_info")); - newObj.put("if_metro", metroDistanceMap.get(districtKey).get("if_metro")); - } else { - newObj.put("if_metro", 0); - } - +// // 地铁信息 +// if (metroDistanceMap != null && metroDistanceMap.get(districtKey) != null) { +// newObj.put("metro", metroDistanceMap.get(districtKey).get("metro")); +// newObj.put("station", metroDistanceMap.get(districtKey).get("station")); +// newObj.put("metro_info", metroDistanceMap.get(districtKey).get("metro_info")); +// newObj.put("if_metro", metroDistanceMap.get(districtKey).get("if_metro")); +// } else { +// newObj.put("if_metro", 0); +// } +// //2018-07-31新增字段 alert by jenis newObj.put("estate_type", object.get("estate_type"));//物业类型 newObj.put("house_number", object.get("house_number"));//房屋编号 diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java index 687b1346f..772d5594c 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/AverageStatisticsServiceImpl.java @@ -187,11 +187,11 @@ public class AverageStatisticsServiceImpl implements AverageStatisticsService { Integer zhongyuanPrice = (Integer) dataMap.get("zhongyuan_price"); if(null == zhongyuanPrice){ - if (isValid(entity.getCentanetEsfPrice())) { - dataMap.put("zhongyuan_price", entity.getCentanetEsfPrice()); + if (isValid(entity.getZhongyuanEsfPrice())) { + dataMap.put("zhongyuan_price", entity.getZhongyuanEsfPrice()); }else{ - if(isValid(entity.getCentanetNewPrice())){ - dataMap.put("zhongyuan_price", entity.getCentanetNewPrice()); + if(isValid(entity.getZhongyuanNewPrice())){ + dataMap.put("zhongyuan_price", entity.getZhongyuanNewPrice()); } } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterService.java index 4fafa6663..f03e13be6 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterService.java @@ -4,6 +4,7 @@ import java.util.Date; import java.util.List; import java.util.Map; +import com.fangjia.dal.SeawaterQuery; import com.fangjia.dal.seawater.model.Seawater; public interface SeawaterService { @@ -37,9 +38,16 @@ public interface SeawaterService { */ void deleteByIds(String ids); + void batchDeleteByParams(SeawaterQuery query); + + void batchUpdateEstateType(SeawaterQuery query); + @Deprecated List findAllSeawater(String category, String city, String name, String region, Integer room, String root, String sort, Integer startDate, Integer endDate, int start, int limit, boolean order, String[] fields); @Deprecated int findAllSeawaterCount(String category, String city, String name, String region, Integer room, String root, Integer startDate, Integer endDate); + + List findSeawatersByQuery(SeawaterQuery query); + long findSeawatersCountByQuery(SeawaterQuery query); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterServiceImpl.java index 2dc484035..29a491729 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/SeawaterServiceImpl.java @@ -1,5 +1,6 @@ package com.fangjia.service.mongo.fangjia; +import com.fangjia.dal.SeawaterQuery; import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.datastore.driver.DSCollection; @@ -191,6 +192,16 @@ public class SeawaterServiceImpl implements SeawaterService { seawaterDao.deleteByIds(Arrays.asList(idArr)); } + @Override + public void batchDeleteByParams(SeawaterQuery query) { + seawaterDao.batchDeleteByParams(query); + } + + @Override + public void batchUpdateEstateType(SeawaterQuery query) { + seawaterDao.batchUpdateEstateType(query); + } + public void setSeawaterDao(SeawaterDao seawaterDao) { this.seawaterDao = seawaterDao; } @@ -237,7 +248,13 @@ public class SeawaterServiceImpl implements SeawaterService { sort,order,start,limit,table); } + @Override + public List findSeawatersByQuery(SeawaterQuery query){ + return seawaterDao.findSeawatersByQuery(query); + } - - + @Override + public long findSeawatersCountByQuery(SeawaterQuery query){ + return seawaterDao.findSeawatersCountByQuery(query); + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/EvaluateRatioService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/EvaluateRatioService.java new file mode 100644 index 000000000..6322b06fe --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/EvaluateRatioService.java @@ -0,0 +1,14 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.open.model.EvaluateRatio; + +/** + * @author shenjian + * @since 2022/7/27 + */ +public interface EvaluateRatioService { + + EvaluateRatio findByCityAndRegion(String city, String region); + + EvaluateRatio findByCity(String city); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/EvaluateRatioServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/EvaluateRatioServiceImpl.java new file mode 100644 index 000000000..8ebc56bc0 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/EvaluateRatioServiceImpl.java @@ -0,0 +1,39 @@ +package com.fangjia.service.open.impl; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.mongo.open.dao.EvaluateRatioDao; +import com.fangjia.dal.mongo.open.model.EvaluateRatio; +import com.fangjia.service.open.EvaluateRatioService; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author shenjian + * @since 2022/7/27 + */ +public class EvaluateRatioServiceImpl implements EvaluateRatioService { + + private EvaluateRatioDao evaluateRatioDao; + + @Override + public EvaluateRatio findByCityAndRegion(String city, String region) { + List params = new ArrayList(); + params.add(new QueryParam("category", "region")); + params.add(new QueryParam("city", city)); + params.add(new QueryParam("region", region)); + return evaluateRatioDao.findFirstByParams(params); + } + + @Override + public EvaluateRatio findByCity(String city) { + List params = new ArrayList(); + params.add(new QueryParam("category", "city")); + params.add(new QueryParam("city", city)); + return evaluateRatioDao.findFirstByParams(params); + } + + public void setEvaluateRatioDao(EvaluateRatioDao evaluateRatioDao) { + this.evaluateRatioDao = evaluateRatioDao; + } +} diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/close.png b/projects/fangjia-static/src/main/webapp/images/fangjia/close.png index d7c505004..eb922969a 100644 Binary files a/projects/fangjia-static/src/main/webapp/images/fangjia/close.png and b/projects/fangjia-static/src/main/webapp/images/fangjia/close.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/company.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/company.png new file mode 100644 index 000000000..7adae690c Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/company.png differ diff --git a/projects/fangjia-static/src/main/webapp/images/fangjia/v5/partner_m1.png b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/partner_m1.png new file mode 100644 index 000000000..29a24acc4 Binary files /dev/null and b/projects/fangjia-static/src/main/webapp/images/fangjia/v5/partner_m1.png differ diff --git a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css index 0b498e441..805853bc9 100644 --- a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css +++ b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css @@ -20,7 +20,7 @@ .officeMap{width:100%;height:400px;} .main_content.partner{overflow:hidden;} -.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:24px;margin-bottom:25px;cursor:pointer;} +.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:15px;margin-bottom:25px;cursor:pointer;} .partner a.partner11{background-position:0 0;} .partner a.partner12{background-position:-204px 0;}