diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DevelopmentZoneController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DevelopmentZoneController.java index 71d4760c2..b5832cb81 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DevelopmentZoneController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/DevelopmentZoneController.java @@ -4,9 +4,13 @@ import com.fangjia.bkoff.model.DevelopmentZoneModel; import com.fangjia.bkoff.model.JsonModel; import com.fangjia.bkoff.model.LatlngPolygonModel; import com.fangjia.core.util.GeoUtils; +import com.fangjia.core.util.ThreadLocalData; +import com.fangjia.core.util.ThreadLocalKey; import com.fangjia.dal.mongo.model.DevelopmentZone; import com.fangjia.dal.mongo.model.LatlngPolygon; import com.fangjia.dal.mongo.model.StatsIndex; +import com.fangjia.dal.mongo.model.bkoff.BkoffUsers; +import com.fangjia.service.mongo.bkoff.BkoffUsersService; import com.fangjia.service.mongo.fangjia.DevelopmentZoneService; import com.fangjia.service.mongo.fangjia.LatlngPolygonService; import org.apache.commons.lang.StringUtils; @@ -22,13 +26,16 @@ import static com.fangjia.core.util.GeoUtils.LNG; */ public class DevelopmentZoneController { private DevelopmentZoneService developmentZoneService; + private BkoffUsersService bkoffUsersService; public Object listAction(DevelopmentZoneModel model) { Pager pager = model.getPager(30); if (pager.getCurrentPage() > 100) { pager.setCurrentPage(1); } - List developmentZones = developmentZoneService.findDevelopmentZone(model.getProvince(), model.getName(), pager); + long countDev = developmentZoneService.countDev(model.getProvince(),model.getName(), model.getLevel()); + pager.setTotalRows((int) countDev); + List developmentZones = developmentZoneService.findDevelopmentZone(model.getProvince(),model.getName(), model.getLevel(), pager); model.setDevelopmentZoneList(developmentZones); model.setPager(pager); return model; @@ -62,44 +69,53 @@ public class DevelopmentZoneController { } public Object saveOrUpdateAction(DevelopmentZoneModel model) { + String userId = (String) ThreadLocalData.getInstance().get(ThreadLocalKey.UserId); + BkoffUsers user = bkoffUsersService.findBkoffUsersById(userId); JsonModel ret = new JsonModel(); Map data = new HashMap(); ret.setJsonModel(data); - // name province bdLocation level 不能为空 - if (StringUtils.isBlank(model.getProvince()) || StringUtils.isBlank(model.getName()) || model.getBdShape() == null || model.getBdShape().toString().equals("[]") || StringUtils.isBlank(model.getLevel())) { - data.put("msg", "保存失败"); - return ret; - } - if (model.getId() == null) { - DevelopmentZone entity = new DevelopmentZone(); // name province bdLocation level 不能为空 + if (StringUtils.isBlank(model.getProvince()) || StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getLevel())) { + data.put("msg", "保存失败(开发区名称,所在省份,级别不能为空)"); + return ret; + } + DevelopmentZone entity = new DevelopmentZone(); + // name province bdLocation level 不能为空/ entity.setName(model.getName()); entity.setProvince(model.getProvince()); - String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", ""); - List> bdLocationList = new ArrayList>(); - String[] splitStr = locationStr.split(";"); - for (int i = 0; i < splitStr.length; i++) { - String[] splitStr2 = splitStr[i].split(","); - List location = new ArrayList(); - location.add(Double.parseDouble(splitStr2[0])); - location.add(Double.parseDouble(splitStr2[1])); - bdLocationList.add(location); + // 新加开发区,围栏不是必须有的 + if (model.getBdShape() != null && !model.getBdShape().toString().equals("[]")) { + String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", ""); + List> bdLocationList = new ArrayList>(); + String[] splitStr = locationStr.split(";"); + for (int i = 0; i < splitStr.length; i++) { + String[] splitStr2 = splitStr[i].split(","); + List location = new ArrayList(); + location.add(Double.parseDouble(splitStr2[0])); + location.add(Double.parseDouble(splitStr2[1])); + bdLocationList.add(location); + } + //加上第一个,形成首尾闭合 + bdLocationList.add(bdLocationList.get(0)); + model.setBdShape(bdLocationList); + // 百度经纬度 + Map bdShape = new HashMap(); + bdShape.put("type", "polygon"); + bdShape.put("coordinates", model.getBdShape()); + entity.setBdShape(bdShape); + // 高德经纬度 + Map shape = new HashMap(); + shape.put("type", "polygon"); + Object coordinates = convertGeoList(model.getBdShape()); + shape.put("coordinates", coordinates); + entity.setShape(shape); } - model.setBdShape(bdLocationList); - // 百度经纬度 - Map bdShape = new HashMap(); - bdShape.put("type", "polygon"); - bdShape.put("coordinates", model.getBdShape()); - entity.setBdShape(bdShape); - // 高德经纬度 - Map shape = new HashMap(); - shape.put("type", "polygon"); - Object coordinates = convertGeoList(model.getBdShape()); - shape.put("coordinates", coordinates); - entity.setShape(shape); entity.setLevel(model.getLevel()); + String aliasString = model.getAlias().replaceAll(",",","); + List alias = Arrays.asList(aliasString.split("\\s*,\\s*")); + entity.setAlias(alias); entity.setAddress(StringUtils.isBlank(model.getAddress()) ? "" : model.getAddress()); entity.setArea(StringUtils.isBlank(model.getArea()) ? "" : model.getArea()); entity.setBlock(StringUtils.isBlank(model.getBlock()) ? "" : model.getBlock()); @@ -109,50 +125,86 @@ public class DevelopmentZoneController { entity.setNameEn(StringUtils.isBlank(model.getNameEn()) ? "" : model.getNameEn()); entity.setType(StringUtils.isBlank(model.getType()) ? "" : model.getType()); entity.setUrl(StringUtils.isBlank(model.getUrl()) ? "" : model.getUrl()); + entity.setCityName(StringUtils.isBlank(model.getCityName()) ? "" : model.getCityName()); + //记录操作人员 + entity.setOperator(user.getRealName()); developmentZoneService.saveDevelopmentZone(entity); data.put("msg", "保存成功"); } else { - DevelopmentZone entity = developmentZoneService.findDevelopmentZone(model.getId()); -// Map udata = new HashMap(); - - // name province bdLocation level 不能为空 - entity.setName(model.getName()); - entity.setProvince(model.getProvince()); - String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", ""); - List> bdLocationList = new ArrayList>(); - String[] splitStr = locationStr.split(";"); - for (int i = 0; i < splitStr.length; i++) { - String[] splitStr2 = splitStr[i].split(","); - List location = new ArrayList(); - location.add(Double.parseDouble(splitStr2[0])); - location.add(Double.parseDouble(splitStr2[1])); - bdLocationList.add(location); +// DevelopmentZone entity = developmentZoneService.findDevelopmentZone(model.getId()); + Map udata = new HashMap(); + if (StringUtils.isNotBlank(model.getProvince())) { + udata.put("province", model.getProvince()); } - model.setBdShape(bdLocationList); - // 百度经纬度 - Map bdShape = new HashMap(); - bdShape.put("type", "polygon"); - bdShape.put("coordinates", model.getBdShape()); - entity.setBdShape(bdShape); - // 高德经纬度 - Map shape = new HashMap(); - shape.put("type", "polygon"); - Object coordinates = convertGeoList(model.getBdShape()); - shape.put("coordinates", coordinates); - entity.setShape(shape); - - entity.setLevel(model.getLevel()); - entity.setAddress(StringUtils.isBlank(model.getAddress()) ? "" : model.getAddress()); - entity.setArea(StringUtils.isBlank(model.getArea()) ? "" : model.getArea()); - entity.setBlock(StringUtils.isBlank(model.getBlock()) ? "" : model.getBlock()); - entity.setBuildDate(StringUtils.isBlank(model.getBuildDate()) ? "" : model.getBuildDate()); - entity.setCreateTime(new Date()); - entity.setIndustry(StringUtils.isBlank(model.getIndustry()) ? "" : model.getIndustry()); - entity.setNameEn(StringUtils.isBlank(model.getNameEn()) ? "" : model.getNameEn()); - entity.setType(StringUtils.isBlank(model.getType()) ? "" : model.getType()); - entity.setUrl(StringUtils.isBlank(model.getUrl()) ? "" : model.getUrl()); - developmentZoneService.updateDevelopmentZone(entity); - data.put("msg", "保存成功"); + if (StringUtils.isNotBlank(model.getName())) { + udata.put("name", model.getName()); + } + if (StringUtils.isNotBlank(model.getLevel())) { + udata.put("level", model.getLevel()); + } + if (model.getBdShape() != null && !model.getBdShape().toString().equals("[]")) { + String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", ""); + List> bdLocationList = new ArrayList>(); + String[] splitStr = locationStr.split(";"); + for (int i = 0; i < splitStr.length; i++) { + String[] splitStr2 = splitStr[i].split(","); + List location = new ArrayList(); + location.add(Double.parseDouble(splitStr2[0])); + location.add(Double.parseDouble(splitStr2[1])); + bdLocationList.add(location); + } + //加上第一个,形成首尾闭合 + bdLocationList.add(bdLocationList.get(0)); + model.setBdShape(bdLocationList); + // 百度经纬度 + Map bdShape = new HashMap(); + bdShape.put("type", "polygon"); + bdShape.put("coordinates", model.getBdShape()); + udata.put("bdShape", bdShape); + // 高德经纬度 + Map shape = new HashMap(); + shape.put("type", "polygon"); + Object coordinates = convertGeoList(model.getBdShape()); + shape.put("coordinates", coordinates); + udata.put("shape", shape); + } + if (StringUtils.isNotBlank(model.getAlias()) ){ + String aliasString = model.getAlias().replaceAll(",",","); + List alias = Arrays.asList(aliasString.split("\\s*,\\s*")); + udata.put("alias", alias); + } + if (StringUtils.isNotBlank(model.getAddress())) { + udata.put("address", model.getAddress()); + } + if (StringUtils.isNotBlank(model.getArea())) { + udata.put("area", model.getArea()); + } + if (StringUtils.isNotBlank(model.getBlock())) { + udata.put("block", model.getBlock()); + } + if (StringUtils.isNotBlank(model.getBuildDate())) { + udata.put("buildDate", model.getBuildDate()); + } + if (StringUtils.isNotBlank(model.getIndustry())) { + udata.put("industry", model.getIndustry()); + } + if (StringUtils.isNotBlank(model.getNameEn())) { + udata.put("nameEn", model.getNameEn()); + } + if (StringUtils.isNotBlank(model.getType())) { + udata.put("type", model.getType()); + } + if (StringUtils.isNotBlank(model.getUrl())) { + udata.put("url", model.getUrl()); + } + if (StringUtils.isNotBlank(model.getCityName())) { + udata.put("cityName", model.getCityName()); + } + udata.put("createTime", new Date()); + //记录操作人员 + udata.put("operator", user.getRealName()); + developmentZoneService.updateDevelopmentZone(udata, model.getId()); + data.put("msg", "修改成功"); } return ret; @@ -162,6 +214,10 @@ public class DevelopmentZoneController { public void setDevelopmentZoneService(DevelopmentZoneService developmentZoneService) { this.developmentZoneService = developmentZoneService; } + + public void setBkoffUsersService(BkoffUsersService bkoffUsersService) { + this.bkoffUsersService = bkoffUsersService; + } //将百度坐标转为高德坐标 public List> convertGeoList(List> bdLocationList) { diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/DevelopmentZoneModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/DevelopmentZoneModel.java index a0e31ca10..03460ae58 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/DevelopmentZoneModel.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/DevelopmentZoneModel.java @@ -17,20 +17,38 @@ public class DevelopmentZoneModel extends BaseModel { private String level; private String name; + private String alias; +// private List addr_alias; private String nameEn; private String province; private String type; private String url; + private String district; List> shape; List> bdShape; - private String block; private DevelopmentZone developmentZone; private List developmentZoneList; + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + public String getId() { return id; } diff --git a/projects/fangjia-bkoff/src/main/resources/ftl/city.ftl b/projects/fangjia-bkoff/src/main/resources/ftl/city.ftl index b72f6b106..0a6ac6c87 100644 --- a/projects/fangjia-bkoff/src/main/resources/ftl/city.ftl +++ b/projects/fangjia-bkoff/src/main/resources/ftl/city.ftl @@ -9,7 +9,7 @@
diff --git a/projects/fangjia-bkoff/src/main/webapp/developmentZone/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/developmentZone/edit.ftl index 0d558d442..00e260c6d 100644 --- a/projects/fangjia-bkoff/src/main/webapp/developmentZone/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/developmentZone/edit.ftl @@ -3,147 +3,216 @@ - - 寮�鍙戝尯鍥存爮鍒楄〃 - - + + 寮�鍙戝尯鍥存爮鍒楄〃 + + - <#include "/common_js.ftl" /> - <#include "/layout/header.ftl" /> -
- +<#include "/common_js.ftl" /> +<#include "/layout/header.ftl" /> +
+ -
-
- <#if developmentZone??> - - +
+ + <#if developmentZone??> + + - - - - - - - - -
- <#if latlngPolygon??> - <@d_city defaultValue="${latlngPolygon.city!}" callback="changeCity" callbackField="display" name="cityDisplay" /> - <#else> - <@d_city defaultValue="涓婃捣" callback="changeCity" callbackField="display" name="cityDisplay" /> - - - - - - - - - - - - - - -
- -
-
- -
-
+ + + + - - - - - - - + + + + +
+ <#if developmentZone??> + <@d_city defaultValue="${developmentZone.cityName!}" callback="changeCity" callbackField="display" name="cityDisplay" /> + <#else> + <@d_city defaultValue="涓婃捣" callback="changeCity" callbackField="display" name="cityDisplay" /> + + + + + + +
+ + + +
+ + + +
+ +
+ +
+ +
+
+ +
+
- + + + + + + - function changeCity(display){ - clearAll(); - map.centerAndZoom(display, 13); - } + + + + - \ No newline at end of file + diff --git a/projects/fangjia-bkoff/src/main/webapp/developmentZone/list.ftl b/projects/fangjia-bkoff/src/main/webapp/developmentZone/list.ftl index 93a64fc40..c4ada21b1 100644 --- a/projects/fangjia-bkoff/src/main/webapp/developmentZone/list.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/developmentZone/list.ftl @@ -14,7 +14,7 @@ <#include "/layout/header.ftl" />
@@ -24,32 +24,58 @@ + +
+ 鏌ヨ鍒�${pager.totalRows!}鏉¤褰� - + + - + + + + + + <#list developmentZoneList as it> + + + + + + + + diff --git a/projects/fangjia-bkoff/src/main/webapp/developmentZone/show.ftl b/projects/fangjia-bkoff/src/main/webapp/developmentZone/show.ftl index 90fc09df9..72fc93a1f 100644 --- a/projects/fangjia-bkoff/src/main/webapp/developmentZone/show.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/developmentZone/show.ftl @@ -31,6 +31,14 @@ + + + + + + + + @@ -39,6 +47,16 @@ + + + + @@ -48,21 +66,22 @@
+
鍚嶇О绾у埆鍒悕寮�鍙戝尯绾у埆 鐪佷唤寤虹珛鏃堕棿鍩庡競鍖哄煙鍖哄潡鏄惁鏈夊洿鏍�鎿嶄綔浜哄憳淇敼鏃ユ湡 鎿嶄綔
${it.name!} + <#if it.alias??> + <#list it.alias as item> + ${item?string}<#if item_has_next>, + + + ${it.level!} ${it.province!}${it.cityName!}${it.district!}${it.block!} - ${it.buildDate} + <#if it.bdShape?has_content> + 宸叉湁鍥存爮 + ${it.operator!}${it.createTime!?string("yyyy-MM-dd HH:mm:ss")} 鏌ョ湅 鏂板缓鍥存爮 - 鍒犻櫎 +<#-- 鍒犻櫎-->
鐪佷唤 ${developmentZone.province!}
鍩庡競${developmentZone.cityName!}
鍖哄煙${developmentZone.district!}
绾у埆 ${developmentZone.level!}鍚嶇О ${developmentZone.name!}
鍒悕 + <#if developmentZone.alias??> + <#list developmentZone.alias as item> + ${item?string}<#if item_has_next>, + + +
鍖哄潡 ${developmentZone.block!}
+ src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"> - + src="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"> + - + src="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"> + diff --git a/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl index 9b29ea9e6..bdb33632a 100644 --- a/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/district/edit.ftl @@ -1052,8 +1052,8 @@ <#----> - - + + + + + - +
diff --git a/projects/fangjia-dataanalysis/src/main/webapp/seaweed/map.ftl b/projects/fangjia-dataanalysis/src/main/webapp/seaweed/map.ftl index 3b2e7a413..ce2c8036f 100644 --- a/projects/fangjia-dataanalysis/src/main/webapp/seaweed/map.ftl +++ b/projects/fangjia-dataanalysis/src/main/webapp/seaweed/map.ftl @@ -7,7 +7,7 @@ - +
diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index e7dc05ae8..5f9b37bb7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -14,11 +14,31 @@ import com.fangjia.open.util.CommonDtoUtils; import com.fangjia.service.bo.DistrictStatBo; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.district.DistrictStatService; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import sun.misc.BASE64Encoder; +import java.io.IOException; +import java.net.Authenticator; +import java.net.InetSocketAddress; +import java.net.PasswordAuthentication; +import java.net.Proxy; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; +import java.util.Stack; /** * 地址治理 @@ -358,8 +378,115 @@ public class AddrSvcController { return mav; } + private static final Map areaMap = new HashMap() { + { + // 添加键值对到 Map 中 + put("111", "主城区"); + put("112", "城乡结合区"); + put("121", "镇中心区"); + put("122", "镇乡结合区"); + put("123", "特殊区域"); + put("210", "乡中心区"); + put("220", "村庄"); + } + }; + /** - * 涉农识别-新规则 + * 涉农识别-新规则-南京银行 + * + * @param model + * @return + */ + public Object njcbVillageIdentifyAction(AddressModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null); + String address = model.getAddress(); + params.put("address", address); + String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null; + params.put("type", type); + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8"); + } catch (Exception e) { + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8"); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("msgCode"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if (json.containsKey("result")) { + //我们的结果 + JSONObject result = json.getJSONObject("result"); + //我们省级及以上有开发区,直接用我们的结果 + if (result.containsKey("devDistrictName")) { + basicModel.setResult(result); + + } else { + try { + //如果是江苏省或者浙江省 调用齐梁的结果 + if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) { + //结果比较,如果涉农结果不一致 + //调用齐梁 type :1 对公 0 对私 + //我们 type :1 对私 2 对共 + String qltype = "2".equals(type) ? "1" : "0"; + Map map = senRequest(address, qltype); + // 涉农结果不一样 + if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) { + //城市不一样 用我们 + if (result.containsKey("city")) { + if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) { + basicModel.setResult(result); + } else { + map.put("addrVerify", "地址有效"); + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + } else { + map.put("addrVerify", "地址有效"); + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + + } else { + basicModel.setResult(result); + } + + } else { + basicModel.setResult(result); + + } + + } catch (Exception e) { + e.printStackTrace(); + basicModel.setResult(result); + } + + } + + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + + /** + * 涉农识别-新规则-基础版本 * * @param model * @return @@ -372,6 +499,207 @@ public class AddrSvcController { return mav; } + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null); + String address = model.getAddress(); + params.put("address", address); + String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null; + params.put("type", type); + params.put("showFullValue", "2"); +// params.put("showTestValue", "2"); + Integer radius = model.getRadius(); + params.put("radius", (radius != null && radius != 0) ? radius.toString() : null); + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8"); + } catch (Exception e) { + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8"); + } catch (Exception ex) { + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("msgCode"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if (json.containsKey("result")) { + //我们的结果 + JSONObject result = json.getJSONObject("result"); + //我们省级及以上有开发区,直接用我们的结果 + if (result.containsKey("devDistrictName")) { + basicModel.setResult(result); + + } else { + try { + //如果是江苏省或者浙江省 调用齐梁的结果 + if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) { + + //结果比较,如果涉农结果不一致 + //调用齐梁 type :1 对公 0 对私 + //我们 type :1 对私 2 对共 + String qltype = "2".equals(type) ? "1" : "0"; + Map map = senRequest(address, qltype); + // 涉农结果不一样 + if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) { + //有市辖建制镇, 或者城市不一样 用我们 + if (result.containsKey("city")) { + if ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) { + basicModel.setResult(result); + } else { + map.put("addrVerify", "地址有效"); + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + } else { + map.put("addrVerify", "地址有效"); + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + + } else { + basicModel.setResult(result); + } + + } else { + basicModel.setResult(result); + + } + + } catch (Exception e) { + e.printStackTrace(); + basicModel.setResult(result); + } + + } + + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + private static Map senRequest(String address, String type) throws IOException { + String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; + String proxyHost = "tunnel2.qg.net"; + int proxyPort = 14186; + final String proxyUsername = "113BBC06"; + final String proxyPassword = "3B2FBD05B0AF"; + String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}"; + String authKey = proxyUsername; + String password = proxyPassword; + String auth = authKey + ":" + password; + String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8"))); + + CloseableHttpClient httpClient = HttpClients.createDefault(); + HttpPost httpPost = new HttpPost(targetURL); + Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); + Authenticator.setDefault(new Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()); + } + }); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setHeader("Accept", "*/*"); + httpPost.setHeader("Host", "146.56.217.44:9010"); + httpPost.setHeader("Proxy-Authorization", authHeader); + StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON); + httpPost.setEntity(stringEntity); + + CloseableHttpResponse httpResponse = null; + int retryCount = 2; // 设置重试次数 + int retryDelay = 10; // 设置重试延迟时间(毫秒) + boolean success = false; + for (int i = 0; i < retryCount; i++) { + try { + httpResponse = httpClient.execute(httpPost); + success = true; + break; + } catch (IOException e) { + System.err.println("代理请求失败,进行重试..."); + try { + Thread.sleep(retryDelay); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + } + + if (success) { + HttpEntity responseEntity = httpResponse.getEntity(); + String responseString = EntityUtils.toString(responseEntity); + + // 使用 Gson 解析 JSON 字符串 + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class); + + + // 创建 Map 存储各个值 + Map dataMap = new HashMap(); + + // 获取各个值并放入 Map +// dataMap.put("code", jsonObject.get("code").getAsString()); +// dataMap.put("msg", jsonObject.get("msg").getAsString()); + JsonElement dataElement = jsonObject.get("data"); + if (dataElement != null && !dataElement.isJsonNull() && dataElement.isJsonObject()) { + JsonObject data = dataElement.getAsJsonObject(); + // 在这里继续处理 data 对象 + + if (data != null && !data.isJsonNull()) { + +// dataMap.put("version", data.get("version").getAsString()); + dataMap.put("province", data.get("province").getAsString()); + dataMap.put("city", data.get("city").getAsString()); + dataMap.put("district", data.get("district").getAsString()); + dataMap.put("town", data.get("town").getAsString()); + dataMap.put("committee", data.get("village").getAsString()); + dataMap.put("provinceCode", data.get("provinceCode").getAsString()); + dataMap.put("cityCode", data.get("cityCode").getAsString()); + dataMap.put("districtCode", data.get("districtCode").getAsString()); + dataMap.put("townCode", data.get("townCode").getAsString()); + dataMap.put("committeeCode", data.get("villageCode").getAsString()); + dataMap.put("typeCode", data.get("areaCode").getAsString()); + dataMap.put("verifyFlag", "0".equals(data.get("farmer").getAsString()) ? "非农" : "农"); + dataMap.put("countyGov", "0".equals(data.get("isGovt").getAsString()) ? "否" : "是"); +// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空 + JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray(); + + if (indusParkList != null) { + JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class); + dataMap.put("devDistrictName", jsonObject1.get("indusParkName").getAsString()); + dataMap.put("devDistrict", "是"); + } + } + } + + httpResponse.close(); + return dataMap; + } else { + System.err.println("重试次数已用完,请求失败。"); + } + + httpClient.close(); + return null; + } + + /** + * 涉农识别-新规则-标准版 + * + * @param model + * @return + */ + public Object villageIdentifyStandardAction(AddressModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + HttpRequest request = new HttpRequest(); JSONObject json = null; Map params = new HashMap(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java index 4d264b835..9d891b609 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java @@ -233,10 +233,13 @@ public class ComprehensiveQueryController { communityAvgPriceParams.put("city", model.getCity()); communityAvgPriceParams.put("address", model.getAddress()); communityAvgPriceParams.put("category", model.getCategory()); - communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap())); +// communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap())); + communityAvgPriceParams.put("suggestName", String.valueOf(model.getName())); + communityAvgPriceParams.put("preciseLevel", "strict"); + communityAvgPriceParams.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"); JSONObject matchResponse = null; try { - matchResponse = httpRequest.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8"); + matchResponse = httpRequest.getJsonContent("http://open.fangjia.com/address/match", communityAvgPriceParams, "UTF-8"); // matchResponse = httpRequest.getJsonContent("http://192.168.0.19:31080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8"); } catch (HttpRequestException e) { @@ -256,9 +259,21 @@ public class ComprehensiveQueryController { DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(district)); if (null != statBo) { if (statBo.getDistrictStat() != null) { - dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice()); + // 有2B价格,使用2B价格 + Integer avgPrice2b = statBo.getDistrictStat().getAvg_price_2b(); + if (avgPrice2b != null && avgPrice2b > 0) { + dto.setAvgPrice(avgPrice2b); + } else { + dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice()); + } } else if (statBo.getVillaDistrictStat() != null) { - dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice()); + // 有2B价格,使用2B价格 + Integer avgPrice2b = statBo.getVillaDistrictStat().getAvg_price_2b(); + if (avgPrice2b != null && avgPrice2b > 0) { + dto.setAvgPrice(avgPrice2b); + } else { + dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice()); + } } } @@ -638,10 +653,18 @@ public class ComprehensiveQueryController { if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { int factorPrice = evaluation.getAvgPrice(); int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); - Integer minPrice = (int) (evaluation.getAvgPrice() * 0.95); - Integer maxPrice = (int) (evaluation.getAvgPrice() * 1.05); + // 有2B价格用2B价格,没有用2C + Integer minPrice = 0; + Integer maxPrice = 0; + if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() > 0) { + minPrice = (int) (evaluation.getAvgPrice2b() * 0.95); + maxPrice = (int) (evaluation.getAvgPrice2b() * 1.05); + } else { + minPrice = (int) (evaluation.getAvgPrice() * 0.95); + maxPrice = (int) (evaluation.getAvgPrice() * 1.05); + } // 偏离度超过5%不进行调整 - if (minPrice <= avgPrice && avgPrice <= maxPrice) { + if (minPrice != 0 && maxPrice != 0 && minPrice <= avgPrice && avgPrice <= maxPrice) { evaluation.setAvgPrice(avgPrice); evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); } @@ -657,7 +680,11 @@ public class ComprehensiveQueryController { districtPriceMap.put("district", district.getRegion()); districtPriceMap.put("name", district.getName()); districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); - districtPriceMap.put("avgPrice", evaluation.getAvgPrice()); + if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() != 0) { + districtPriceMap.put("avgPrice", evaluation.getAvgPrice2b()); + } else { + districtPriceMap.put("avgPrice", evaluation.getAvgPrice()); + } // 根据城市等 级设置D级可信度 ApiUserAuthority apiUserAuthority = apiUserAuthorityService @@ -882,7 +909,11 @@ public class ComprehensiveQueryController { otherPriceRets = new HashMap(); } - otherPriceRets.put("fjw_price", evaluation.getAvgPrice()); + if (evaluation.getAvgPrice2b()!=null&&evaluation.getAvgPrice2b() > 0){ + otherPriceRets.put("fjw_price", evaluation.getAvgPrice2b()); + }else { + otherPriceRets.put("fjw_price", evaluation.getAvgPrice()); + } Map levelMap = getPriceLevel(otherPriceRets, evaluation.getCredit()); Long lastPrice = getLastPrice(levelMap, otherPriceRets, evaluation, dCredit); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Controller.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Controller.java index 1c508b26b..95877519f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Controller.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Controller.java @@ -55,38 +55,44 @@ public class Controller { return mav; } - /** - * 缓存刷新接口 - * @return - */ - @Authority(role = Role.NONE) - public Object refreshCacheAction(RefreshCacheModel model){ - RestfulModelAndView mav = new RestfulModelAndView(); - mav.setViewType("rest"); - Token token = FangjiaApiUtil.parse2Token(model.getToken()); - if(token == null){ - mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token")); - return mav; - } - final ThreadLocalData threadLocalData = ThreadLocalData.getInstance(); - threadLocalData.put(ThreadLocalKey.DisableCache, "1"); - // 在验证appKey - ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId()); - if(apiUser == null){ - mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token")); - return mav; - } - try { - apiUserAuthorityService.findByUserId(apiUser.getId()); - if(StringUtils.isNotBlank(model.getViewConfigUrl())){ - apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl()); - } - } catch (IllegalAccessException e) { - e.printStackTrace(); - } - mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功")); - return mav; - } + /** + * 缓存刷新接口 + * + * @return + */ + @Authority(role = Role.NONE) + public Object refreshCacheAction(RefreshCacheModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + Token token = FangjiaApiUtil.parse2Token(model.getToken()); + if (token == null) { + mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token")); + return mav; + } + final ThreadLocalData threadLocalData = ThreadLocalData.getInstance(); + threadLocalData.put(ThreadLocalKey.DisableCache, "1"); + // 在验证appKey + ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId()); + if (apiUser == null) { + mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token")); + return mav; + } + try { + // 获取用户 + User user = userService.findById(apiUser.getId()); + if (null != user && StringUtils.isNotBlank(user.getUserName())) { + userService.findByUserName(user.getUserName()); + } + apiUserAuthorityService.findByUserId(apiUser.getId()); + if (StringUtils.isNotBlank(model.getViewConfigUrl())) { + apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl()); + } + } catch (IllegalAccessException e) { + e.printStackTrace(); + } + mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功")); + return mav; + } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java index 845358f5b..fd1709bf3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GdBankController.java @@ -6,9 +6,28 @@ import com.fangjia.open.interceptors.Role; import com.fangjia.open.model.AddressModel; import com.fangjia.open.model.BasicModel; import com.fangjia.open.model.RestfulModelAndView; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; +import sun.misc.BASE64Encoder; +import java.io.IOException; +import java.net.Authenticator; +import java.net.InetSocketAddress; +import java.net.PasswordAuthentication; +import java.net.Proxy; +import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; @@ -18,8 +37,21 @@ import java.util.Map; @Authority(role = Role.USER) public class GdBankController { + private static final Map areaMap = new HashMap() { + { + // 添加键值对到 Map 中 + put("111", "主城区"); + put("112", "城乡结合区"); + put("121", "镇中心区"); + put("122", "镇乡结合区"); + put("123", "特殊区域"); + put("210", "乡中心区"); + put("220", "村庄"); + } + }; + /** - * 涉农识别-新规 + * 涉农识别-新规则-光大银行 * * @param model * @return @@ -31,12 +63,14 @@ public class GdBankController { mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); return mav; } + String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null; + String address = model.getAddress(); HttpRequest request = new HttpRequest(); JSONObject json = null; Map params = new HashMap(); params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null); - params.put("type", StringUtils.isNotBlank(model.getType()) ? model.getType() : null); + params.put("type", type); params.put("redis", (model.getRadius() != null && model.getRadius() != 0) ? model.getRadius().toString() : null); params.put("address", model.getAddress()); params.put("showFullValue", "2"); @@ -54,13 +88,63 @@ public class GdBankController { String msg = json.getString("msg"); BasicModel basicModel = new BasicModel(code, msg); if (json.containsKey("result")) { + //我们的结果 JSONObject result = json.getJSONObject("result"); - if (("是").equals(result.get("countyLevelCity"))){ + if (("是").equals(result.get("countyLevelCity"))) { result.put("prefectureLevelCity", "否"); - } else if(("否").equals(result.get("countyLevelCity"))){ + } else if (("否").equals(result.get("countyLevelCity"))) { result.put("prefectureLevelCity", "是"); } - basicModel.setResult(result); + // 如果是无效信息或者没有行政区划,则用我们的 + if (result.containsKey("addrVerify") && ("无效".equals(result.getString("addrVerify")) || "没有行政区划".equals(result.getString("addrVerify")))) { + basicModel.setResult(result); + //我们省级及以上有开发区,直接用我们的结果 + } else if (result.containsKey("devDistrictName")) { + basicModel.setResult(result); + } else { + try { + //如果是江苏省或者浙江省 调用齐梁的结果 + if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) { + + //结果比较,如果涉农结果不一致 + //调用齐梁 type :1 对公 0 对私 + //我们 type :1 对私 2 对共 + String qltype = "2".equals(type) ? "1" : "0"; + Map map = senRequest(address, qltype); + // 涉农结果不一样 + if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) { + //有市辖建制镇, 或者城市不一样 用我们 + if (result.containsKey("city") && ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) + && StringUtils.isNotBlank(result.getString("city"))))) { + basicModel.setResult(result); + } else { + if (("是").equals(result.get("countyLevelCity"))) { + map.put("prefectureLevelCity", "否"); + } else if (("否").equals(result.get("countyLevelCity"))) { + map.put("prefectureLevelCity", "是"); + } + map.put("addrVerify", "地址有效"); + if (result.containsKey("countyLevelCity")) { + map.put("countyLevelCity", result.getString("countyLevelCity")); + } + map.put("typeName", areaMap.get(map.get("typeCode"))); + basicModel.setResult(map); + } + } else { + basicModel.setResult(result); + } + + } else { + basicModel.setResult(result); + + } + + } catch (Exception e) { + e.printStackTrace(); + basicModel.setResult(result); + } + + } } mav.setModel(basicModel); } else { @@ -68,4 +152,107 @@ public class GdBankController { } return mav; } + + private static Map senRequest(String address, String type) throws IOException { + String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; + String proxyHost = "tunnel2.qg.net"; + int proxyPort = 14186; + final String proxyUsername = "113BBC06"; + final String proxyPassword = "3B2FBD05B0AF"; + String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}"; + String authKey = proxyUsername; + String password = proxyPassword; + String auth = authKey + ":" + password; + String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8"))); + + CloseableHttpClient httpClient = HttpClients.createDefault(); + HttpPost httpPost = new HttpPost(targetURL); + Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); + Authenticator.setDefault(new Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()); + } + }); + httpPost.setHeader("Content-Type", "application/json"); + httpPost.setHeader("Accept", "*/*"); + httpPost.setHeader("Host", "146.56.217.44:9010"); + httpPost.setHeader("Proxy-Authorization", authHeader); + StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON); + httpPost.setEntity(stringEntity); + + CloseableHttpResponse httpResponse = null; + int retryCount = 2; // 设置重试次数 + int retryDelay = 10; // 设置重试延迟时间(毫秒) + boolean success = false; + for (int i = 0; i < retryCount; i++) { + try { + httpResponse = httpClient.execute(httpPost); + success = true; + break; + } catch (IOException e) { + System.err.println("代理请求失败,进行重试..."); + try { + Thread.sleep(retryDelay); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + } + + if (success) { + HttpEntity responseEntity = httpResponse.getEntity(); + String responseString = EntityUtils.toString(responseEntity); + + // 使用 Gson 解析 JSON 字符串 + Gson gson = new Gson(); + JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class); + + + // 创建 Map 存储各个值 + Map dataMap = new HashMap(); + + // 获取各个值并放入 Map +// dataMap.put("code", jsonObject.get("code").getAsString()); +// dataMap.put("msg", jsonObject.get("msg").getAsString()); + JsonElement dataElement = jsonObject.get("data"); + if (dataElement != null && !dataElement.isJsonNull() && dataElement.isJsonObject()) { + JsonObject data = dataElement.getAsJsonObject(); + // 在这里继续处理 data 对象 + + if (data != null && !data.isJsonNull()) { + +// dataMap.put("version", data.get("version").getAsString()); + dataMap.put("province", data.get("province").getAsString()); + dataMap.put("city", data.get("city").getAsString()); + dataMap.put("district", data.get("district").getAsString()); + dataMap.put("town", data.get("town").getAsString()); + dataMap.put("committee", data.get("village").getAsString()); + dataMap.put("provinceCode", data.get("provinceCode").getAsString()); + dataMap.put("cityCode", data.get("cityCode").getAsString()); + dataMap.put("districtCode", data.get("districtCode").getAsString()); + dataMap.put("townCode", data.get("townCode").getAsString()); + dataMap.put("committeeCode", data.get("villageCode").getAsString()); + dataMap.put("typeCode", data.get("areaCode").getAsString()); + dataMap.put("verifyFlag", "0".equals(data.get("farmer").getAsString()) ? "非农" : "农"); + dataMap.put("countyGov", "0".equals(data.get("isGovt").getAsString()) ? "否" : "是"); +// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空 + JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray(); + + if (indusParkList != null) { + JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class); + dataMap.put("devDistrictName", jsonObject1.get("indusParkName").getAsString()); + dataMap.put("devDistrict", "是"); + } + } + } + + httpResponse.close(); + return dataMap; + } else { + System.err.println("重试次数已用完,请求失败。"); + } + + httpClient.close(); + return null; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 81a281610..b5efca725 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -10,17 +10,7 @@ import java.net.URLEncoder; import java.nio.charset.Charset; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Calendar; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -30,6 +20,7 @@ import javax.validation.Valid; import com.fangjia.core.util.*; import com.fangjia.dal.*; +import com.fangjia.dal.mongo.dao.AliasFangDao; import com.fangjia.dal.mongo.model.*; import com.fangjia.dal.mongo.open.model.CommunityBasePrice; import com.fangjia.dal.mongo.open.model.CommunityPoi; @@ -39,6 +30,7 @@ import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.open.dto.*; import com.fangjia.open.dto.addressGovernance.StructureDto; import com.fangjia.open.util.CommonDtoUtils; +import com.fangjia.service.address.AddressParserUtil; import com.fangjia.service.district.*; import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.open.*; @@ -138,7 +130,6 @@ import com.mongodb.DBObject; @Authority(role = Role.USER) public class PropertyController { - private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class); private DistrictService districtService; private AddressParser addressParser; @@ -186,7 +177,7 @@ public class PropertyController { private FileDao fileDao = FileDao.getInstance(); private static final int DEFAULT_PAGE_SIZE = 20; - private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias" }; + private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias", "alias" }; static final int BAI_DU_GEO = 1;// 百度系坐标 static final int GAO_DE_GEO = 2;// 高德系坐标 @@ -196,6 +187,7 @@ public class PropertyController { private DistrictRoomSqmetreRentService districtRoomSqmetreRentService; private RentDealPriceService rentDealPriceService; private EvaluateRatioService evaluateRatioService; + private AliasFangService aliasFangService; private static final int EVALUATE_SHOW_DETAIL = 1; private static final Map PROPERTY_TYPES = new HashMap() { @@ -609,6 +601,25 @@ public class PropertyController { dto.setBlock(district.getBlock()); } dto.setPropertyType(district.getEstateType2()); + if (CollectionUtils.isNotEmpty(district.getAlias())) { + // 将最相近的别名输出给用户 + String alias = null; + double similarity = 0; + if (StringUtils.isNotBlank(model.getKeyword())) { + for (String mappingAlias : district.getAlias()) { + double curr = AddressParserUtil.getSimilarityRatio(mappingAlias, model.getKeyword()); + if (curr > similarity ) { + similarity = curr; + alias = mappingAlias; + } + } + } + if (StringUtils.isNotBlank(alias)) { + dto.setAlias(alias); + } else { + dto.setAlias(district.getAlias().get(0)); + } + } list.add(dto); } @@ -2516,7 +2527,7 @@ public class PropertyController { model.setMax(100); } - List districtNames = new ArrayList(); + Set districtNames = new HashSet(); if (null == district) { districtNames.add(model.getName()); } else { @@ -2524,14 +2535,56 @@ public class PropertyController { if (district.getAlias() != null) { districtNames.addAll(district.getAlias()); } - } - List dealPrices = null; - for (String name : districtNames) { - dealPrices = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model); - if (!dealPrices.isEmpty()) { - break; + final String masterDistrictName = district.getMasterDistrictName(); + if (StringUtils.isNotBlank(masterDistrictName)) { + List districts; + if ("master".equals(masterDistrictName)) { + // 所有的分期数据 + districts = districtService.findSlaves(model.getCity(), model.getDistrict(), model.getName()); + } else { + // 根据主小区名查询所有名称 + districts = districtService.findSlaves(model.getCity(), model.getDistrict(), masterDistrictName); + } + if (CollectionUtils.isNotEmpty(districts)) { + for (District districtTemp : districts) { + districtNames.add(districtTemp.getName()); + } + } + + // districtNames 不包含 主小区名,加进去 + if (!districtNames.contains(masterDistrictName)){ + districtNames.add(masterDistrictName); + } + } } + // 将搜房中的别名与名称加入查询集合 + try { + List aliasFangs = aliasFangService.findFangByNameAndAlias(model.getCity(), model.getDistrict(), districtNames); + Set aliasSet = new HashSet(); + for (AliasFang aliasFang : aliasFangs) { + if (StringUtils.isNotBlank(aliasFang.getName())) { + aliasSet.add(aliasFang.getName()); + } + if (StringUtils.isNotBlank(aliasFang.getAlias())) { + aliasSet.add(aliasFang.getAlias()); + } + } + districtNames.addAll(aliasSet); + } catch (Exception ex) { + ex.printStackTrace(); + } + + +// List dealPrices = new ArrayList(); + // 将主名与映射别名查得成交加入 存在不能查出最新成交问题 +// for (String name : districtNames) { +// List dealPricesTempList = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model); +// if (CollectionUtils.isNotEmpty(dealPricesTempList)) { +// dealPrices.addAll(dealPricesTempList); +// } +// } + List dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), new ArrayList(districtNames), model.getMax() * 5, model); if (model.getSize() != null) { Collections.sort(dealPrices, new Comparator() { @Override @@ -2580,15 +2633,9 @@ public class PropertyController { } deal = new LinkedHashMap(); dealPrice.setId(null); - if (district != null) { - deal.put("city", district.getCity()); - deal.put("district", district.getRegion()); - deal.put("name", district.getName()); - } else { - deal.put("city", dealPrice.getCity()); - deal.put("district", dealPrice.getRegion()); - deal.put("name", dealPrice.getDistrictName()); - } + deal.put("city", dealPrice.getCity()); + deal.put("district", dealPrice.getRegion()); + deal.put("name", dealPrice.getDistrictName()); if (dealPrice.getArea() == null) { deal.put("size", ""); } else { @@ -3501,6 +3548,33 @@ public class PropertyController { return null; } + private List findDealPricesByNames(String city, String region, List names, int max, + DealPriceModel model) { + List params = new ArrayList(); + params.add(new QueryParam("city", city)); + params.add(new QueryParam("region", region)); + params.add(new QueryParam("districtName", QueryParam.IN_OPERATOR, names)); + if (model.getRoom() != null) { + params.add(new QueryParam("room", model.getRoom())); + } + if (model.getHall() != null) { + params.add(new QueryParam("hall", model.getHall())); + } + // 18年之后抓住的数据没有score字段 +// params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l)); + List dealPrices = dealPriceService.findDealPrices(params, max); + List distinctDealPrices = new ArrayList(); + for (DealPrice dealPrice : dealPrices) { + if (dealPrice.getAvgPrice() == null) { + dealPrice.setAvgPrice((int) (dealPrice.getTotalPrice() / dealPrice.getArea())); + } + if (!checkExists(distinctDealPrices, dealPrice)) { + distinctDealPrices.add(dealPrice); + } + } + return distinctDealPrices; + } + private List findDealPricesByName(String city, String region, String name, int max, DealPriceModel model) { List params = new ArrayList(); @@ -5959,4 +6033,8 @@ public class PropertyController { public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) { this.communityBasePriceService = communityBasePriceService; } + + public void setAliasFangService(AliasFangService aliasFangService) { + this.aliasFangService = aliasFangService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Test.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Test.java new file mode 100644 index 000000000..e03484f3e --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/Test.java @@ -0,0 +1,190 @@ +//package com.fangjia.open.controllers; +// +//import com.google.gson.Gson; +//import com.google.gson.JsonArray; +//import com.google.gson.JsonObject; +//import org.apache.http.HttpEntity; +//import org.apache.http.client.methods.CloseableHttpResponse; +//import org.apache.http.client.methods.HttpPost; +//import org.apache.http.entity.ContentType; +//import org.apache.http.entity.StringEntity; +//import org.apache.http.impl.client.CloseableHttpClient; +//import org.apache.http.impl.client.HttpClients; +//import org.apache.http.util.EntityUtils; +//import sun.misc.BASE64Encoder; +// +//import java.io.IOException; +//import java.net.Authenticator; +//import java.net.InetSocketAddress; +//import java.net.PasswordAuthentication; +//import java.net.Proxy; +//import java.nio.charset.Charset; +//import java.nio.charset.StandardCharsets; +//import java.util.Base64; +//import java.util.HashMap; +//import java.util.Map; +// +////public class Test { +//// public static void main(String[] args) throws IOException { +//// // 目标URL +//// String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; +//// +//// // 设置代理 +//// String proxyHost = "tunnel2.qg.net"; +//// int proxyPort = 14186; +//// +//// // 设置代理用户名和密码 +//// final String proxyUsername = "113BBC06"; +//// final String proxyPassword = "3B2FBD05B0AF"; +//// +//// // 设置请求的JSON数据 +//// String address = "江苏省扬州市宝应县安宜镇宝应大道88号"; +//// String type = "1"; +//// String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}"; +//// +//// // 设置HTTP基本认证 +//// String authKey = proxyUsername; +//// String password = proxyPassword; +//// String auth = authKey + ":" + password; +//// String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8"))); +//// +//// // 创建HTTP客户端 +//// CloseableHttpClient httpClient = HttpClients.createDefault(); +//// HttpPost httpPost = new HttpPost(targetURL); +//// +//// // 设置代理 +//// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); +//// +//// // 设置代理认证 +//// Authenticator.setDefault(new Authenticator() { +//// protected PasswordAuthentication getPasswordAuthentication() { +//// return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()); +//// } +//// }); +//// +//// // 设置请求头 +//// httpPost.setHeader("Content-Type", "application/json"); +//// httpPost.setHeader("Accept", "*/*"); +//// httpPost.setHeader("Host", "146.56.217.44:9010"); +//// httpPost.setHeader("Proxy-Authorization", authHeader); +//// +//// // 设置请求体 +//// StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON); +//// httpPost.setEntity(stringEntity); +//// +//// // 发送POST请求 +//// CloseableHttpResponse httpResponse = httpClient.execute(httpPost); +//// +//// // 获取响应 +//// HttpEntity responseEntity = httpResponse.getEntity(); +//// String responseString = EntityUtils.toString(responseEntity); +//// System.out.println(responseString); +//// +//// // 关闭连接 +//// httpResponse.close(); +//// httpClient.close(); +//// } +////} +// +//public class Test { +// public static void main(String[] args) throws IOException { +// Map map = senRequest("江苏省泰州市靖江市通江路37号", "1"); +// System.out.println(map); +// +// } +// +// private static Map senRequest(String address, String type) throws IOException { +// String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; +// String proxyHost = "tunnel2.qg.net"; +// int proxyPort = 14186; +// final String proxyUsername = "113BBC06"; +// final String proxyPassword = "3B2FBD05B0AF"; +// String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}"; +// String authKey = proxyUsername; +// String password = proxyPassword; +// String auth = authKey + ":" + password; +// String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8"))); +// +// CloseableHttpClient httpClient = HttpClients.createDefault(); +// HttpPost httpPost = new HttpPost(targetURL); +// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); +// Authenticator.setDefault(new Authenticator() { +// protected PasswordAuthentication getPasswordAuthentication() { +// return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()); +// } +// }); +// httpPost.setHeader("Content-Type", "application/json"); +// httpPost.setHeader("Accept", "*/*"); +// httpPost.setHeader("Host", "146.56.217.44:9010"); +// httpPost.setHeader("Proxy-Authorization", authHeader); +// StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON); +// httpPost.setEntity(stringEntity); +// +// CloseableHttpResponse httpResponse = null; +// int retryCount = 2; // 设置重试次数 +// int retryDelay = 10; // 设置重试延迟时间(毫秒) +// boolean success = false; +// for (int i = 0; i < retryCount; i++) { +// try { +// httpResponse = httpClient.execute(httpPost); +// success = true; +// break; +// } catch (IOException e) { +// System.err.println("代理请求失败,进行重试..."); +// try { +// Thread.sleep(retryDelay); +// } catch (InterruptedException ex) { +// Thread.currentThread().interrupt(); +// } +// } +// } +// +// if (success) { +// HttpEntity responseEntity = httpResponse.getEntity(); +// String responseString = EntityUtils.toString(responseEntity); +// +// // 使用 Gson 解析 JSON 字符串 +// Gson gson = new Gson(); +// JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class); +// +// +// // 创建 Map 存储各个值 +// Map dataMap = new HashMap(); +// +// // 获取各个值并放入 Map +// dataMap.put("code", jsonObject.get("code").getAsString()); +// dataMap.put("msg", jsonObject.get("msg").getAsString()); +// JsonObject data = jsonObject.getAsJsonObject("data"); +// dataMap.put("version", data.get("version").getAsString()); +// dataMap.put("province", data.get("province").getAsString()); +// dataMap.put("city", data.get("city").getAsString()); +// dataMap.put("district", data.get("district").getAsString()); +// dataMap.put("town", data.get("town").getAsString()); +// dataMap.put("village", data.get("village").getAsString()); +// dataMap.put("provinceCode", data.get("provinceCode").getAsString()); +// dataMap.put("cityCode", data.get("cityCode").getAsString()); +// dataMap.put("districtCode", data.get("districtCode").getAsString()); +// dataMap.put("townCode", data.get("townCode").getAsString()); +// dataMap.put("villageCode", data.get("villageCode").getAsString()); +// dataMap.put("areaCode", data.get("areaCode").getAsString()); +// dataMap.put("farmer", data.get("farmer").getAsString()); +// dataMap.put("isGovt", data.get("isGovt").getAsString()); +// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空 +// JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray(); +// if (indusParkList != null) { +// JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class); +// dataMap.put("devName",jsonObject1.get("indusParkName").getAsString()); +// dataMap.put("devLevel",jsonObject1.get("indusParkLevel").getAsString()); +// } +// +// httpResponse.close(); +// return dataMap; +// } else { +// System.err.println("重试次数已用完,请求失败。"); +// } +// +// httpClient.close(); +// return null; +// } +//} +// diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java index 8e9097e2a..aac0f83b2 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/PropertyDto.java @@ -15,6 +15,10 @@ public class PropertyDto { private String propertyType; private String block; private String distance; + /** + * 新增小区别名 + */ + private String alias; public String getId() { return id; } @@ -88,4 +92,12 @@ public class PropertyDto { public void setDistance(String distance) { this.distance = distance; } + + public String getAlias() { + return alias; + } + + public void setAlias(String alias) { + this.alias = alias; + } } diff --git a/projects/fangjia-priceanalysis/src/main/webapp/task/country.ftl b/projects/fangjia-priceanalysis/src/main/webapp/task/country.ftl index 103e932ed..0e784d6b0 100644 --- a/projects/fangjia-priceanalysis/src/main/webapp/task/country.ftl +++ b/projects/fangjia-priceanalysis/src/main/webapp/task/country.ftl @@ -5,7 +5,7 @@ 鍩庡競鍦板浘 - +
diff --git a/projects/fangjia-priceanalysis/src/main/webapp/task/map.ftl b/projects/fangjia-priceanalysis/src/main/webapp/task/map.ftl index e43c784d4..df0e86ddb 100644 --- a/projects/fangjia-priceanalysis/src/main/webapp/task/map.ftl +++ b/projects/fangjia-priceanalysis/src/main/webapp/task/map.ftl @@ -6,7 +6,7 @@ 鍦板浘 - +
diff --git a/projects/fangjia-priceanalysis/src/main/webapp/task/schools.ftl b/projects/fangjia-priceanalysis/src/main/webapp/task/schools.ftl index 7111b91c2..0e954b8a9 100644 --- a/projects/fangjia-priceanalysis/src/main/webapp/task/schools.ftl +++ b/projects/fangjia-priceanalysis/src/main/webapp/task/schools.ftl @@ -5,7 +5,7 @@ 瀛︽牎鍦板浘 - +
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy index c2f4eaee4..8c6633b4d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/FindStageDistrictBatch.groovy @@ -9,7 +9,7 @@ import com.mongodb.BasicDBObject import org.apache.commons.lang.StringUtils import java.util.regex.Pattern - +// 分期分区与主小区映射关系表 @BatchOwner(Watcher.WangWenYan) class FindStageDistrictBatch extends BaseBatch { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index a7b589cf9..cb4abf8cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -75,18 +75,22 @@ for(def d : datas){ map.area = d['F'].toString().toDouble() map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) map.avg_price = (int)(d['H'].toString().toDouble()) - map.sell_price = (int)(d['I'].toString().toDouble()*10000) - def trade_date = d['J'] - if(trade_date){ - map.trade_date = trade_date - }else{ - def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30") - map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") + if(d['I']){ + map.sell_price = (int)(d['I'].toString().toDouble()*10000) } +// def trade_date = d['J'] +// if(trade_date){ +// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd') +// }else{ + def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31") + map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") +// } + map.source = "深房家族" map.create_date = new Date() map.m_date = new Date() + println map DEAL_PRICE_COLL.save(map) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy index ca14ea514..e0da72624 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy @@ -23,7 +23,7 @@ class AddThirdPartyPrice { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def c : citys){ println c +" "+cityCodeDatas.get(c) - def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c) def rets = createDate(dists, distCodeMap) println dists.size() +" "+rets.size() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 431972b91..2eda22bba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{ def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null); def distMap = [:] for(def d : districts){ def alias = d.alias; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 0ea40b8c2..47159ee0e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -58,7 +58,7 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null) def ccbMaps = null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 4d473e36a..bcd484c2b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl import groovy.sql.Sql +import org.bson.types.ObjectId import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -61,13 +62,21 @@ class BigdataDealBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) + def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd") + def objectId = dateToObjectId(date) + println objectId + + //664a22000000000000000000 5-20 + //664b73800000000000000000 5-21 + def citys = CcbCommonUtils.getRunningCitys(args) +// +// println "curDate:"+curDate +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +// addDealPriceMatch(city); +// } +// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月 - println "curDate:"+curDate - def allCitys = CommonDatautils.findAllCity() - for(def city : allCitys){ - addDealPriceMatch(city); - } def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] @@ -529,7 +538,7 @@ class BigdataDealBatch extends BaseBatch{ def getCityData(def city, def statMap, def ppeDistsMap){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null) println city +" 小区量:"+dists.size() def tradesList = Collections.synchronizedList(new ArrayList()); def totalCount = new AtomicLong() @@ -679,6 +688,16 @@ class BigdataDealBatch extends BaseBatch{ return map } + def updateCrawlTradeDate(){ + def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20" + def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd") + def objectId = dateToObjectId(date) + def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2) + def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1) + println(objectId) + CcbCommonUtils.updateTrades(startDate, endDate, objectId) + } + def addDealPriceMatch(city){ // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) @@ -816,4 +835,15 @@ class BigdataDealBatch extends BaseBatch{ println "end: "+new Date() } + + + def dateToObjectId(Date date) { + // 获取时间戳(单位:秒) + def timestamp = (date.time / 1000) as int + def hexTimestamp = Integer.toHexString(timestamp) + + // 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值 + def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000' + return new ObjectId(objectIdStr) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 8b444eaba..e329c8732 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter); //测试通过匹配接口配房源效果 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 65f84ab77..7a4062270 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -15,9 +15,10 @@ import groovy.sql.Sql class CcbCommonUtils { - // -// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] - public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] + + public static def citys_fjw_all = ["深圳","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"] + + public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"] public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"] public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"] @@ -69,6 +70,10 @@ class CcbCommonUtils { citys = CcbCommonUtils.citys_252 sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + }else if(countStr=="all"){ + citys = CcbCommonUtils.citys_fjw_all + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } }else{ citys = CcbCommonUtils.citys_all @@ -428,14 +433,14 @@ class CcbCommonUtils { return rows } - public static def findAllDistrictByCity(def city, def cat, def fields){ + public static def findAllDistrictByCity(def city, def cat, def fields, def kind){ DBObject q = new BasicDBObject(); q.put("status",0) q.put("cat", cat); q.put("city", city); // q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 -// q.put("name","汤堡小区") +// q.put("name","鸿运大院") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) @@ -445,14 +450,13 @@ class CcbCommonUtils { keys.put(field, 1); } } -// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000) println "[${city}] 原始:"+dbsour.count() def rets = [] def ids = [] for(def d : dbsour){ - if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ + if(kind){ if(d.address){ d.address = d.address.replaceAll("\r|\n|\\s", ""); } @@ -460,6 +464,16 @@ class CcbCommonUtils { ids << d._id.toString() rets << d } + }else{ + if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ + if(d.address){ + d.address = d.address.replaceAll("\r|\n|\\s", ""); + } + if(!ids.contains(d._id.toString())){ + ids << d._id.toString() + rets << d + } + } } } println "[${city}] 小区量:"+rets.size() @@ -665,10 +679,6 @@ class CcbCommonUtils { } public static def deleteGuidedPrice(def dealdate){ - def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')" - sqlInstance.executeUpdate(sql) - - def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'" pgsqlInstance.executeUpdate(sql2) } @@ -702,4 +712,16 @@ class CcbCommonUtils { return map } + public static def updateTrades(startDate, endDate, cDate){ + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + q.put("_id", new BasicDBObject('$gte', cDate)) + def count = DEAL_PRICE_COLL.count(q) + if(count>0){ + def udata = new BasicDBObject() + udata.put("trade_date", endDate) + DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata)) + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index fca28ce3e..5072828fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -388,19 +388,21 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水" - def citys = ["469005"]//CcbCommonUtils.citys_all +// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all + def citys = ["上海","广州"]//CcbCommonUtils.citys_all def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() - for(def cityid : citys) { + for(def city : citys) { + def cityid = cityCodeDatas.get(city) def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 - println cityid +" "+rets.size() + println city +" "+rets.size() if(rets){ - set.add(cityid) + set.add(city) } for(def m : rets){ -// def cityId = cityCodeDatas.get(city) + // def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" // sqlInstance236.executeUpdate(sql) def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" @@ -409,7 +411,7 @@ class DealPriceDataProcess { total = r.count }) if(total>0){ - println cityid +" "+m.commid+" (按价格)删除:"+total + println city +" "+m.commid+" (按价格)删除:"+total def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" pgsqlInstance.executeUpdate(sql) }else{ @@ -419,7 +421,7 @@ class DealPriceDataProcess { total = r.count }) if(total>0){ - println cityId +" "+m.commid+" (按小区)删除:"+total + println city +" "+m.commid+" (按小区)删除:"+total def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} " pgsqlInstance.executeUpdate(sql) } @@ -450,7 +452,7 @@ class DealPriceDataProcess { } def findPeGteThirdParty2(cityid, yearmonth){ - def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20" + def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25" def rets = [] pgsqlInstance.eachRow(sql,{r-> def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index ed5ed5f03..fdf28955b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null); def distMap = [:] for(def d : districts){ distMap.put(d.region+"##"+d.name, d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 3c2ebd153..c43a791b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28") + def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -207,7 +207,7 @@ class GuiDingPriceBatch extends BaseBatch{ } def getDistrictMap(def city){ - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null); def distMap = [:] for(def d : districts){ distMap.put(d.region+"##"+d.name, d) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy index cfbd78a2d..0a1584be9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -2,6 +2,9 @@ package com.fangjia.script.ccbevaluation import groovy.sql.Sql +/** + * 建行使用 + */ class PgsqlCommUtils { public static def database = "warehouse" @@ -19,6 +22,9 @@ class PgsqlCommUtils { } + + + public static def findAllDistrictCount(def cityid){ def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index cae04cf55..82d2c96c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{ def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] for(def city : citys){ - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null) def fenQiMap = getFenQiMap(dists); def rets = createCityData(dists, fenQiMap) rows.addAll(rets) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 3979f56c4..d527b08dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.looplineinfo.Point @@ -1288,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx" ExcelUtils.multiThreadingModify(fileName, {d-> - if(d['A']){ + if(d['J']){ // def region = RegionData.formatRegion(d['B'], d['C']) // if(region){ // def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") @@ -1300,22 +1301,81 @@ ExcelUtils.multiThreadingModify(fileName, {d-> // d['H'] = dist._id.toString() // } // } - if(!d['I']){ - def dist = CommonDatautils.findDistrictById(d['C']) + if(!d['A'] && d['J'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['J']) if(dist){ - d['F'] = dist.region + d['A'] = dist.city + d['B'] = dist.region + d['C'] = dist.name +// d['K'] = dist.estate_type2 +// d['L'] = dist.status +// d['M'] = dist.name_cat + // d['H'] = dist.name - d['G'] = dist.status -// if(d['A']=="天门" || d['A']=="迪庆"){ -// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200) -// d['K'] = val.toInteger() -// } } } } },0) +//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx" +//ExcelUtils.multiThreadingModify(filePath, { d -> +// if(d['B']){ +// +// def obj = CommonDatautils.findDistrictById(d['B']) +// if(obj){ +// def markMap = obj.mark_map +// if(!obj.mark_map){ +// markMap = [:] +// } +// def data = new BasicDBObject() +// markMap.put("exclusive", "是") +// data.put("mark_map", markMap) +// data.put("m_date", new Date()) +// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data)) +// } +// } +// +//}) + + +//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx" +//ExcelUtils.multiThreadingModify(fileName, { d -> +// if(d['B'] && !d['D']){ +// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district") +// if(jsonObject != null && jsonObject.credit>0) { +// d['D'] = jsonObject.get("city") +// d['E'] = jsonObject.get("district") +// d['F'] = jsonObject.get("name") +// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id")) +// if(dbo){ +// d['G'] = dbo.status +// d['H'] = dbo.name_cat +// d['I'] = dbo.estate_type2 +// d['J'] = "接口匹配" +// } +// } +// +//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district") +//// if(!objs){ +//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district") +//// } +//// if(objs && objs.size()==1){ +//// def o = objs[0] +//// if(o){ +//// d['D'] = o.city +//// d['E'] = o.region +//// d['F'] = o.name +//// d['G'] = o.status +//// d['H'] = o.name_cat +//// d['I'] = o.estate_type2 +//// d['J'] = "别名" +//// } +//// } +// +// +// } +//}) //def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx" //ExcelUtils.multiThreadingModify(fileName, { d -> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8483a84a0..4e70b43c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -55,7 +55,7 @@ class ThirdPartyPriceBatch extends BaseBatch { // } // } // - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) @@ -71,7 +71,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def statMap = getDistrictStatByCity(city, weekend) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"]) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null) def thirdPriceMap = findThirdPartyPrice(city, statMap) //def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index 5a38c237e..535748054 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{ def size = regionDatas.size() final CountDownLatch latch = new CountDownLatch(size) def totalCount = new AtomicLong() + for(def i = 0;i3){ - sell_avg_price = avgValue(sellMap.avg_price) + if(sellMap){ sell_avg_area = avgValue(sellMap.area) - sell_avg_total = avgValue(sellMap.total_price) - }else{ - sell_count = 0 - sellDataType = null + sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price) +// if(sell_count>3){ +// if(allCcbPriceMap.containsKey(dist._id.toString())){ +// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price) +// sell_avg_area = avgValue(sellMap.area) +// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price) +// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString()) +// }else{ +// sell_count = 0 +// sellDataType = null +// } +// }else{ +// sell_count = 0 +// sellDataType = null +// } } } + def rent_avg_price = null def rent_avg_area = null def rent_avg_total = null @@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{ rentDataType = null } } - if(!sell_avg_price){ - sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) - } + + //三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格 +// if(!sell_avg_price){ +// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) +// } if(!rent_avg_price){ rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name) } @@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{ def sell_avg_area = null def sell_avg_total = null def sellStartDate = null + + def avgPrice = null + if(region){ + avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6)) + }else{ + avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6)) + } + + if (sell_count < 10) { sellDataType = 3 sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger(); @@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{ sellStartDate = startDate } if (sell_count) { - sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price") + sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price") sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area") sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price") } @@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{ return datas; } + def findCommunityPriceByCityMap(cityCode, month){ + def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; + def dataMap = [:] + pgsqlInstance.eachRow(sql, { r -> + dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc) + }) + return dataMap; + } + + def findCommunityPriceCountByCityMap(cityCode, month){ + def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}"; + def dataMap = [:] + pgsqlInstance.eachRow(sql, { r -> + if(r.gujiabdsrt3mlgcasenum){ + dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum) + } + }) + return dataMap; + } + def findCcbCityPrice(cityCode, month){ def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}"; def avgPrice = null @@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{ return regionPriceMap; } + def findCcbRegionPriceByRegion(cityCode, regionCode, month){ + def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'"; + def price = null + pgsqlInstance.eachRow(sql, { r -> + price = r.gujia_bds_clc_avprc + }) + return price; + } + def districtCityPrictRate(cityPrice, datas){ def distCityRateMap = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index d2ad41623..286172afd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -159,7 +159,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat" } @@ -246,7 +246,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat" } @@ -304,7 +304,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat" } @@ -362,7 +362,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat" } @@ -425,7 +425,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat" } @@ -504,7 +504,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat" } @@ -590,7 +590,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { rets << ret } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" + def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 50cc6343f..846a244a8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -4,17 +4,15 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.ccbevaluation.CcbCommonUtils -import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils -import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql -import java.sql.ResultSet -import java.sql.ResultSetMetaData - +/** + * 建行融合数据导出 + */ @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHe extends BaseBatch{ @@ -239,7 +237,7 @@ class ExportDistrictRongHe extends BaseBatch{ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'"; + def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'"; pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id @@ -255,23 +253,23 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" // FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_ - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "EVALUATE_RATE_CITY_FJW_ADD 数据文件输出完成......" + println "EVALUATE_RATE_CITY_FJW_HIS_ADD 数据文件输出完成......" CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "EVALUATE_RATE_CITY_FJW_ADD 上传Ftp完成......" + println "EVALUATE_RATE_CITY_FJW_HIS_ADD 上传Ftp完成......" } def exportEvaluateRateComm(queryDate){ def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' "; + def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' "; pgsqlInstance.eachRow(sql, { r -> def data = [] data << r.urbn_id @@ -284,6 +282,7 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.unitprc_avg data << r.comm_type data << r.area_avg_coef + data << r.unitprc_avg_coef data << r.basic_price_coef data << r.buildarea_coef data << r.type_coef @@ -296,16 +295,16 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" // FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 数据文件输出完成......" + println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 数据文件输出完成......" CcbFtpUtils.ftpUpload("rh",fileName, curDate) - println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 上传Ftp完成......" + println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 上传Ftp完成......" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e0e58814f..e9fa92613 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.commData import com.fangjia.core.RegionData +import com.fangjia.datastore.driver.FacetResult import com.fangjia.httpclient.UserAgents import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.looplineinfo.InsidePolygon @@ -8,11 +9,18 @@ import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.JacksonUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.AggregationOutput +import com.mongodb.client.MongoCursor +import com.mongodb.client.model.Accumulators +import com.mongodb.client.model.Aggregates +import com.rabbitmq.client.impl.AMQImpl import groovy.json.JsonSlurper import net.sf.json.JSONArray import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils +import org.apache.poi.ss.util.RegionUtil +import org.bson.Document import org.bson.types.ObjectId import com.fangjia.core.util.HttpRequest; @@ -110,6 +118,8 @@ class CommonDatautils { FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) } + public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009") + public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { @@ -267,7 +277,7 @@ class CommonDatautils { q.put("status", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","泰瑞家园") +// q.put("name","托斯卡纳庄园") DBObject keys = new BasicDBObject(); if(fields){ @@ -276,7 +286,7 @@ class CommonDatautils { } } - def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000) + def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000) List rets = [] for(def obj : cursor){ rets << obj @@ -1459,6 +1469,28 @@ class CommonDatautils { return obj; } + static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + q.put('status',0) + + BasicDBObject query = new BasicDBObject(); + query.putAt(QueryOperators.IN, [name]); + q.put("mapping_alias", query); + + def dbcursor = SEAWEED_COLL.find(q); + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } + static def findSeaweeds(city, cat, name_cat){ DBObject q = new BasicDBObject(); q.put('city',city) @@ -1691,7 +1723,34 @@ class CommonDatautils { } return regionMap; } - + + + public static def getAllCityFullRegions(){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("category","region"); + //queryCondition.put("show",1); + def dbCursor = REGION_BLOCK_COLL.find(queryCondition) + def regionMap = [:]; + for(DBObject obj : dbCursor){ + def city = obj.get("city"); + def region = obj.get("name"); + def pattern = obj.get("pattern"); + if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) { + def r_map = regionMap.get(city); + if(!r_map){ + r_map = [:] + regionMap.put(city, r_map); + } + + if(!pattern){ + pattern = []; + pattern << region; + } + r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]); + } + } + return regionMap; + } public static def getRegionsByCity(def city){ @@ -1834,6 +1893,25 @@ class CommonDatautils { def dbo = null; while(cur.hasNext()) { dbo = cur.next() + if(dbo.value_type == 1){ + dbo.cal_type = "指导价" + }else if(dbo.value_type == 2){ + dbo.cal_type = "自身样本" + }else if(dbo.value_type == 3){ + dbo.cal_type = "第三方" + }else if(dbo.value_type == 4){ + dbo.cal_type = "交易数据" + }else if(dbo.value_type == 5){ + dbo.cal_type = "周边楼盘" + }else if(dbo.value_type == 6){ + dbo.cal_type = "推导" + }else if(dbo.value_type == 9){ + dbo.cal_type = "神经网络" + }else if(dbo.value_type == 10){ + dbo.cal_type = "无方式" + }else{ + dbo.cal_type = "" + } } return dbo; } @@ -4496,6 +4574,28 @@ class CommonDatautils { return townMap } + static def findAllTownRanges(){ + def q = new BasicDBObject() + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("town", 1) + def dbcursor = TOWN_POLYLINECOLL.find(q, keys) + def townMap = [:] + for(def d : dbcursor){ + def lists = townMap.get(d.city) + if(!lists){ + lists = [] + } + def t = [:] + t.region = d.region + t.town = d.town + lists << t + townMap.put(d.city, lists) + } + return townMap + } + static def queryFullNameCity(){ def q = new BasicDBObject() def dbcursor = administrative_division_100_CONN.find(q) @@ -4530,6 +4630,108 @@ class CommonDatautils { } + static def queryFullNameByCity(){ + def q = new BasicDBObject() + q.put("type","城市") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q) + def cityMap = [:] + for(def d : dbcursor){ + def city = RegionData.formatCity(d.city) + if(city){ + cityMap.put(city, d) + }else{ + //println d.city +" 格式化不出" + } + + } + return cityMap + } + + static def queryDistNameByCity(){ + + def q = new BasicDBObject() + q.put("type","区域") + def keys = new BasicDBObject() + keys.put("province",1) + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + keys.put("mzb_district_name", 1) + keys.put("shape", 1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000) + def regionMap = [:] + for(def d : dbcursor){ + if(d.mzb_district_name){ + def regions = regionMap.get(d.city) + if(!regions){ + regions = [] + } + regions << d + regionMap.put(d.city, regions) + }else{ + println d.province+" "+d.district +" 格式化不出民政部区域" + } + } + return regionMap + } + + static def queryCommunityByCity(){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + q.put("province",new BasicDBObject('$in', ['福建省','重庆市'])) + + def keys = new BasicDBObject() + keys.put("province",1) + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + keys.put("shape",1) + keys.put("alias",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) + def townMap = [:] + for(def d : dbcursor){ + def key = d.city+"#"+d.district + def list = townMap.get(key) + if(!list){ + list = [] + } + list << d + townMap.put(key, list) + } + return townMap + + } + + + static def queryNewTownByCity(){ + + def q = new BasicDBObject() + q.put("type","乡镇街道") + + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("district",1) + keys.put("street",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000) + def townMap = [:] + for(def d : dbcursor){ + def key = d.city + def list = townMap.get(key) + if(!list){ + list = [] + } + def t = [:] + t.region = d.district + t.town = d.street + list << t + townMap.put(key, list) + } + return townMap + + } + + static def queryDistNameCity(){ def q = new BasicDBObject() @@ -4600,5 +4802,55 @@ class CommonDatautils { } + static def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + if(addressComponent.city.size()>0){ + city = addressComponent.city + }else{ + if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){ + city = addressComponent.district + }else{ + city = addressComponent.province + } + + } + + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } + + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictMatch.groovy new file mode 100644 index 000000000..ee24ae8ae --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictMatch.groovy @@ -0,0 +1,3 @@ +package com.fangjia.script.fuzhou + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index beeafcb09..1930b6d2d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -283,7 +283,12 @@ class DistrictSellStatBatch extends BaseBatch { def dbcursor = DISTRICT_SELL_STAT.find(q) def priceMap = [:] for(def d : dbcursor){ - priceMap.put(d.region +"##"+d.district_name, d) + if(d.estate_type=="别墅"){ + priceMap.put(d.region +"##"+d.district_name+"##别墅", d) + }else{ + priceMap.put(d.region +"##"+d.district_name+"##小区", d) + } + } return priceMap } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 893ad9876..585e7bc0b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -29,11 +29,11 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() -// task.priceMarkupRun(); + task.priceMarkupRun(); // task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() - task.testNewPrice() +// task.testNewPrice() } def margeExcel(){ @@ -58,7 +58,7 @@ class FzDataResultStat { } def priceMarkupRun(){ def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15"); - def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) // def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"]) @@ -82,7 +82,7 @@ class FzDataResultStat { } // List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) - def list2 = findFzDistrictStat(lastMonth) + def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"))) def map2 = [:] for(def obj : list2) { def key = obj.city + "##" + obj.region + "##" + obj.name diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy index 2b96edc91..d4e07eab4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDistrictMatch.groovy @@ -22,8 +22,8 @@ class FzDistrictMatch { // task.addPrice() // task.addThirdPrice() -// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") - task.longyanPrice() + task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx") +// task.longyanPrice() } def longyanPrice(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index ad2cf8430..f96bf753a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index db6f86848..1b11ee16a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -238,6 +238,7 @@ class FiledStrand { } static def getValue(def val){ + val = val.replace(",","") val = getNumber(val); if(val){ if(val.indexOf(".")>-1 && !val.toString().endsWith(".0")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index a2ec7c80b..2003b4c4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -1,23 +1,115 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.poi.ss.util.RegionUtil +import org.bson.types.ObjectId class DevProcess { def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + + + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + static main(args) { def task = new DevProcess(); - task.run() +// task.run() +// task.parseCityAndDist() + task.devSplitRoad() +// task.devNameSplitRoad() +// task.addFlag() +// task.matchDev() + +// task.calAddrDistince() +// task.updateCityAndDist() +// task.addDevAddres1() //按添加地址中包含开发区名称的地址 +// task.addDevAddres2()//添加相差2公里范围内的地址 +// task.matchDevAndAddAddress() + +// task.stat() +// task.addNewDev() +// task.createDevMapping() +// task.deleteAddr() +// task.deleteAddr2() +// task.synchDevTo43() +// task.addNewDev2() + +// task.addAlias() +// task.addDevShape()//添加开发区围栏 +// task.matchDevTownship()//添加乡镇街道 + +// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中 + } + + + + def parseCityAndDist(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d-> + if(!d['J']){ +// def obj = getMapInfo(null, d['A']+d['C']) +// if(obj){ +// d['E'] = obj.city.toString() +// d['F'] = obj.district.toString() +// d['G'] = obj.formatted_address.toString() +// d['H'] = obj.province.toString() +// } + + def obj = FangjiaClientUtils.formatNinelevel(null, d['C']); + if(obj) { + def jsonObject = obj.get("result") + + if (jsonObject) { + if (jsonObject.get("province") && jsonObject.get("province") != "null") { + d['J'] = jsonObject.get("province") + } + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + d['K'] = jsonObject.get("city") + } + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + d['L'] = jsonObject.get("district") + } + } + } + } + }) + } + + def getMapInfo(city, address){ + def json = GaoDeApiUtils.geo(city, address) + println json + if(json && json.status=="1"){ + def ret = json.geocodes + if(ret){ + def obj = ret.get(0) + def str = obj.location + def level = obj.level + +// if(str && ["门牌号","单元号","道路","道路交叉路口","热点商圈","村庄","兴趣点","门址","住宅区"].contains(level)){ + def arrs = str.split(",") + return ["lng":arrs[0], "lat":arrs[1],"province":obj.province,"city":obj.city, "district":obj.district, "formatted_address":obj.formatted_address] +// }else{ +// println address +" "+level +// return null +// } + } + } } def run(){ def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls") @@ -77,10 +169,611 @@ class DevProcess { }else{ println udata.name +" 已存在" } - - } - } } + + def findDev(city, name){ + def q = new BasicDBObject() + q.put("name", name) + q.put("city", city) + def obj = development_zone_CONN.findOne(q) + return obj + } + + def devSplitRoad(){ + def i = 0; + while(i<20){ + def fileName = "/home/goojia/gj/qzd-${i}.xlsx" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx" + } + + File file=new File(fileName); + if(file.exists()){ + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['E'] && !d['G']){ + def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E']) + if(jsonObject1) { + if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") { + def devName = jsonObject1.get("dev") + def obj = findDev(d['B'], devName) + d['F'] = devName + if (obj) { + d['Q'] = obj._id.toString() + d['R'] = obj.name + } + } + } + } + + if(d['C'] && !d['S']) { + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C']) + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if (jsonObject.get("province") && jsonObject.get("province") != "null") { + d['G'] = jsonObject.get("province") + } + + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + d['H'] = jsonObject.get("city") + } + + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + d['I'] = jsonObject.get("district") + } + + if (jsonObject.get("township") && jsonObject.get("township") != "null") { + d['J'] = jsonObject.get("township") + } + + if (jsonObject && street && streetNumber && street != "null") { + d['K'] = jsonObject.get("street") + def road = jsonObject.get("street") + if (streetNumber != 'null') { + d['L'] = jsonObject.get("streetNumber") + road = road + jsonObject.get("streetNumber") + } else { + d['N'] = "信息不完整" + } + + if (!d['N'] && !d['C'].contains(road)) { + d['N'] = "原地址不含路" + } + + if (!d['N']) { + def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$') + if (!flag) { + d['N'] = "路非法" + } + def roadsize = jsonObject.get("street").size() + if (!d['N'] && (roadsize < 3 || roadsize > 7)) { + d['N'] = "路太长|路太短" + } + } + }else{ + d['N'] = "无地址" + } + + if (jsonObject.get("dev") && jsonObject.get("dev") != "null") { + def devName = jsonObject.get("dev") + def obj = findDev(d['B'], devName) + d['M'] = devName + if (obj) { + d['S'] = obj._id.toString() + d['T'] = obj.name + } + } + } + },0) + i++ + } + } + + } + + + + def devNameSplitRoad(){ + def fileName = "/home/goojia/gj/company.xlsx" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\数据建设\\开发区\\56万需修改地址.xlsx" + } + ExcelUtils.multiThreadingModify(fileName,{d-> + if(d['P'] && d['P']=="路正常" && !d['O']){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E']) + if(jsonObject){ + def street = jsonObject.get("street") + def streetNumber = jsonObject.get("streetNumber") + if(jsonObject.get("province") && jsonObject.get("province")!="null"){ + d['J'] = jsonObject.get("province") + } + + if(jsonObject.get("city") && jsonObject.get("city")!="null"){ + d['K'] = jsonObject.get("city") + } + + if(jsonObject.get("district") && jsonObject.get("district")!="null"){ + d['L'] = jsonObject.get("district") + } + + if(jsonObject && street && streetNumber && street!="null"){ + d['M'] = jsonObject.get("street") + if(streetNumber!='null'){ + d['N'] = jsonObject.get("streetNumber") + } + } + + if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){ + d['O'] = jsonObject.get("dev") + } + } + + } + },0) + } + + def addFlag(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d-> + if(d['H'] && d['D']){ + if(d['D'].contains(d['H'])){ + d['O'] = "是" + }else{ + def newName = null + if(d['H'].toString().startsWith(d['I'])){ + newName = d['H'].toString().substring(d['I'].toString().length()) + }else if(d['H'].toString().startsWith(d['I']+"省")){ + newName = d['H'].toString().substring(d['I'].toString().length()+1) + } + if(newName && d['D'].contains(newName)){ + d['O'] = "是2" + } + } + } + },1); + } + + def matchDev(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d-> + if(!d['Q'] && d['L']){ + def q = new BasicDBObject() + q.put("name", d['L']) + q.put("city",d['G']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['Q'] = obj._id.toString() + d['R'] = obj.name + } + } + },0); + } + + def calAddrDistince(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> + if(d['M'] && d['N'] && d['Y'] && d['Z']){ + def addr1 = getFullAddress(d['J'], d['K'],d['L'],d['M'] , d['N']) + def addr2 = getFullAddress(d['V'], d['W'], d['X'], d['Y'] , d['Z']) + + def json1 = GaoDeApiUtils.geo(d['K'], addr1) + def json2 = GaoDeApiUtils.geo(d['W'], addr2) + def arrs1 = getLocation(json1) + def arrs2 = getLocation(json2) + if(arrs1 && arrs2){ + def distance = GeoUtils.calculateDistance(arrs1[0].toString().toDouble(), arrs1[1].toString().toDouble(), arrs2[0].toString().toDouble() , arrs2[1].toString().toDouble() ) + d['AA'] = distance + } + } + },0); + } + + def getFullAddress(province, city, region, street, streetName){ + def addr = "" + if(addr){ + addr = province + } + if(city){ + addr = addr + city + } + if(region){ + addr = addr + region + } + addr = addr + street + streetName + return addr + } + def getLocation(json1){ + def arrs1 = null + if(json1 && json1.status=="1") { + def ret = json1.geocodes + if (ret) { + def str = ret.get(0).location + if(str){ + arrs1 = str.split(",") + } + } + } + return arrs1 + } + + def updateCityAndDist(){ + ExcelUtils.modify("D:\\数据\\数据建设\\等级开发区 - 汇总.xlsx",{d-> + if(d['L']){ + def udata = new BasicDBObject() + if(d['L']=="采用拆分结果"){ + if(d["I"]){ + udata.put("province", d["I"]) + } + if(d["J"]){ + udata.put("city", d["J"]) + } + if(d["K"]){ + udata.put("district", d["K"]) + } + }else if(d['L']=="采用地图结果"){ + if(d["H"]){ + udata.put("province", d["H"]) + } + if(d["E"]){ + udata.put("city", d["E"]) + } + if(d["F"]){ + udata.put("district", d["F"]) + } + } + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['C'])) + if(obj && (!obj.city && !obj.district)){ + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + } + }) + } + + def addDevAddres1(){ + def rets = [] + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d-> + if(d['O'] && d['P']=="正常" && d['U']){ + def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + if(obj){ + d['AB'] = "存在" + def addr_alias = obj.addr_alias + if(!addr_alias){ + addr_alias = [] + } + def old_count = addr_alias.size() + if(!addr_alias.contains(d['M']+d['N'])){ + addr_alias << d['M']+d['N'] + } + if(old_count != addr_alias.size()){ + def udata = new BasicDBObject() + udata.put("addr_alias", addr_alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }else{ + rets << d + println "匹配:"+d['U']+" 原始:"+d['H'] + } + } + },0) +// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增") + } + + def addDevAddres2(){ + + def rets = [] + ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d-> + if((d['R'] && d['R'].toString()=="正常") || (d['P'] && d['P'].toString()=="正常")){ + def obj = null + if(d['U']){ + try{ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + }catch(Exception e){ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U'])) + } + } + + if(!obj){ + try{ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H'])) + }catch(Exception e){ + obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H'])) + } + } + if(obj){ + d['AB'] = "存在" + def addr_alias = obj.addr_alias + if(!addr_alias){ + addr_alias = [] + } + def old_count = addr_alias.size() + + if(d['R'] && d['R']=="正常" && !addr_alias.contains(d['Y']+d['Z']) && d['X'] && d['F'] && d['X']==d['F']){ + addr_alias << d['Y']+d['Z'] + } + + if(d['P'] && d['P']=='正常' && !addr_alias.contains(d['M']+d['N']) && d['F'] && d['L'] && d['L']==d['F']){ + addr_alias << d['M']+d['N'] + } + + if(old_count != addr_alias.size()){ + def udata = new BasicDBObject() + udata.put("addr_alias", addr_alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }else{ +// rets << d + println "匹配:"+d['U']+" 原始:"+d['H'] + } + } + }, 0) + +// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增") + } + + def stat(){ + def dbcursor = development_zone_CONN.find(new BasicDBObject()) + def total = 0 + for(def d : dbcursor){ + if(d.addr_alias){ + total = total + d.addr_alias.size() + println d.addr_alias.size() + } + } + println total + } + + def addNewDev(){ + ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d-> + def udata = new BasicDBObject() + udata.address = d['M']+d['N'] + udata.create_time = new Date() + if(d['G']){ + if(d['G'].contains("国家级")){ + udata.level = "国家级" + }else if(d['G'].contains("省级")){ + udata.level = "省级" + } + } + + udata.name = d['H'] + udata.province = d['J'] + udata.city = d['K'] + udata.district = d['L'] + udata.type = d['G'] + + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + }) + } + + def addNewDev2(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d-> + if(d['F'] && d['F']=="新增"){ + def udata = new BasicDBObject() + udata.create_time = new Date() + udata.province = d['G'] + udata.city = d['I'] + udata.district = d['J'] + udata.level = "省级" + udata.name = d['D'] + + def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name)) + if(!obj){ + development_zone_CONN.save(udata) + }else{ + println udata.name +" 已存在" + } + } + }) + } + + def createDevMapping(){ + def dbcursor = development_zone_CONN.find(new BasicDBObject()) + for(def d : dbcursor){ + def formatAddress = null + if(d.address){ + formatAddress = AddressUtil.formatAddress(d.city, d.address) + } + if(formatAddress && !formatAddress.endsWith("号") && !formatAddress.endsWith("弄")){ + formatAddress = null + } + def addr_alias = [] + if(formatAddress){ + addr_alias << formatAddress + } + if(d.addr_alias){ + addr_alias = addr_alias + d.addr_alias + addr_alias.unique() + } + if(addr_alias){ + addr_alias.each{addr-> + def udata = new BasicDBObject() + udata.put("province", d.province) + udata.put("city", d.city) + udata.put("district", d.district) + udata.put("name",d.name) + udata.put("type", "地址") + if(d.township){ + udata.put("street", d.township) + } + udata.put("address", addr) + DEV_MAPPING_CONN.save(udata) + } + } + } + } + + def deleteAddr(){ + ExcelUtils.modify("D:\\数据\\数据建设\\删除开发区地址.xlsx",{d-> + DEV_MAPPING_CONN.remove(new BasicDBObject("_id", new ObjectId(d['A']))) + }); + } + + def deleteAddr2(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d-> + if(d['Q'] && d['M']){ + def q = new BasicDBObject() + q.put("_id", new ObjectId(d['Q'])) + def obj = development_zone_CONN.findOne(q) + if(obj){ + def q2 = new BasicDBObject() + q2.put("name", obj.name) + q2.put("address", d['I']+d['J']) + def sameObj = DEV_MAPPING_CONN.findOne(q2) + if(sameObj){ + DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id)) + } + } + } + + }); + } + + def synchDevTo43(){ + def dbcursor = development_zone_CONN.find() + def rets = [] + for(def r : dbcursor){ + rets << r + } + DEVELOPMENT_ZONE_43_COLL.insert(rets) + } + + + def matchDevAndAddAddress(){ + ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d-> + if(d['R']){ + def q = new BasicDBObject() + q.put("name", d['R']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + + d['P'] = obj._id.toString() + d['Q'] = obj.name + def address = d['L']+d['M'] + q.put("address", address) +// def street = d['G'] +// q.put("township", street) + def sameObj = DEV_MAPPING_CONN.findOne(q) + if(!sameObj){ + def udata = new BasicDBObject() + udata.put("province", obj.province) + if(obj.city){ + udata.put("city", obj.city) + } + if(obj.district){ + udata.put("district", obj.district) + } + + udata.put("name",obj.name) + udata.put("type", "地址") + udata.put("address",address) +// udata.put("type", "乡镇街道") +// udata.put("township", street) + DEV_MAPPING_CONN.save(udata) + println obj.name +" "+address + } + } + } + },0); + } + + + + def matchDevTownship(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> + if(d['C']){ + def q = new BasicDBObject() + q.put("name", d['C']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['E'] = obj._id.toString() + def street = d['D'] + q.put("township", street) + def sameObj = DEV_MAPPING_CONN.findOne(q) + if(!sameObj){ + def udata = new BasicDBObject() + udata.put("province", obj.province) + if(obj.city){ + udata.put("city", obj.city) + } + if(obj.district){ + udata.put("district", obj.district) + } + + udata.put("name",obj.name) +// udata.put("type", "乡镇街道") +// udata.put("township", street) + + udata.put("type", "区域") + udata.put("dev_district", street) + udata.put("c_date",new Date()) + DEV_MAPPING_CONN.save(udata) + println obj.name +" "+street + } + } + } + },1); + } + + def addAlias(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d-> + def q = new BasicDBObject() + q.put("name", d['A']) + def obj = development_zone_CONN.findOne(q) + if(obj){ + d['D'] = obj._id.toString() + def arrs = d['B'].split("、") + def alias = obj.alias + if(!alias){ + alias = [] + } + for(def s : arrs){ + if(!alias.contains(s)){ + alias << s + } + } + + def udata = new BasicDBObject() + udata.put("alias", alias) + development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + }) + } + + def addDevShape(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d-> + if(d['W']){ + def s = d['S'] + JSONObject jsonObject = JSONObject.fromObject(s); + Map map = DataHandleUtils.JsonToMap(jsonObject); + + def udata = new BasicDBObject() + udata.put("shape", map) + development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata)) + } + }) + } + + def addDevStreet(){ + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d-> + if(d['J']=="是"){ + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道")) + for(def obj : dbcursor){ + def o = new BasicDBObject() + o.put("province" , obj.province) + o.put("city" , obj.city) + o.put("district" , obj.mzb_district_name) + o.put("name" , d['E']) + o.put("type" , "乡镇街道") + o.put("township" , obj.street) + DEV_MAPPING_CONN.save(o) + } + } + }) + + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy new file mode 100644 index 000000000..c987e3b7c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DistrictAliasProcess.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId + +class DistrictAliasProcess { + + static main(args) { + def task = new DistrictAliasProcess(); + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d-> + if(d['H']=="是"){ + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata)) + }else if(d['H']=="否"){ + def aliasDist = d['F'] + def arrs = aliasDist.split("-") + def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district") + if(obj){ + def alias = obj.alias + def mapping_alias = obj.mapping_alias + def udata = new BasicDBObject() + if(alias && alias.contains(d['D'])){ + alias.remove(d['D']) + udata.put("alias", alias) + } + if(mapping_alias && mapping_alias.contains(d['D'])){ + mapping_alias.remove(d['D']) + udata.put("mapping_alias", mapping_alias) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + } + } + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy index 2a638980a..60d12e13b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/GXDdistrictMatch.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.ccbevaluation.PgsqlCommUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsSearchUtils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils @@ -32,7 +33,9 @@ class GXDdistrictMatch { // task.updateDistStatus()//恢复name_cat不正常数据 // task.saveToDB() //保存新增小区 - task.restoreDist()//恢复小区status=0 +// task.restoreDist()//恢复小区status=0 + + task.runMatch() } def addOldMatch(){ @@ -133,6 +136,22 @@ class GXDdistrictMatch { }) } + def runMatch(){ + def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx + def city = "福州" + ExcelUtils.modify(fileName, { d -> + println d['C'] + if(d['B'] && d['A']){ + def region = RegionData.formatRegion(city, d['B']) + if (region) { + d = match(city, region, d) + } else { + println d['A'] + "#" + d['E'] + " 区域无法格式化" + } + } + }) + } + def updateDistStatus(){ def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx" def index = 0 @@ -187,27 +206,32 @@ class GXDdistrictMatch { } def saveToDB(){ - def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" +// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx" + def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx" ExcelUtils.modify(fileName, { d -> - if(d['U'] && d['AG']!='过滤'){ + if(d['A']){ def udata = new BasicDBObject() - udata.put("city", d['AD']) - udata.put("region", d['AE']) - def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") + udata.put("city", d['A']) + udata.put("region", d['B']) + def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "") udata.put("name", name) def alias = [] - if(name!=d['C']){ - alias << d['C'] - } - if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){ - alias << d['AI'] + if(d['G'] && name!=d['G']){ + alias << d['G'] } +// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){ +// alias << d['AI'] +// } if(alias){ udata.put("alias", alias) } - udata.put("address", d['AF']) + def addr = AddressUtil.formatAddress(d['A'], d['D']) + if(addr){ + udata.put("address", addr) + } + udata.put("status", "0") udata.put("name_cat", "0") udata.put("cat", "district") @@ -215,11 +239,11 @@ class GXDdistrictMatch { def estateType2 = "普通住宅" def type = "" - if(d['AH']){ - type = d['AH'] - }else{ - type = d['Y'] - } +// if(d['AH']){ +// type = d['AH'] +// }else{ +// type = d['Y'] +// } if(type){ if(type.contains("别墅")){ estateType2 = "别墅" @@ -228,12 +252,12 @@ class GXDdistrictMatch { } } udata.put("estateType2", estateType2) - if(d['AA']){ - Long lng = d['AA']*10000000000 + if(d['F']){ + Long lng = d['F']*10000000000 udata.put("lng", lng) } - if(d['AB']){ - Long lat = d['AB'] * 10000000000 + if(d['E']){ + Long lat = d['E'] * 10000000000 udata.put("lat", lat) } @@ -265,7 +289,7 @@ class GXDdistrictMatch { if(!d['U']){ d = matchApi(city, region, d, "property", false) if(!d['U']){ - d = matchApi(city, region, d, null, true) + d = matchApi(city, region, d, null, false) } } @@ -285,14 +309,18 @@ class GXDdistrictMatch { } def matchApi(city, region, d, cat, useMap){ - def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap) - if(!jsonObject){ + def jsonObject = null + if(!jsonObject && d['H']){ + jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap) + } + + if(!jsonObject && d['C']){ jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap) if(!jsonObject && d['D']){ jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap) } } - if(jsonObject != null && jsonObject.credit>0) { + if(jsonObject != null) { d['U'] = "接口匹配" d['V'] = jsonObject.get("city") d['W'] = jsonObject.get("district") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy new file mode 100644 index 000000000..8edf46edf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/PoiLocation.groovy @@ -0,0 +1,317 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class PoiLocation { + + def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity() + def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity() + def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges() + def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity() + + static main(args) { + def task = new PoiLocation(); + task.run(); + } + def run(){ + def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"] + for(def city : citys){ + findPois(city) + } + } + + + def findPois(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("run_status", new BasicDBObject('$exists', false)) + + def keys = new BasicDBObject() + keys.put("_id", 1) + keys.put("province", 1) + keys.put("city", 1) + keys.put("district", 1) + keys.put("township", 1) + keys.put("location", 1) + + def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def rets = [] + def index = 0 + for(def d : dbcursor){ + rets << d + if(rets.size()>=10000){ + index++ + def datas = checkData(index, rets) +// updateToEs(datas) + updateToEs2(datas) + updateToMG(datas) + rets = [] + print "." + } + } + return rets; + } + + def checkData(def index, def datas){ + def totalCount = new AtomicLong() + def rets = new CopyOnWriteArrayList() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i= batchSize) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + print "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + + request = new BulkRequest(); + } + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + + println "结束时间 "+new Date() + } + + + def updateToMG(rows){ + def rets = [] + for(def i=0;i + if(k!="id"){ + udata.put(k,v) + } + } + def q = new BasicDBObject() + q.put("_id", m.id) + if(udata){ + MAPPOI_CONN.update(q, new BasicDBObject('$set', udata)) + }else{ + println m.toString()+" 无法修改-无值" + } + + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy new file mode 100644 index 000000000..4d865d728 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TempExportData.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +class TempExportData { + + ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new TempExportData() + task.run() + } + + def run() { + def citys = CommonDatautils.findAllCity() + def rets = [] + rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"] + for (def city : citys) { + def cityRets = getResults(city) + rets.addAll(cityRets) + } + ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据") + } + + def getResults(city) { + def rets = [] + def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"]) + + final CountDownLatch latch = new CountDownLatch(dists.size()) + final AtomicInteger count_ = new AtomicInteger(0) + + for (def i = 0; i < dists.size(); i++) { + def d = dists[i] + fixedThreadPool.execute( + new Runnable() { + void run() { + count_.addAndGet(1) + try { + def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district") + if (objs) { + def ret = [] + ret << d._id.toString() + ret << d.city + ret << d.region + ret << d.name + ret << d.address + + def str = "" + for (def o : objs) { + str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n" + } + ret << str + ret << objs.size() + rets << ret + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + } + ); + } + latch.await() + println(city +" "+rets.size()) + return rets; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy new file mode 100644 index 000000000..4d77b4cdf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/Test.groovy @@ -0,0 +1,105 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.google.common.hash.BloomFilter +import com.google.common.hash.Funnels + +import java.nio.charset.StandardCharsets +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class Test{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) +// 最大重试次数 + private static final int MAX_TRY_COUNT = 5; + + static main(args) { + def task = new Test() + task.run() + } + + def run(){ + def i=1 + while(i<2){ + def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all"); + def rets = process(districts) + println i + " "+districts.size() +" "+rets.size() + i++ + } + + } + + def process(districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + + def rets = new CopyOnWriteArrayList() + for(def i=0;i + def udata = new BasicDBObject() + udata.put("province", key) + udata.put("tjj_province_code", val) + udata.put("mzb_province_code", mzbMaps.get(key)) + udata.put("tjj_source_year",2023) + udata.put("mzb_source_year",2023) + lastRets << udata + } + if(diffKeys){ + diffKeys.each {k-> + def udata = new BasicDBObject() + udata.put("province", k) + if(tjjMaps.get(k)){ + udata.put("tjj_province_code", tjjMaps.get(k)) + udata.put("tjj_source_year",2023) + } + if(mzbMaps.get(k)){ + udata.put("mzb_province_code", mzbMaps.get(k)) + udata.put("mzb_source_year",2023) + } + lastRets << udata + } + } + saveToDB(lastRets, "省份") + } + + + def createCity(){ + def cat = "城市" + def tjjMaps = findAllProcincesFromTJJ(cat) + def mzbMaps = findAllProcincesFromMZB(cat) + def tjjKeys = tjjMaps.keySet() + def mzbKeys = mzbMaps.keySet() + def sameKeys = tjjKeys.intersect(mzbKeys) + def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys) + def lastRets = [] + sameKeys.each {k-> + def v = tjjMaps.get(k) + lastRets << getAdministrativeCodeInfo(k, v, mzbMaps) + } + + if(diffKeys){ + diffKeys.each {k-> + def v = tjjMaps.get(k) + lastRets < + def city = RegionData.formatCity(o.city) + if(city){ + def cityObj = CommonDatautils.findAlCityDetail(city) + o.put("alias", cityObj.alias) + }else{ + println o.province+" "+o.city +" 房价网无此城市" + } + } + //添加城市围栏 + def cityPolylineMap = cityPolyline() + //添加pid + def provPidMap = createPidData("省份") + lastRets.each{o-> + def shapeData = cityPolylineMap.get(o.city) + if(shapeData){ + o.shape = shapeData.shape + o.location = shapeData.location + } + def provId = provPidMap.get(o.province) + if(provId){ + o.pid = provId + } + } + saveToDB(lastRets, cat) + } + + + + def createRegion(){ + def cat = "区域" + def tjjMaps = findAllProcincesFromTJJ(cat) + def mzbMaps = findAllProcincesFromMZB(cat) + def tjjKeys = tjjMaps.keySet() + def mzbKeys = mzbMaps.keySet() + def sameKeys = tjjKeys.intersect(mzbKeys) + def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys) + def lastRets = [] + sameKeys.each {k-> + def v = tjjMaps.get(k) + def ud = getAdministrativeCodeInfo(k, v, mzbMaps) +// ud.source = "tjj|mzb" + lastRets << ud + } + + if(diffKeys){ + diffKeys.each {k-> + def v = tjjMaps.get(k) + def ud = getAdministrativeCodeInfo(k, v, mzbMaps) +// if(v){ +// ud.source = "tjj" +// }else{ +// ud.source = "mzb" +// } + lastRets << ud + } + } + //添加别名 + def regionMap = CommonDatautils.getAllCityFullRegions() + lastRets.each {o-> + if(o.district.contains("开发区")){ + println "ssss" + } + def city = RegionData.formatCity(o.city) + def district = RegionData.formatRegion(city, o.district) + if(city && district){ + def cMap = regionMap.get(city) + if(cMap){ + def regionInfoMap = cMap.get(district) + if(regionInfoMap){ + def regions = regionInfoMap.get("pattern") + o.put("alias", regions) + o.put("mzb_district_name", regionInfoMap.full_region) + if(!o.mzb_district_code && regionInfoMap.full_region){ + def mzb_val_str = mzbMaps.get(o.province+"|@|"+o.city+"|@|"+regionInfoMap.full_region) + if(mzb_val_str){ + def mzb_vals = mzb_val_str.split("\\|@\\|") + o.put("mzb_province_code", mzb_vals[0]) + o.put("mzb_city_code", mzb_vals[1]) + o.put("mzb_district_code", mzb_vals[2]) + } + } + } + } + + }else{ + println o.province+" "+o.city+" "+ o.district +" 房价网无此城市" + } + } + + //添加pid + def provPidMap = createPidData("城市") + + //添加区域围栏 + def regionPolylineMap = regionPolyline() + + lastRets.each{o-> + def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district) + if(shapeData){ + o.shape = shapeData.shape + o.location = shapeData.location + } + def provId = provPidMap.get(o.city) + if(provId){ + o.pid = provId + } + } + //是否为县级市 + saveToDB(lastRets, cat) + } + + def createTown(){ + def cat = "乡镇街道" + def tjjMaps = findAllProcincesFromTJJ(cat) + def lastRets = [] + def govMap = findAllGovTown() + tjjMaps.each {k,v-> + def ud = getAdministrativeCodeInfo(k, v, null) + if(govMap && govMap.containsKey(k)){ + ud.put("has_gov", govMap.get(k)) + } + lastRets << ud + } + + //添加区域围栏 + def regionPolylineMap = townPolyline() + //添加pid + def provPidMap = createPidData("区域") + + lastRets.each{o-> + def shapeData = getStreetPolyline(regionPolylineMap, o, o.district) + if(!shapeData){ + def newRegion = o.district + def standardRegion = RegionData.formatRegion(o.city,o.district) + if(standardRegion && standardRegion!=newRegion){ + newRegion = standardRegion + shapeData = getStreetPolyline(regionPolylineMap, o, newRegion) + } + } + if(shapeData){ + o.shape = shapeData.shape + if(shapeData.location){ + o.location = shapeData.location + } + } + def provId = provPidMap.get(o.city+"@"+o.district) + if(provId){ + o.pid = provId + } + } + //是否为县级市 + saveToDB(lastRets, cat) + } + + def getStreetPolyline(regionPolylineMap, o, district){ + def shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+o.street) + if(!shapeData){ + def streetKey = o.street + if(streetKey.toString().endsWith("镇")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"街道" + }else if(streetKey.toString().endsWith("乡")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇" + }else if(streetKey.toString().endsWith("街道")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"镇" + }else if(streetKey.toString().endsWith("地区")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("地区"))+"镇" + } + shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey) + if(!shapeData){ + streetKey = o.street + if(streetKey.toString().endsWith("镇")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"乡" + }else if(streetKey.toString().endsWith("乡")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇" + }else if(streetKey.toString().endsWith("街道")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"社区" + } + shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey) + } + if(!shapeData){ + streetKey = o.street + if(streetKey.toString().endsWith("镇")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"地区" + }else if(streetKey.toString().endsWith("街道")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"乡" + } + shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey) + } + if(!shapeData){ + if(streetKey.toString().endsWith("镇")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇")) + }else if(streetKey.toString().endsWith("乡")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡")) + }else if(streetKey.toString().endsWith("街道")){ + streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道")) + } + shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey) + } + } + return shapeData + } + + + def getAdministrativeCodeInfo(key, val, mzbMaps){ + def udata = new BasicDBObject() + def keys = key.split("\\|@\\|") + def vals = null + if(val){ + vals = val.split("\\|@\\|") + } + + def mzb_vals = null + if(mzbMaps && mzbMaps.containsKey(key)){ + def mzb_val_str = mzbMaps.get(key) + mzb_vals = mzb_val_str.split("\\|@\\|") + } + if(keys.size()>=1){ + udata.put("province", keys[0]) + if(vals && vals.size()>=1){ + udata.put("tjj_province_code", vals[0]) + udata.put("tjj_source_year",2023) + } + if(mzb_vals && mzb_vals.size()>=1){ + udata.put("mzb_province_code", mzb_vals[0]) + udata.put("mzb_source_year",2023) + } + } + + + if(keys.size()>=2){ + udata.put("city", keys[1]) + if(vals && vals.size()>=2){ + udata.put("tjj_city_code", vals[1]) + } + if(mzb_vals && mzb_vals.size()>=2){ + udata.put("mzb_city_code", mzb_vals[1]) + } + } + + + if(keys.size()>=3){ + udata.put("district", keys[2]) + if(vals && vals.size()>=3) { + udata.put("tjj_district_code", vals[2]) + } + if(mzb_vals && mzb_vals.size()>=3){ + udata.put("mzb_district_code", mzb_vals[2]) + } + } + + if(keys.size()>=4){ + udata.put("street", keys[3]) + if(vals && vals.size()>=4) { + udata.put("tjj_street_code", vals[3]) + } + if(mzb_vals && mzb_vals.size()>=4){ + udata.put("mzb_street_code", mzb_vals[3]) + } + } + + return udata + } + + + def updateTownAlias(){ + ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇别名问题审核交付_0418.xlsx",{ row-> + if(row['G'] && row['G']=="不处理"){ + def q = new BasicDBObject() + q.put("city",row['C']) + q.put("district",row['D']) + q.put("street",row['E']) + def d = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + if(d){ + def alias = [] + alias << row['F'] + def udata = new BasicDBObject() + udata.put("alias", alias) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + }else{ + println row['C']+" "+row['D']+" "+row["E"] + } + } + }) + } + + + def synchTown(){ + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000) + def datas = [] + for(def d : dbcursor){ + datas << d + if (datas.size() >= 2000) { + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); + println datas.size() + datas.clear(); + } + } + + if (datas.size() >= 0) { + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); + datas.clear(); + } + } + + + def saveToDB(lastRets, type){ + def dataList = [] + for(def ret : lastRets){ + ret.put("c_date", new Date()) + ret.put("m_date", new Date()) + ret.put("type", type) + dataList << ret + + if (dataList.size() >= 5000) { + ADMINISTRATIVE_MAPPING_COLL.insert(dataList); + dataList.clear(); + } + } + + if (dataList.size() > 0) { + ADMINISTRATIVE_MAPPING_COLL.insert(dataList); + dataList.clear(); + } + } + + + def createPidData(def type){ + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type)).batchSize(5000) + def map = [:] + def pids = [] + def step = 2 + if(type == "城市"){ + step = 4 + }else if(type == "区域"){ + step = 6 + } + for(def d : dbcursor){ + def pid = null + def code = null + if(d.mzb_province_code){ + if(type == "省份") { + code = d.mzb_province_code.toString() + }else if(type == "城市"){ + code = d.mzb_city_code.toString() + }else if(type == "区域"){ + code = d.mzb_district_code.toString() + } + }else if(d.tjj_province_code){ + if(type == "省份") { + code = d.tjj_province_code.toString() + }else if(type == "城市"){ + code = d.tjj_city_code.toString() + }else if(type == "区域"){ + code = d.tjj_district_code.toString() + } + } + + pid = code.substring(0,step) + if(pids.contains(d.city+"-"+pid)){ + pid = code + } + if(!pids.contains(d.city+"-"+pid)){ + pids << d.city+"-"+pid + } + if(type == "省份"){ + map.put(d.province, pid) + }else if(type == "城市"){ + map.put(d.city, pid) + }else if(type == "区域"){ + map.put(d.city+"@"+d.district, pid) + } + } + return map; + } + + def findAllGovTown(){ + def q = new BasicDBObject(); + q.put("has_gov", new BasicDBObject('$ne', '')) + def keys = new BasicDBObject() + keys.put("province_name",1) + keys.put("city_name",1) + keys.put("area_name", 1) + keys.put("street_name", 1) + keys.put("has_gov", 1) + def dbcursor = COMMUNITY_CONN.find(q,keys).batchSize(5000) + def map = [:] + for(def d : dbcursor){ + def key = d.province_name+"|@|"+d.city_name+"|@|"+d.area_name+"|@|"+d.street_name + if(!map.containsKey(key)){ + map.put(key, d.has_gov) + } + } + return map; + } + + + def findAllProcincesFromTJJ(cat){ + // 定义聚合管道,按 city 字段分组,并计算每个城市的数量 + DBObject groupFields = new BasicDBObject(); + def match = null + if(cat=="省"){ + groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code')) + }else if(cat=="城市"){ + groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code')) + }else if(cat=="区域"){ + groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code')) + }else if(cat=="乡镇街道"){ + groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("street_name", '$street_name').append("street_code", '$street_code')) + } + groupFields.put("count", new BasicDBObject('$sum', 1)); + DBObject group = new BasicDBObject('$group', groupFields); + + // 执行聚合查询 + AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group)); + + def map = [:] + // 遍历结果 + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + if(obj.get("city_name").toString()!="省直辖县级行政区划"){ + def key = obj.get("province_name") + def value = obj.get("province_code") + if(cat=="城市"){ + key = key +"|@|"+ obj.get("city_name") + value = value +"|@|"+ obj.get("city_code") + }else if(cat=="区域"){ + key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name") + value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code") + }else if(cat=="乡镇街道"){ + key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("street_name") + value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("street_code") + } + int count = (Integer) result.get("count"); + map.put(key, value) + } + } + return map; + } + + + def findAllProcincesFromMZB(cat){ + + BasicDBObject match = null; + // 定义聚合管道,按 city 字段分组,并计算每个城市的数量 + DBObject groupFields = new BasicDBObject(); + + if(cat == "省"){ + match = new BasicDBObject('$match', new BasicDBObject("level", "省份")); + groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code')) + }else if(cat == "城市"){ + match = new BasicDBObject('$match', new BasicDBObject("level", "城市")); + groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code')) + }else if(cat == "区域"){ + match = new BasicDBObject('$match', new BasicDBObject("level", "区域")); + groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code').append("district",'$district').append("mca_district_code",'$mca_district_code')) + } + + + groupFields.put("count", new BasicDBObject('$sum', 1)); + DBObject group = new BasicDBObject('$group', groupFields); + + // 执行聚合查询 + AggregationOutput output = ADMINISTRATIVE_DIVISION_CONN.aggregate(Arrays.asList(match, group)); + def map = [:] + // 遍历结果 + for (DBObject result : output.results()) { + def obj = (DBObject ) result.get("_id"); + if(obj.get("city").toString()!="省直辖县级行政区划"){ + def key = obj.get("province") + def value = obj.get("mca_province_code") + if(cat == "城市"){ + key = key +"|@|"+obj.get("city") + value = value + "|@|" + obj.get("mca_city_code") + }else if(cat == "区域"){ + key = key +"|@|"+obj.get("city")+"|@|"+obj.get("district") + value = value + "|@|" + obj.get("mca_city_code")+ "|@|" + obj.get("mca_district_code") + } + int count = (Integer) result.get("count"); + map.put(key, value) + } + } + return map; + } + + def cityPolyline(){ + def dbcursor = CITYPOLYLINE_COLL.find(new BasicDBObject()) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.name+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + map.put(d.name, udata) + } + return map; + } + + def regionPolyline(){ + def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district")) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + + def fullCity = fullCityMap.get(d.city) + map.put(fullCity.city+"|@|"+d.region, udata) + } + return map; + } + + def townPolyline(){ + def dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject()).batchSize(5000) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+d.town+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + map.put(d.city+"|@|"+d.fjw_region+"|@|"+d.town, udata) + } + return map; + } + + def addCityFlag(){ +// ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row-> +// if(row['A']){ +// def q = new BasicDBObject("type", "城市") +// q.put("city", row['A']) +// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) +// if(obj){ +// def udata = new BasicDBObject() +// udata.put("prefecture_level_city", "是") +// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) +// }else{ +// println row['A']+" 没找到该城市" +// } +// } +// }); + + ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row-> + if(row['A']){ + def q = new BasicDBObject("type", "区域") + q.put("district", row['A']) + def count = ADMINISTRATIVE_MAPPING_COLL.count(q) + if(count && count == 1){ + def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q) + def udata = new BasicDBObject() + udata.put("county_level_city", "是") + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + }else{ + println row['A']+" 没找到该城市" + } + } + },1); + } + + def addPinyin(){ + def q = new BasicDBObject("type", "乡镇街道") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000) + for(def d : dbcursor){ + if(d.street){ + def street_pinyin = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.street, " ") + def udata = new BasicDBObject() + udata.put("street_pinyin", street_pinyin) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + } + } + + } + + def addTjjDevFlag(){ + def q = new BasicDBObject("type", "区域") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000) + for(def d : dbcursor){ + if(d.district && d.mzb_district_name && d.district==d.mzb_district_name){ + def udata = new BasicDBObject() + udata.put("district_diff", 1) //民政部,统计局 名称相同 district_diff打1 + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata)) + } + } + } + + def addStreetFlag(){ + ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道-结果.xlsx",{ row-> + if(row['I']){ + if(row['I']=="标0"){ + def udata = new BasicDBObject() + udata.put("street_output", 0) //0:不输出 1:输出 + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + }else if(row['I']=="标1"){ + def udata = new BasicDBObject() + udata.put("street_output", 1) //0:不输出 1:输出 + if(row['G']){ + udata.put("street", row['G']) //街道中去除区域 + } + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + } + } + }); + } + + def addDistrictDiffFlag(){ + ExcelUtils.modify("D:\\数据\\数据建设\\区域-结果.xlsx",{ row-> + def flag = row['F'].toString().toDouble().toInteger() + def udata = new BasicDBObject() + udata.put("district_diff", flag) + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata)) + }); + } + + def updateCommunityDiffStreet(){ + def q = new BasicDBObject("type", "乡镇街道") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000) + for(def d : dbcursor){ + def query = new BasicDBObject() + query.put("city_name", d.city) + query.put("area_name", d.district) + query.put("street_name", d.street) + def obj = COMMUNITY_CONN.findOne(query) + if(!obj){ + query.put("street_name", d.district+d.street) + obj = COMMUNITY_CONN.findOne(query) + if(obj){ + def newStreet = obj.street_name.replace(d.district, "") + def udata = new BasicDBObject() + udata.put("street_name", newStreet) + COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true) + println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}" + }else{ + println d.city +" "+d.district+" "+d.street + } + + } + } + + } + + def updateTownPoline(){ + def city = "唐山市" + def name = "高新技术开发区街道" + def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区") + println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q) + def d = TOWNPOLYLINE_COLL.findOne(q) + if(d){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+d.town+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + + + def updateQuery = new BasicDBObject("city", city).append("street",name) + ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + } + + def checkTown(){ + def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道") + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q) + for(def ad : dbcursor){ + def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street) + def count = TOWNPOLYLINE_COLL.count(polyline_q) + if(count==1){ + println ad.city+" "+ ad.street+" "+count + + def d = TOWNPOLYLINE_COLL.findOne(polyline_q) + if(d){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + udata.put("polyline_source", "区域+街道匹配") + + def updateQuery = new BasicDBObject("_id", ad._id) + ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata)) + } + } + } + } + + + def updateTownshipByRegion(){ + def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域") + def keys = new BasicDBObject() + keys.put("mzb_district_name",1) + keys.put("mzb_district_code",1) + + keys.put("city",1) + keys.put("district",1) + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys) + for(def d : dbcursor){ + def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city) + def udata = new BasicDBObject() + udata.put("mzb_district_name",d.mzb_district_name) + udata.put("mzb_district_code",d.mzb_district_code) + ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy new file mode 100644 index 000000000..f6e830017 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -0,0 +1,324 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.DBCollection + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class TownshipStrandUpdate { + + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + def fullCityMap = null + def regionMap = null + def commMap = null + def polylineMap = null + def newAdvMappingMap = null + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + static main(args) { + def task = new TownshipStrandUpdate() + task.process() + + } + + + void process() { + fullCityMap = CommonDatautils.queryFullNameByCity() + regionMap = CommonDatautils.queryDistNameByCity() + commMap = CommonDatautils.queryCommunityByCity() + polylineMap = CommonDatautils.findAllTownRanges() + newAdvMappingMap = CommonDatautils.queryNewTownByCity() +// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() + citys = citys.reverse() + for(def city : citys){ + if(["中山"].contains(city)){ + continue + } + def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx" + File file=new File(fileName); + if(!file.exists()){ + def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"]) + dists = checkData(dists) + def datas = createDatas(dists) + ExcelUtils.write(fileName, datas, "${city}") + } + } + } + + def checkData(def dists){ + def totalCount = new AtomicLong() + def rets = new CopyOnWriteArrayList() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i maxScore){ + maxScore = score + town = townObj + } + } + if(town){ + town.score = maxScore + } + return town + } + + + def getSimilarTown(regions, fullCity, townstr){ + def town = null + for(def r : regions){ + def townList = commMap.get(fullCity.city+"#"+r) + + for(def townObj : townList){ + if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){ + town = townObj + }else if(townObj.alias && townObj.alias.contains(townstr)){ + town = townObj + }else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){ + def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","") + if(townstr.contains(newTown)){ + town = townObj + } + } + if(town){ + break + } + } + } + return town + } + + + def getFullRegion(fullCity, d){ + def regions = regionMap.get(fullCity.city) + def names = [] + if(regions){ + for(def r : regions){ + if(r.mzb_district_name.contains(d.region)){ + names << r.district + } + } + } + return names; + } + + + def createDatas(dists){ + def rets = [] + rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"] + for(def d : dists){ + rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score] + } + return rets; + } + + def getTownshipInfo(def glocation){ + def township = null + def district = null + def city = null + def format_addr = null + def jsonObject = null + def stop = false + def count = 0 + while(!stop){ + if(glocation){ + jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + district = addressComponent.district + if(addressComponent.city.size()>0){ + city = addressComponent.city + }else{ + if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){ + city = addressComponent.district + }else{ + city = addressComponent.province + } + + } + + format_addr = jsonObject.regeocode.formatted_address + + } + } + } + if(district || count>10){ + stop = true + }else{ + count ++ + Thread.sleep(3000) + print "." + } + } + + def map = [:] + map.city = city + map.district = district + map.township = township + map.format_addr = format_addr + return map; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy index 7c9472e42..7d34463a2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TradeDatasDistBatch.groovy @@ -1,13 +1,46 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject import com.mongodb.DBCollection class TradeDatasDistBatch { public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com") static main(args) { -// def + def task = new TradeDatasDistBatch() + task.run() + } + + def run(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d-> + + def udata = new BasicDBObject() + udata.put("city", d['A']) + udata.put("region", d['B']) + udata.put("name", d['C']) + udata.put("alias", d['D']) + alias_fang_COLL.save(udata) +// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") +// def flag = null +// if(!dist){ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district") +// if(rets && rets.size()==1){ +// flag = "别名匹配" +// dist = rets[0] +// } +// }else{ +// flag = "名称匹配" +// } +// if(flag){ +// d['F'] = dist.city +// d['G'] = dist.region +// d['H'] = dist.name +// } + }) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 260ef5cb7..0bdd0555f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -135,6 +135,26 @@ class FangjiaClientUtils { return jsonObj; } + public static def format(def city, def address){ + def jsonObj = null + def content = null + try { + if(address){ + address = address.replace("{","") + def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8") + if(city){ + url = url +"&city="+URLEncoder.encode(city,"UTF-8") + } + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static def arroundPrice(city,address){ def jsonObj = null def content = null @@ -228,9 +248,9 @@ class FangjiaClientUtils { def url = null; try { address = address.replace("{","").replace("\\","") - url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" +// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" // url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" -// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" + url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" if(city){ url = url + "&city="+URLEncoder.encode(city,"UTF-8") } @@ -334,13 +354,13 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true" + def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } content = new HttpRequest().newGet(url, null, "utf-8") jsonObj = JSONObject.fromObject(content) - println jsonObj +// println jsonObj } catch (Exception e) { println address +" "+content e.printStackTrace(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy new file mode 100644 index 000000000..16e99a043 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommbuildtypeBatch.groovy @@ -0,0 +1,138 @@ +package com.fangjia.script.gj.fjwPriceBaseData + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 房价网内部模型估值项目 + * 小区建筑类型数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataCommbuildtypeBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + + //235为房价网内部计算均价的库 + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwBigdataCommbuildtypeBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def rets = [] + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + + for(def city : citys){ + def distIds = FjwPgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city)) + + def rows = createDatas(cityCodeDatas.get(city), distIds) + rets.addAll(rows) + println city +" "+rows.size() + } + + delPgData() + saveToPgSql(rets) + } + + def createDatas(cityCode, dists){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + for(def r : rows) { + index++ + println index + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14]) + } + } + println "end: "+new Date() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy new file mode 100644 index 000000000..4a0358411 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataCommunityBatch.groovy @@ -0,0 +1,370 @@ +package com.fangjia.script.gj.fjwPriceBaseData + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 房价网内部估值项目使用 + * 小区数据同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataCommunityBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) + + //235为房价网内部计算均价的库 + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwBigdataCommunityBatch(), args); + } + + @Override + public void process(Object args) { + + def citys = CcbCommonUtils.getRunningCitys(args) + def rows = [] + + def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", + //"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司", + //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"] + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + def today = new Date().getDate() + + println "starting 。。。。。。。。。" + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all") + def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 + + def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) + rets = addDeleteDist(dists, ccbMaps, rets) + rows.addAll(rets) + println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size() + } + + if(today > 10){ + delPgData() + saveToPgSql(rows) + } + } + + def addDeleteDist(dists, ccbMaps, rets){ + def fangjiaDists = [] + for(def d : dists){ + fangjiaDists << d._id.toString() + } + println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size() + def ccbDists = ccbMaps.keySet() + for(def ccbId : ccbDists){ + if(!fangjiaDists.contains(ccbId)){ + def datas = ccbMaps.get(ccbId) + datas[datas.size()-1] = 1 + rets << datas + } + } + println "添加删除状态小区:"+rets.size() + return rets + } + + def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){ + //def rets = Collections.synchronizedList(new ArrayList()); + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + for(def i=0;i + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.gujiabdsocpoldtotarea + data << r.gujiabds_bld_tot_area + data << r.gujia_bds_grnrate + data << r.gujia_bds_vol_rate + data << r.gujia_bds_park_num + data << r.gujia_bds_bld_yr_rc + data << r.gujia_bds_hs_use_val + data << r.totunits + data << r.gujia_bds_data_tm_val + data << r.gujiabds_udt_flag_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_bds_st + map.put(r.gujia_comm_id, data) + }); + return map + } + + + def delPgData(){ + sqlInstance.execute("delete from base.bigdata_community") + println "base.bigdata_community 数据已删除" + } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.bigdata_community values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + sqlInstance.withBatch(50000, sql){ ps-> + for(def r : rows) { + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + def saveToPgSql2(rows){ + def newRows = [] + for(def r : rows) { + def newR = [] + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ +// r[i] = null + newR << null + }else{ + newR << r[i] + } + } + newR << null + newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + newR << null + newR << '2999-12-31' + newR << null + newR << null + newR << null + + newRows << newR + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + def index = 0 + sqlInstance.withBatch(50000, sql){ ps-> + for(def r : newRows) { + index++ + try{ + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + index + }catch(Exception e){ + e.printStackTrace() + } + } + } + + println "end: "+new Date() + } + + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("\\|@\\|") + rows << row + } + return rows; + } + + def updateVersion(def time){ + def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"] + for(def tableName : tables){ + def sql = "update base."+tableName+" set p9_start_date='"+time+"'" + sqlInstance.executeUpdate(sql) + } + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy new file mode 100644 index 000000000..21bb3652d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataDealBatch.groovy @@ -0,0 +1,786 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DataUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import groovy.sql.Sql + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern + +/** + * 建行模型估值项目 + * 成交数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataDealBatch extends BaseBatch{ + + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"] + + public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; + public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwBigdataDealBatch(), args); + } + + @Override + public void process(Object args) { + + def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args) + +// println "curDate:"+curDate +// def allCitys = CommonDatautils.findAllCity() +// for(def city : allCitys){ +// addDealPriceMatch(city); +// } + + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city)) + println "历史id加载量:"+dealIds.size() + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds) + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds) + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + } + + saveToPgSql(rows) + +// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat") +// saveToPgSql2(datas) + } + + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("\\|@\\|") + rows << row + } + return rows; + } + + + def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){ + def result = Collections.synchronizedList(new ArrayList()); + dealIds = Collections.synchronizedList(dealIds); + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + +// def lianjiaPriceMap = findThirdPartyPrice("上海") +// def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') +// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) + + def ccbPriceMap = null + if(city == "上海"){ + ccbPriceMap = getCommunityPrices("上海") + } + + for(def i=0;i 3000) { + price = db.lianjia_esf_price + } + if (!price && db.beike_esf_price && db.beike_esf_price > 3000) { + price = db.beike_esf_price + } + if(price){ + map.put(db.region+"##"+db.name, price) + } + + } + return map + } + + def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def ccbPriceMap){ + def ret = [] + def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district") + if(!obj){ + return ret; + } + def price = ccbPriceMap.get(obj._id.toString()) + if(price){ + def mod = java.lang.Math.abs((trade.price-price)*100/price) + if(mod<=20){ + ret << cityCodeDatas.get(trade.city) + ret << trade._id.toString() + ret << trade.city + def data = distCodeMap.get(trade.fjRegion) + if(data){ + ret << data.district_code + ret << data.district + }else{ + ret << "" + ret << "" + } + def street = "" + def streetNumber = "" + if(trade.address){ + if(trade.address.toString().contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + streetNumber = number + } + } + if(!street){ + def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) + if(jsonObj && jsonObj.searchAddress){ + if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + street = trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + + } + } + } + + if(!street){ + if(obj.address){ + if(obj.address.toString().contains("弄")){ + String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄")) + if(number){ + street = obj.address.toString().substring(0, obj.address.toString().indexOf(number)) + streetNumber = number + } + }else{ + def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name) + if(jsonObj && jsonObj.searchAddress){ + street = jsonObj.searchAddress.street + streetNumber = jsonObj.searchAddress.streetNumber + } + } + } + } + + ret << street + ret << streetNumber + + + if(obj){ + ret << obj._id.toString() + ret << trade.fjName + } + + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属 + ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life) + ret << ""//期数 + String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室")) + if(roomNum && roomNum.contains("0")){ + ret << roomNum.substring(0, roomNum.lastIndexOf("0")) + }else{ + ret << trade.floor + } + + ret << "" + ret << trade.height + def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType + ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure) + ret << trade.area + ret << "" + ret << getHouseRoom(buildType, trade.room, trade.hall) + ret << trade.room + ret << trade.hall + ret << "" + ret << trade.toilet + ret << CcbCodeUtils.parseHouseUse(obj.estate_type2) + ret << "" + ret << CcbCodeUtils.getDirectionCode(trade.direction) + ret << CcbCodeUtils.getFitmentCode(trade.fitment) + ret << "" + ret << trade.price + ret << trade.totalPrice * 10000 + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd") + ret << "02" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "99" + + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << trade.totalPrice * 10000 + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + } + } + + return ret; + } + + + def converData(def trade, def cityCodeDatas, def distCodeMap){ + def ret = [] + def data = distCodeMap.get(trade.region) + if(!data){ + println trade.city +" "+trade.region+" 没有找到对应的区域" + return ret + } + ret << cityCodeDatas.get(trade.city) + ret << trade._id.toString() + ret << CcbCommonUtils.getCcbCity(trade.city) + if(data){ + ret << data.district_code + ret << data.district + }else{ + ret << "" + ret << "" + } + + if(trade.address){ + if(trade.address.toString().contains("弄")){ + String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄")) + if(number){ + ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number)) + ret << number + }else{ + ret << "" + ret << "" + } + }else{ + def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name) + if(jsonObj && jsonObj.searchAddress){ + ret << jsonObj.searchAddress.street + ret << jsonObj.searchAddress.streetNumber + }else{ + ret << "" + ret << "" + } + } + }else{ + ret << "" + ret << "" + } + ret << trade.dist_id.toString() + ret << trade.master_name + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属 + ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life) + ret << ""//期数 + ret << trade.floor + ret << "" + ret << trade.height +// ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型 + def arrs = CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型 + def buildType = '' + if(arrs){ + buildType = arrs[0] + } + ret << buildType + ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure) + ret << trade.area + ret << "" + ret << getHouseRoom(buildType, trade.room, trade.hall) + ret << trade.room + ret << trade.hall + ret << "" + ret << trade.toilet + ret << CcbCodeUtils.parseHouseUse(trade.estate_type2) + ret << "" + ret << CcbCodeUtils.getDirectionCode(trade.direction) + ret << CcbCodeUtils.getFitmentCode(trade.fitment) + ret << "" + ret << trade.avg_price + ret << trade.total_price + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd") + ret << "02" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + ret << "" + def root = sourceMap.get(trade.source) + if(root){ + if(CcbCommonUtils.rootMap.get(root)){ + ret << CcbCommonUtils.rootMap.get(root) + }else{ + ret << "99" + } + }else{ + ret << "" + } + + ret << "" + ret << "" + ret << "" + ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + ret << trade.total_price + ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd") + return ret; + } + + def getHouseRoom(buildtype, def roomcnt, def livingroomcnt){ + def housetype = null + if(buildtype && ["0112","0110","0109"].contains(buildtype)){ + housetype = "07" + }else if(buildtype && ["0107","0106","0105"].contains(buildtype)){ + housetype = "16" + }else{ + if(roomcnt && roomcnt==4){ + if(livingroomcnt>1){ + housetype = "14" + }else{ + housetype = "11" + } + }else if(roomcnt && roomcnt==3){ + if(livingroomcnt>1){ + housetype = "13" + }else{ + housetype = "10" + } + }else if(roomcnt && roomcnt==2){ + if(livingroomcnt>1){ + housetype = "12" + }else{ + housetype = "09" + } + }else if(roomcnt && roomcnt==1){ + housetype = "08" + }else{ + housetype="99" + } + } + return housetype; + } + + def getCityData(def city, def statMap, def ppeDistsMap){ + + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all") + println city +" 小区量:"+dists.size() + def tradesList = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + +// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] + +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd")) + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) + + + for(def i=0;i=startPrice && obj.price<=endPrice){ +// rets << obj; +// }else{ +// index++ +// println index + " "+obj +// } +// } + } + } + return rets; + } + + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", new BasicDBObject('$gte', weekend)) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1)) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + def price = map.get(d.region+"##"+d.district_name) + if(!price && d.value_type==2){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } + } + return map + } + + def findHistoryDealIds(def city){ + def sql = "SELECT gujia_bds_mdl_id from base.bigdata_deal where urbn_nm='${city}'" + def ids = [] + try{ + pgsqlInstance.eachRow(sql,{ + ids << it.gujia_bds_mdl_id + }); + }catch(Exception e){ + + } + + return ids + } + + + + def addDealPriceMatch(city){ +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd") + def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) +// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) + def trades = CcbCommonUtils.findTradesByMonth(city, startDate) + + final CountDownLatch latch = new CountDownLatch(trades.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + + for(def i=0;i + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], + r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], + r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) + } + } + + println "end: "+new Date() + } + + + def saveToPgSql2(rows){ + def newRows = [] + for(def r : rows) { + def newR = [] + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ +// r[i] = null + newR << null + }else{ + newR << r[i] + } + } + newR << null + newR << "2023-09-28" + newR << null + newR << '2999-12-31' + newR << null + newR << null + newR << null + + newRows << newR + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : newRows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], + r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], + r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) + } + } + + println "end: "+new Date() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy new file mode 100644 index 000000000..50f070a3a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwBigdataForsaleBatch.groovy @@ -0,0 +1,968 @@ +package com.fangjia.script.gj.fjwPriceBaseData + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.google.common.hash.* +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.sql.Sql +import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.nio.charset.StandardCharsets +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + + +/** + * 建行模型估值项目 + * 出售房源数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwBigdataForsaleBatch extends BaseBatch{ + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"] + private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>() + static{ + FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"]) + FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"]) + FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"]) + FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"]) + } + + public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + + public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200) + + +// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + + def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); + def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"); + + // 最大重试次数 + private static final int MAX_TRY_COUNT = 5; + + static main(args) { + runBatch(new FjwBigdataForsaleBatch(), args); + } + + @Override + public void process(Object args) { + //目前查询库中是否已存在,用的是Oracle + def index = 0 + def citys = CcbCommonUtils.getRunningCitys(args) + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def startDate = lastMonth+"18" + def endDate = curMonth+"19" //正式运行时curMonth要改 + println "房源时间:"+startDate+" "+endDate + + def houseIdsfilter = findAllHouseIdsByCity() + + for(def city : citys){ + index++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all"); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter); + } + } + + + def statCity(def citys){ + def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1) + def houseCitys = [] + for(def d : datas){ + houseCitys << d['C'] + } + for(def city : citys){ + if(!houseCitys.contains(city)){ + def scity = CcbCommonUtils.specialCityMap.get(city) + if(!scity){ + println city + } + } + } + } + + def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){ + + def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter) + println city + "(原)房源总量:"+retMap.size() + + def allDatas = retMap.values() + saveToPgSql(allDatas) + } + + def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println city +" 小区量:"+districts.size() + + + BloomFilter curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); + def retMap = new ConcurrentHashMap() + for(def i=0;i0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure + if(CcbCommonUtils.rootMap.get(seawater.root)){ + rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(seawater.p_date<=20200101){ + //seawater.p_date.toString().replace("2019","2020")// + rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) + } + + rets << rowMap + +// if(houseIds.contains(seawater.id)){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + +// def count = isExistFromHouse(seawater.id) +// if(count){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + + } + } + + println name +" "+houses.size()+" last:"+rets.size() + return rets; + } + + + def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ + def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") + def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + + QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); + + //用以下方法会查出发布时间很久以前的 +// QueryBuilder dateQueryBuilder = QueryBuilders.boolQuery() +// dateQueryBuilder.should(date_range) +// dateQueryBuilder.should(date_range_mdate) +// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) +// .must(area_range).must(dateQueryBuilder) + + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com")) + + + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); + + if(houses){ + + def ids = new HashSet() + for(def h : rets1){ + ids << h.houseId + } + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure + if(CcbCommonUtils.rootMap.get(seawater.root)){ + rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment + + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(seawater.p_date<=20200101){ + //seawater.p_date.toString().replace("2019","2020")// + rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) + } + + rets << rowMap + +// if(houseIds.contains(seawater.id)){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + +// def count = isExistFromHouse(seawater.id) +// if(count){ +// println seawater.id +" 已存在" +// }else{ +// rets << rowMap +// } + } + } + + println name +" "+houses.size()+" last:"+rets.size() + return rets; + } + + + + def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) { + int retryCount = 0; + boolean success = false; + def rets = [] + + while (retryCount < MAX_TRY_COUNT && !success) { + try { + rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) + // 如果操作成功,设置success为true + success = true; + } catch (Exception e) { + retryCount++; + println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT + // 这里可以选择在重试之前等待一段时间 + Thread.sleep(3000); // 例如等待1秒 + } + } + + if (!success) { + println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name + } + return rets; + } + + + /** + * 是否是别墅 + * @return + */ + public boolean isVilla(area, name, height, room) { + if (area == null) { + return false; + } + if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { + return true; + } + + boolean ret = false; + if (height != null && room != null) { + if (height <= 3) { + if (area > 200 && room >= 5) { + ret = true; + } + } + } else if (height == null && room != null) { + if (area > 360 && room >= 6) { + ret = true; + } + } else if (height == null && room == null) { + if (area > 500) { + ret = true; + } + } + + return ret; + } + + def getPublicDate(def pDate){ + + if(pDate>=20181001 && pDate<20200201){ + return DateUtils.getRandomDate("2018-10-01", "2020-02-01") + } + def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss") + return pdate; + } + + static def parseFloor(def floorDesc){ + if(floorDesc){ + floorDesc = floorDesc.replaceAll("[\\s ]+", "") + if(floorDesc){ + Pattern pat1 = Pattern.compile("第(\\d{1,2})层"); + Matcher matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } + pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?"); + matcher1 = pat1.matcher(floorDesc); + if (matcher1.find()) { + return matcher1.group(1) + } +// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){ +// return "中层" +// } +// +// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){ +// return "高层" +// } +// +// +// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){ +// return "低层" +// } + if(floorDesc.isNumber()){ + return floorDesc; + } + } + } + return null; + } + + private String csvHandlerStr(String str) { + //csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了 + String tempDescription=str; + //如果有逗号 + if(str.contains(",")||str.contains(",")){ + //如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误 + if(str.contains("\"")){ +// tempDescription=str.replace("\"", "\"\""); + tempDescription=str.replace("\"", ""); + } + //在将逗号转义 + tempDescription="\""+tempDescription+"\""; + } + return tempDescription; + } + + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("非毛坯")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + + + + def marge(){ + File file = new File("D:\\数据\\建行\\新建文本文档.txt"); + def ids = new HashSet(); + def index = 0 + + file.eachLine{it-> + index++ + if(index>1){ + def arrs = it.split("\\|@\\|") + def id = arrs[1] + if(!ids.contains(id)){ + ids << id + } + } + } + println index + println ids.size() + } + + def getAllHouseIds(def commid, def cityCode){ + def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'"; + + def houseIds = new HashSet() + sqlInstance.eachRow(sql,{it-> + houseIds << it.FORSALEID + }); + return houseIds; + } + + def findAllHouseIdsByCity(){ + // 创建布隆过滤器对象 + BloomFilter filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001); +// def houseIds = new HashSet() + pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it-> + filter.put(it.gujia_sell_id) + }); + +// pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale_2021",{it-> +// filter.put(it.gujia_sell_id) +// }); + return filter; + } + + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.bigdata_forsale values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40], + r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58]) + ps.toString() + } + } + + println "end: "+new Date() + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy new file mode 100644 index 000000000..a528b4f9c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwGuiDingPriceBatch.groovy @@ -0,0 +1,226 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.CcbCodeUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import org.bson.types.ObjectId +/** + * 建行模型估值项目 + * 人工指导价同步 + */ + +@BatchOwner(Watcher.GaoJuan) +class FjwGuiDingPriceBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); + + def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15"; + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + + static main(args) { + runBatch(new FjwGuiDingPriceBatch(), args); + } + + @Override + public void process(Object args) { + def citys = CcbCommonUtils.getRunningCitys(args) + def rows = [] + + FjwPgsqlCommUtils.deleteGuidedPrice(curDate) + for (def city : citys) { + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + //添加自己库中的人工指导价 + def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01") + if(prices){ + def distMap = getDistrictMap(city) + def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) + rows.addAll(rets) + println city + " " + rets.size() + } + } + saveToPgSql(rows) + } + + def findFangDatas(def districts){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + println " 小区量:"+districts.size() + def result = Collections.synchronizedList(new ArrayList()); + for(def i=0;i + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20]) + } + } + + println "end: "+new Date() + } + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy new file mode 100644 index 000000000..21ecf0551 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwPgsqlCommUtils.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.gj.fjwPriceBaseData + +import groovy.sql.Sql + +/** + * 房价网内部使用 + */ +class FjwPgsqlCommUtils { + + public static def database = "fangjia" + + public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + public static def findAllDistrictIdsByOracle(def cityid){ + def sql = "select gujia_comm_id from base.bigdata_community where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def set = new HashSet() + sqlInstance.eachRow(sql,{r-> + set << r.gujia_comm_id + }); + return set + } + + public static def findDistGovPrice(def city){ + def ids = [] + sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it-> + ids << it.gujia_bds_id + }); + return ids; + } + + + public static def deleteGuidedPrice(def dealdate){ + def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'" + sqlInstance.executeUpdate(sql2) + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy new file mode 100644 index 000000000..63f75b256 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/fjwPriceBaseData/FjwThirdPartyPriceBatch.groovy @@ -0,0 +1,675 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql +import org.bson.types.ObjectId + +import java.text.SimpleDateFormat +import java.util.concurrent.ConcurrentHashMap + +/** + * 建行模型估值项目 + * 第三方均价数据同步 + */ +@BatchOwner(Watcher.GaoJuan) +class FjwThirdPartyPriceBatch extends BaseBatch { + + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304// + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 +// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); + + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + + def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12", + "房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"] + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") + + static main(args) { + runBatch(new FjwThirdPartyPriceBatch(), args); + } + + @Override + public void process(Object args) { + + def citys = CcbCommonUtils.getRunningCitys(args) + def stop = false + def month = -1 + while(!stop){ + if(sDate<202004){ + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) + println weekend.toString() +" "+sDate + runDatas(citys) + } + month = month -1 + sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM")); + if(sDate == 201912){ + stop = true; + } + } + } + + def runDatas(citys){ + def rows = [] + def index = 0 + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + for(def city : citys){ + index++ + + def gocIdsPrices = [] + if(city == "深圳"){ + gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city) + } + + def statMap = getDistrictStatByCity(city, weekend) + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all") + def thirdPriceMap = findThirdPartyPrice(city, statMap) + + def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices) + rows.addAll(rets) + println index+" "+city+" "+rets.size() +" "+rows.size() + } + + saveToPgSql(rows) + } + + + def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){ + def rets = [] + for(def d : dists){ + + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def data = distCodeMap.get(d.region) + if(data){ + def ret = [] + ret << third.id.toString() + ret << d._id.toString() + ret << d.name + ret << cityCodeDatas.get(d.city) + ret << d.city + ret << data.district_code + ret << data.district + ret << sDate + + if(third.zhugezhaofang_esf_price){ + ret << third.zhugezhaofang_esf_price + }else{ + ret << "" + } + if(third.anjuke_esf_price){ + ret << third.anjuke_esf_price + }else{ + ret << "" + } + if(third.soufang_esf_price){ + ret << third.soufang_esf_price + }else{ + ret << "" + } + + if(third.lianjia_esf_price){ + ret << third.lianjia_esf_price + }else{ + ret << "" + } + + if(third.cityhouse_esf_price){ + ret << third.cityhouse_esf_price + }else{ + ret << "" + } + + if(third.maitian_esf_price){ + ret << third.maitian_esf_price + }else{ + ret << "" + } + + if(third.beike_esf_price){ + ret << third.beike_esf_price + }else{ + ret << "" + } + + if(third.get("365house_esf_price")){ + ret << third.get("365house_esf_price") + }else{ + ret << "" + } + + if(third.zhongyuan_esf_price){ + ret << third.zhongyuan_esf_price + }else{ + ret << "" + } + + if(third.tuboshi_esf_price){ + ret << third.tuboshi_esf_price + }else{ + ret << "" + } + if(third.qfang_esf_price){ + ret << third.qfang_esf_price + }else{ + ret << "" + } + + if(third.fangtan_esf_price){ + ret << third.fangtan_esf_price + }else{ + ret << "" + } + + if(third.fangduoduo_esf_price){ + ret << third.fangduoduo_esf_price + }else{ + ret << "" + } + if(third.get("5i5j_esf_price")){ + ret << third.get("5i5j_esf_price") + }else{ + ret << "" + } + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + + } + return rets; + } + + + def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){ + def allRows = [] + for(def d : dists){ + if(gocPrices && gocPrices.contains(d._id.toString())){ + continue; + } + def third = thirdPriceMap.get(d.region+"##"+d.name) + if(third){ + def data = distCodeMap.get(d.region) + if(data){ + def dataMap = [:] + dataMap.COMMID = d._id.toString() + dataMap.COMMNAME = d.name + dataMap.CITYID = cityCodeDatas.get(d.city) + dataMap.CITYNAME = d.city + dataMap.DISTID = data.district_code + dataMap.DISTNAME = data.district + dataMap.YEAR_MONTH = sDate + def price_map = [:] + dataMap.price_map = price_map + if(third.zhugezhaofang_esf_price){ + price_map.zhuge_avg_price = third.zhugezhaofang_esf_price + } + if(third.anjuke_esf_price){ + price_map.anjuke_avg_price = third.anjuke_esf_price + } + if(third.soufang_esf_price){ + price_map.fang_avg_price = third.soufang_esf_price + } + if(third.lianjia_esf_price){ + price_map.lianjia_avg_price = third.lianjia_esf_price + } + if(third.cityhouse_esf_price){ + price_map.cityhouse_avg_price = third.cityhouse_esf_price + } + if(third.maitian_esf_price){ + price_map.maitian_avg_price = third.maitian_esf_price + } + if(third.beike_esf_price){ + price_map.ke_avg_price = third.beike_esf_price + } + if(third.get("365house_esf_price")){ + price_map.taofang_avg_price = third.get("365house_esf_price") + } + if(third.zhongyuan_esf_price){ + price_map.zhongyuan_avg_price = third.zhongyuan_esf_price + } + if(third.tuboshi_esf_price){ + price_map.tbs_avg_price = third.tuboshi_esf_price + } + if(third.qfang_esf_price){ + price_map.qfang_avg_price = third.qfang_esf_price + } + if(third.fangtan_esf_price){ + price_map.fangtan_avg_price = third.fangtan_esf_price + } + if(third.fangduoduo_esf_price){ + price_map.fangduoduo_avg_price = third.fangduoduo_esf_price + } + if(third.get("5i5j_esf_price")){ + price_map.wiwj_avg_price = third.get("5i5j_esf_price") + } + dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + allRows << dataMap + } + } + } + + def results = [] + for(def m : allRows){ + def priceMap = m.price_map + priceMap.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << m.CITYID + ret << m.DISTID + ret << m.COMMID + ret << m.COMMNAME + + def web = "" + if(k=="zhuge_avg_price"){ + web = "诸葛找房" + }else if(k=="anjuke_avg_price"){ + web = "安居客" + }else if(k=="fang_avg_price"){ + web = "房天下" + }else if(k=="lianjia_avg_price"){ + web = "链家" + }else if(k=="cityhouse_avg_price"){ + web = "禧泰数据" + }else if(k=="maitian_avg_price"){ + web = "麦田" + }else if(k=="ke_avg_price"){ + web = "贝壳" + }else if(k=="taofang_avg_price"){ + web = "淘房365" + }else if(k=="zhongyuan_avg_price"){ + web = "中原" + }else if(k=="tbs_avg_price"){ + web = "兔博士" + }else if(k=="qfang_avg_price"){ + web = "Q房" + }else if(k=="fangtan_avg_price"){ + web = "房探" + }else if(k=="fangduoduo_avg_price"){ + web = "房多多" + }else if(k=="wiwj_avg_price"){ + web = "我爱我家" + } + + ret << sourceMap.get(web) + ret << web + ret << m.YEAR_MONTH + ret << v + ret << "" + ret << "" + ret << m.updatetime + results << ret + } + } + return results + } + + def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) { + def rets = [] + for (def d : dists) { + + def third = thirdPriceMap.get(d.region + "##" + d.name) + if (third) { + def data = distCodeMap.get(d.region) + if (data) { + third.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << cityCodeDatas.get(d.city) + ret << data.district_code + ret << d._id.toString() + ret << d.name + def web = "" + if(k=="zhugezhaofang_esf_price"){ + web = "诸葛找房" + }else if(k=="anjuke_esf_price"){ + web = "安居客" + }else if(k=="soufang_esf_price"){ + web = "房天下" + }else if(k=="lianjia_esf_price"){ + web = "链家" + }else if(k=="cityhouse_esf_price"){ + web = "禧泰数据" + }else if(k=="maitian_esf_price"){ + web = "麦田" + }else if(k=="beike_esf_price"){ + web = "贝壳" + }else if(k=="365house_esf_price"){ + web = "淘房365" + }else if(k=="zhongyuan_esf_price"){ + web = "中原" + }else if(k=="tuboshi_esf_price"){ + web = "兔博士" + }else if(k=="qfang_esf_price"){ + ret << "Q房" + }else if(k=="fangtan_esf_price"){ + ret << "房探" + }else if(k=="fangduoduo_esf_price"){ + ret << "房多多" + }else if(k=="5i5j_esf_price"){ + ret << "我爱我家" + }else{ + ret << "" + } + + ret << sourceMap.get(web) + ret << web + ret << sDate + ret << v + + ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "FJW" + rets << ret + } + } + } + } + return rets; + } + + def createData3(cityid){ + def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def dataMap = [:] + + dataMap.COMMID = r.COMMID + dataMap.COMMNAME = r.COMMNAME + dataMap.CITYID = r.CITYID + dataMap.CITYNAME = r.CITYNAME + dataMap.DISTID = r.DISTID + dataMap.DISTNAME = r.DISTNAME + dataMap.YEAR_MONTH = r.YEAR_MONTH + def price_map = [:] + dataMap.price_map = price_map + if(r.ZHUGE_AVG_PRICE){ + price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE + } + + if(r.ANJUKE_AVG_PRICE){ + price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE + } + + if(r.FANG_AVG_PRICE){ + price_map.fang_avg_price = r.FANG_AVG_PRICE + } + + if(r.LIANJIA_AVG_PRICE){ + price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE + } + + if(r.CITYHOUSE_AVG_PRICE){ + price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE + } + + if(r.MAITIAN_AVG_PRICE){ + price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE + } + + if(r.KE_AVG_PRICE){ + price_map.ke_avg_price = r.KE_AVG_PRICE + } + + if(r.TAOFANG_AVG_PRICE){ + price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE + } + + if(r.ZHONGYUAN_AVG_PRICE){ + price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE + } + + if(r.TBS_AVG_PRICE){ + price_map.tbs_avg_price = r.TBS_AVG_PRICE + } + + if(r.QFANG_AVG_PRICE){ + price_map.qfang_avg_price = r.QFANG_AVG_PRICE + } + + if(r.FANGTAN_AVG_PRICE){ + price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE + } + + if(r.FANGDUODUO_AVG_PRICE){ + price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE + } + + if(r.WIWJ_AVG_PRICE){ + price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE + } + dataMap.updatetime = r.UPDATETIME + + allRows << dataMap + }); + + + def results = [] + for(def m : allRows){ + def priceMap = m.price_map + priceMap.each{k,v-> + def ret = [] + ret << new ObjectId().toString() + ret << m.CITYID + ret << m.DISTID + ret << m.COMMID + ret << m.COMMNAME + + def web = "" + if(k=="zhuge_avg_price"){ + web = "诸葛找房" + }else if(k=="anjuke_avg_price"){ + web = "安居客" + }else if(k=="fang_avg_price"){ + web = "房天下" + }else if(k=="lianjia_avg_price"){ + web = "链家" + }else if(k=="cityhouse_avg_price"){ + web = "禧泰数据" + }else if(k=="maitian_avg_price"){ + web = "麦田" + }else if(k=="ke_avg_price"){ + web = "贝壳" + }else if(k=="taofang_avg_price"){ + web = "淘房365" + }else if(k=="zhongyuan_avg_price"){ + web = "中原" + }else if(k=="tbs_avg_price"){ + web = "兔博士" + }else if(k=="qfang_avg_price"){ + web = "Q房" + }else if(k=="fangtan_avg_price"){ + web = "房探" + }else if(k=="fangduoduo_avg_price"){ + web = "房多多" + }else if(k=="wiwj_avg_price"){ + web = "我爱我家" + } + + ret << sourceMap.get(web) + ret << web + ret << m.YEAR_MONTH + ret << v + ret << "" + ret << "" + ret << m.updatetime + results << ret + } + } + + return results + + } + + + def findThirdPartyPrice(def city, def statMap){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + dataMap.put("id", db._id.toString()) + map.put(key, dataMap) + } + if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){ + dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){ + dataMap.put("soufang_esf_price", db.soufang_esf_price) + } + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ + dataMap.put("maitian_esf_price", db.maitian_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + + if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){ + dataMap.put("365house_esf_price", db.get("365house_esf_price")) + } + + if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ + dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price) + } + + if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ + dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price) + } + + if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){ + dataMap.put("qfang_esf_price", db.qfang_esf_price) + } + + if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){ + dataMap.put("fangtan_esf_price", db.fangtan_esf_price) + } + + if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){ + dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price) + } + + if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ + dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price")) + } +// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ +// map.put(key, db) +// } + } + + def resultMap = [:] + map.each {key, value-> + if(value && value.size()>1){ + if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){ + def price = statMap.get(key) + if(price){ + def priceMap = [:] + def startPrice = (int)(price - price*0.3) + def endPrice = (int)(price*0.3 + price) + value.each{pk,pv-> + if(pk.contains("price")){ + if(pv>=startPrice && pv<=endPrice){ + priceMap.put(pk,pv) + } + }else{ + priceMap.put(pk,pv) + } + } + if(priceMap.size()>1){ + resultMap.put(key, priceMap) + } + } + }else{ + resultMap.put(key, value) + } + + } + } + return resultMap + } + + + def getDistrictStatByCity(city, weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def dbcursor = DIST_SELL_STAT.find(q) + def map = new ConcurrentHashMap() + for(def d : dbcursor){ + map.put(d.region+"##"+d.district_name, d.avg_price) + } + return map + } + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28" + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(50000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18]) + } + } + + println "end: "+new Date() + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy index f517a80b8..99b913307 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meishubao/MsbPocTest.groovy @@ -13,7 +13,7 @@ class MsbPocTest { task.run(); } def run(){ -// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx" + def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx" // def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx // standard(inputfile) // match(inputfile) @@ -24,7 +24,7 @@ class MsbPocTest { // cityPrice() // mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx") - addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx") + addCCBPrice(inputfile) } def cityPrice(){ @@ -69,21 +69,21 @@ class MsbPocTest { def matchOld(path){ ExcelUtils.multiThreadingModify(path, { row -> - if(!row['S'] && row['D']){ + if(!row['Y'] && row['G']){ def city = row['C'] - def region = row['D'] - def name = row['F'] - def address = row['H'] + def region = null //row['D'] + def name = row['E'] + def address = row['G'] println "[match] city:"+city+" region:"+region+" address:"+address - def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name) + def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name) if(jsonObject != null && jsonObject.precise!=-1) { - row['S'] = jsonObject.get("id") - row['T'] = jsonObject.get("city") - row['U'] = jsonObject.get("district") - row['V'] = jsonObject.get("name") - row['W'] = jsonObject.get("propertyType") - row['X'] = jsonObject.get("credit") + row['Y'] = jsonObject.get("id") + row['Z'] = jsonObject.get("city") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") + row['AC'] = jsonObject.get("propertyType") +// row['X'] = jsonObject.get("credit") } } }) @@ -111,23 +111,18 @@ class MsbPocTest { def match(file){ ExcelUtils.multiThreadingModify(file, { row -> - if(!row['J'] && row['F'] && row['P']=="没匹配" ){ + if(!row['Y']){ def city = row['C'] - def region = row['D'] + def region = null //row['B'] def address = row['G'] //row['G']//row['F'] println "[match] city:"+city+" region:"+region+" address:"+address - def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false) - if(jsonObject != null && jsonObject.credit>0) { - row['J'] = jsonObject.get("id") - row['K'] = jsonObject.get("city") - row['L'] = jsonObject.get("district") - row['M'] = jsonObject.get("name") - def type = jsonObject.get("propertyType") - if(type && type!='null'){ - row['N'] = type - } - row['O'] = jsonObject.get("credit") + def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false) + if(jsonObject != null) { + row['Y'] = jsonObject.get("id") + row['Z'] = jsonObject.get("city") + row['AA'] = jsonObject.get("district") + row['AB'] = jsonObject.get("name") } } }) @@ -174,19 +169,22 @@ class MsbPocTest { def addCCBPrice(path){ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(path, { row -> - if(!row['M']){ - def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") - if(dist){ - def d = CommonDatautils.findDistrictById(dist._id.toString()) + if(!row['V']){ +// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district") +// if(dist){ + if(row['Y'] instanceof String){ + def d = CommonDatautils.findDistrictById(row['Y']) if(d){ def cityId = cityCodeDataMap.get(d.city) - def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId) + def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId) if(map){ - row['M'] = map.avg_price + row['V'] = map.avg_price +// row['I'] = map.calc_type } - } } + +// } } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index db5b55b54..f0c4282e1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -33,17 +33,31 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "C:\\Users\\gaojuan\\Desktop\\7月成交案例badcase.xlsx" -// addGXDPrice(filePath) + def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx" + addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) // addFjwPrice(filePath) - exportPkPrice() +// exportPkPrice() // addRegion() +// addTradePrice(filePath) } + def addTradePrice(filePath){ + ExcelUtils.multiThreadingModify(filePath, { d -> + if(d['Y']){ + def dist = CommonDatautils.findDistrictById(d['Y']) + def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01') + if (tradePrice) { + d['W'] = tradePrice.toInteger() + } + } + + }); + } + def addRegion(){ ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d-> def dist = CommonDatautils.findDistrictById(d['A']) @@ -153,15 +167,15 @@ class AddPriceTest { } def addCcbPrice(filepath){ - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() ExcelUtils.multiThreadingModify(filepath,{ d-> - if(d['M']){ - def cityId = cityCodeDataMap.get(d['N']) - def map = findAvgPrice(d['M'], cityId) + if(d['A']){ +// def cityId = cityCodeDataMap.get(d['N']) + def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger()) if(map){ - d['AK'] = map.avg_price - d['AL'] = map.calc_type - d['AM'] = map.credit_value +// d['AK'] = map.avg_price + d['P'] = map.calc_type +// d['AM'] = map.credit_value } } }) @@ -190,11 +204,11 @@ class AddPriceTest { def addThirdPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['M']) { - def dist = CommonDatautils.findDistrictById(d['M']) + if (!d['X']) { + def dist = CommonDatautils.findDistrictById(d['Y']) if(dist){ def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name) - d['AN'] = thirdPrice + d['X'] = thirdPrice } } }) @@ -206,8 +220,8 @@ class AddPriceTest { q.put("city", city) q.put("region", region) q.put("name", name) - q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302)) - +// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404)) + q.put("s_date", 202404) def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) def price = null for(def db : dbcursor){ @@ -216,10 +230,10 @@ class AddPriceTest { break; } - if(!price && db.beike_esf_price && db.beike_esf_price>2000){ - price = db.beike_esf_price - break - } +// if(!price && db.beike_esf_price && db.beike_esf_price>2000){ +// price = db.beike_esf_price +// break +// } } return price } @@ -227,12 +241,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> - if (d['B']) { - def dist = CommonDatautils.findDistrictById(d['E']) + if (!d['M']) { + def dist = CommonDatautils.findDistrictById(d['B']) if(dist){ def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) // def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['L'] = gxdPrice + d['M'] = gxdPrice } } }) @@ -256,14 +270,17 @@ class AddPriceTest { def addFjwPrice(path){ ExcelUtils.multiThreadingModify(path,{d-> - if(!d['K']){ - def dist = CommonDatautils.findDistrictById(d['E']) - if(dist){ + if(!d['L']){ + if(d['B'] instanceof String){ + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ // d['N'] = dist.region - def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd")) - if(dbo){ -// d['K'] = dbo.avg_price - d['K'] = dbo.avg_price_2b + def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name) + if(dbo){ +// d['W'] = dbo.avg_price +// d['X'] = dbo.cal_type + d['L'] = dbo.avg_price_2b + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy index dade0fa1f..35f821ce3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/QiliangTest.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.HttpClient +import groovy.json.JsonOutput import net.sf.json.JSONObject import okhttp3.*; import java.io.IOException; @@ -15,177 +16,76 @@ import java.util.concurrent.TimeUnit; import okhttp3.Request.Builder; class QiliangTest { + String proxyHost = 'tunnel2.qg.net'; + Integer proxyPort = 14186; + String proxyUsername = '113BBC06'; + String proxyPassword = '3B2FBD05B0AF'; + String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; -// static main(args) { -// def task = new QiliangTest() -//// task.run() -// task.getHttpResult("扬州扬子江北路399号11041") -// } + static main(args) { + QiliangTest task = new QiliangTest(); +// task.run() - def run(){//地址测试-2024.3.4-结果.xlsx - ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> - if(!row['N']){ - def address = null - if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){ - address = row['B']+row['C'] - }else{ - address = row['C'] - } - if(address){ - def result = getHttpResult(address) - if(result && result.code==20000){ - def data = result.data - if(data){ - if(data.province && data.province!='null'){ - row['N'] = data.province - } - if(data.city && data.city!='null'){ - row['O'] = data.city - } - if(data.district && data.district!='null'){ - row['P'] = data.district - } - if(data.town && data.town!='null'){ - row['Q'] = data.town - } - if(data.village && data.village!='null'){ - row['R'] = data.village - } - - if(data.areaCode && data.areaCode!='null'){ - row['S'] = data.areaCode - } - - if(data.farmer == "0"){ - row['T'] = "非农" - }else{ - row['T'] = "农" - } - if(data.isGovt == "0"){ - row['U'] = "否" - }else{ - row['U'] = "是" - } - - def list = data.indusParkList - if(list && list!="null"){ - def s = "" - for(def r : list){ - s = s + r.indusParkName +"-"+r.indusParkLevel+"\n" - } - row['V'] = s - } - if(data.govtAddr && data.govtAddr!='null'){ - row['W'] = data.govtAddr?data.govtAddr:'' - } - - } - } - } - } - }) + def data = [ + 'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发', + 'type': '1' //1:对公 0:对私 + ] + def s = task.getProxyResult(data); + print(s); } - def getHttpResult(address){ - def data = new JSONObject() - data.put("address", address) - data.put("type","2") - Map headers = new HashMap(); - headers.put("Accept", "application/json, text/plain, */*"); - headers.put("Accept-Encoding", "gzip, deflate"); - headers.put("Accept-Language","zh-CN,zh;q=0.9") - headers.put("Content-Type","application/json;charset=UTF-8") - headers.put("Host","146.56.217.44:9010") - headers.put("Origin","http://146.56.217.44:8088") - headers.put("Proxy-Connection","keep-alive") - headers.put("Referer","http://146.56.217.44:8088/") - headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") - HttpRequest request1 = new HttpRequest() - try{ - String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8") - def response = JSONObject.fromObject(res) - println response - return response - }catch(Exception e){ - e.printStackTrace() + + String getProxyResult(data){ + // 将 JSON 对象转换为字符串 + String jsonData = new JsonOutput().toJson(data); + // 创建代理对象 + Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); + + // 创建 URL 对象 + URL url = new URL(targetUrl); + + // 创建连接对象 + URLConnection connection = url.openConnection(proxy); + + // 设置代理认证 + java.net.Authenticator.setDefault(new java.net.Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()) + } + }) + + // 设置请求方法为 POST + connection.setRequestMethod("POST"); + + // 允许输出流 + connection.setDoOutput(true); + + // 设置请求头 + connection.setRequestProperty("Content-Type", "application/json"); + + // 发送请求参数 + OutputStream outputStream = connection.getOutputStream(); + outputStream.write(jsonData.getBytes("UTF-8")); + outputStream.flush(); + outputStream.close(); + + // 读取响应 + int responseCode = connection.getResponseCode() + if (responseCode == HttpURLConnection.HTTP_OK) { + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + StringBuilder response = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + response.append(line); + } + reader.close(); + return response.toString(); + } else { + println "HTTP request failed with response code ${responseCode}"; } return null; } - - final static String proxyIp = "tunnel2.qg.net"; - final static Integer proxyPort = 14186; - final static String authKey = "113BBC06"; - final static String password = "3B2FBD05B0AF"; - public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8"); - -// public String request(params) throws IOException { -// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort)); -// OkHttpClient client = new OkHttpClient.Builder() -// .proxy(proxy) -// .proxyAuthenticator((route, response){ -// String credential = Credentials.basic(authKey, password); -// return response.request().newBuilder().header("Proxy-Authorization", credential).build(); -// }). -// build(); -//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build(); -// -// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params)); -// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build(); -// -// Response response = client.newCall(request).execute(); -// return response.body().string(); -// -//// return client.newCall(request).execute(); -// } - public static void main(String[] args) { - def proxyHost = "tunnel2.qg.net" - def proxyPort = 14186 - def proxyUsername = '113BBC06' - def proxyPassword = '3B2FBD05B0AF' - - def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign' - def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2'] - - -// 创建代理服务器的连接 - def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}") - -// 打开代理连接 - def proxyConnection = proxy.openConnection() - -// 创建目标URL的连接 - def url = new URL(targetUrl) - def connection = url.openConnection(proxy) - -// 设置请求方法 - connection.setRequestMethod("GET") - -// 添加请求头(可选) - connection.setRequestProperty("Content-Type", "application/json") - -// 构建查询参数 - def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&') - -// 将查询参数附加到目标URL - def finalUrl = "${targetUrl}?${queryString}" - -// 发送请求 - connection.connect() - -// 获取响应 - def responseCode = connection.getResponseCode() - def responseMessage = connection.getResponseMessage() - -// 读取响应内容 - def reader = new BufferedReader(new InputStreamReader(connection.getInputStream())) - def responseContent = reader.readLine() - reader.close() - - println "Response Code: ${responseCode}" - println "Response Message: ${responseMessage}" - println "Response Content: ${responseContent}" - } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy index 785955d49..980d2863d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy @@ -18,77 +18,75 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun //def s = getTownshipInfo("118.702988,39.740979") def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx" -//ExcelUtils.multiThreadingModify(filePath, { row -> -//// if(!row['I']){ -//// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府") -//// if(rets){ -//// def obj = findOneByName(rets, row['C']+"人民政府") -//// if(obj){ -//// row['I'] = obj.lng -//// row['J'] = obj.lat -//// println row['C'] + " "+obj -//// } -//// } -//// } -// -//// if(row['D'] && !row['F']){ -//// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL) -//// def map = ["lon":row['D'], "lat":row['E']] -//// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap) -//// row['F'] = township -//// } -// -//// if(row['F'] && !row['J']){ -//// def q = new BasicDBObject() -//// q.put("city_name", row['B']) -//// q.put("area_name", row['C']) -//// def streetName = row['F'] -//// q.put("street_name", streetName) -//// def count = COMMUNITY_CONN.count(q); -//// row['J'] = count -////// row['H'] = streetName -//// } -//// if(!row['H']){ -//// def gaodeMap = getTownshipInfo(row['D']+","+row['E']) -//// if(gaodeMap && gaodeMap.township){ -//// row['H'] = gaodeMap.township -//// } -//// } -// -//// if(row['J'] && row['J'].toString().toDouble()>0){ -//// def up_query = new BasicDBObject() -//// up_query.put("city_name", row['B']) -//// up_query.put("area_name", row['C']) -//// up_query.put("street_name", row['F']) -//// def udata = new BasicDBObject() -//// udata.put("has_gov", "是") -//// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true) -//// } -// -// -//}) - - -def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT); -for(def d : objs){ - if(!d.name_pinyin){ - if(d.name){ - println d._id - def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ") - def udata = new BasicDBObject() - udata.put("name_pinyin", py1) - if(d.location){ - def lon = DataHandleUtils.getDoubleRound(6, d.location.lon) - def lat = DataHandleUtils.getDoubleRound(6, d.location.lat) - udata.put("location", ["lat":lat, "lon":lon]) +ExcelUtils.modify(filePath, { row -> + if(!row['I'] && row['I']=='宝应县'){ + def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府") + if(rets){ + def obj = findOneByName(rets, row['C']+"人民政府") + if(obj){ + row['I'] = obj.lng + row['J'] = obj.lat + println row['C'] + " "+obj } - - def q = new BasicDBObject() - q.put("_id", d._id) - COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true) } } -} + + if(row['D'] && !row['F']){ + def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL) + def map = ["lon":row['D'], "lat":row['E']] + def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap) + row['F'] = township + } + +// if(row['F'] && !row['J']){ +// def q = new BasicDBObject() +// q.put("city_name", row['B']) +// q.put("area_name", row['C']) +// def streetName = row['F'] +// q.put("street_name", streetName) +// def count = COMMUNITY_CONN.count(q); +// row['J'] = count +//// row['H'] = streetName +// } +// if(!row['H']){ +// def gaodeMap = getTownshipInfo(row['D']+","+row['E']) +// if(gaodeMap && gaodeMap.township){ +// row['H'] = gaodeMap.township +// } +// } + +// if(row['J'] && row['J'].toString().toDouble()>0){ +// def up_query = new BasicDBObject() +// up_query.put("city_name", row['B']) +// up_query.put("area_name", row['C']) +// up_query.put("street_name", row['F']) +// def udata = new BasicDBObject() +// udata.put("has_gov", "是") +// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true) +// } +}) + + +//def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT); +//for(def d : objs){ +// if(!d.name_pinyin){ +// if(d.name){ +// println d._id +// def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ") +// def udata = new BasicDBObject() +// udata.put("name_pinyin", py1) +// if(d.location){ +// def lon = DataHandleUtils.getDoubleRound(6, d.location.lon) +// def lat = DataHandleUtils.getDoubleRound(6, d.location.lat) +// udata.put("location", ["lat":lat, "lon":lon]) +// } +// +// def q = new BasicDBObject() +// q.put("_id", d._id) +// COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true) +// } +// } +//} def getTownshipInfo(def glocation){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/test.groovy new file mode 100644 index 000000000..b186739ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/test.groovy @@ -0,0 +1,181 @@ +package com.fangjia.script.gj.village + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.HttpClient +import groovy.json.JsonOutput +import net.sf.json.JSONObject +import okhttp3.*; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.util.concurrent.TimeUnit; + + +import okhttp3.Request.Builder; + +class QiliangTest1 { + def proxyHost = 'tunnel2.qg.net' + def proxyPort = 14186 + def proxyUsername = '113BBC06' + def proxyPassword = '3B2FBD05B0AF' + String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign"; + + static main(args) { + def task = new QiliangTest() +// task.run() + + def data = [ + 'address': '江苏省扬州市宝应县安宜镇宝应大道88号', + 'type': '1' + ] + def s = task.getProxyResult(data) + print(s) + } + + def run(){//地址测试-2024.3.4-结果.xlsx + ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row -> + if(!row['N']){ + def address = null + if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){ + address = row['B']+row['C'] + }else{ + address = row['C'] + } + if(address){ + def result = getHttpResult(address) + if(result && result.code==20000){ + def data = result.data + if(data){ + if(data.province && data.province!='null'){ + row['N'] = data.province + } + if(data.city && data.city!='null'){ + row['O'] = data.city + } + if(data.district && data.district!='null'){ + row['P'] = data.district + } + if(data.town && data.town!='null'){ + row['Q'] = data.town + } + if(data.village && data.village!='null'){ + row['R'] = data.village + } + + if(data.areaCode && data.areaCode!='null'){ + row['S'] = data.areaCode + } + + if(data.farmer == "0"){ + row['T'] = "非农" + }else{ + row['T'] = "农" + } + if(data.isGovt == "0"){ + row['U'] = "否" + }else{ + row['U'] = "是" + } + + def list = data.indusParkList + if(list && list!="null"){ + def s = "" + for(def r : list){ + s = s + r.indusParkName +"-"+r.indusParkLevel+"\n" + } + row['V'] = s + } + if(data.govtAddr && data.govtAddr!='null'){ + row['W'] = data.govtAddr?data.govtAddr:'' + } + + } + } + } + } + }) + } + + JSONObject getHttpResult(address){ + JSONObject data = new JSONObject() + data.put("address", address) + data.put("type","2") + + Map headers = new HashMap(); + headers.put("Accept", "application/json, text/plain, */*"); + headers.put("Accept-Encoding", "gzip, deflate"); + headers.put("Accept-Language","zh-CN,zh;q=0.9") + headers.put("Content-Type","application/json;charset=UTF-8") + headers.put("Host","146.56.217.44:9010") + headers.put("Origin","http://146.56.217.44:8088") + headers.put("Proxy-Connection","keep-alive") + headers.put("Referer","http://146.56.217.44:8088/") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") + + HttpRequest request1 = new HttpRequest() + try{ + String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8") + JSONObject response = JSONObject.fromObject(res) + println response + return response + }catch(Exception e){ + e.printStackTrace() + } + return null; + } + + + def getProxyResult(data){ + // 将 JSON 对象转换为字符串 + def jsonData = new JsonOutput().toJson(data) + // 创建代理对象 + def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)) + + // 创建 URL 对象 + def url = new URL(targetUrl) + + // 创建连接对象 + def connection = url.openConnection(proxy) + + // 设置代理认证 + java.net.Authenticator.setDefault(new java.net.Authenticator() { + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray()) + } + }) + + // 设置请求方法为 POST + connection.setRequestMethod("POST") + + // 允许输出流 + connection.setDoOutput(true) + + // 设置请求头 + connection.setRequestProperty("Content-Type", "application/json") + + // 发送请求参数 + def outputStream = connection.getOutputStream() + outputStream.write(jsonData.getBytes("UTF-8")) + outputStream.flush() + outputStream.close() + + // 读取响应 + def responseCode = connection.getResponseCode() + if (responseCode == HttpURLConnection.HTTP_OK) { + def reader = new BufferedReader(new InputStreamReader(connection.getInputStream())) + def response = new StringBuilder() + String line + while ((line = reader.readLine()) != null) { + response.append(line) + } + reader.close() + return response.toString() + } else { + println "HTTP request failed with response code ${responseCode}" + } + return null + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy index cfda2228d..63996e918 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/GaoDePoiDataProcess.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.RegionData import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject import org.bson.types.ObjectId import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse @@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{ def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL) + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL) DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL) @@ -58,6 +60,9 @@ class GaoDePoiDataProcess extends BaseBatch{ def fullCityMap = null + DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + + DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL) static main(args) { runBatch(new GaoDePoiDataProcess(), args) @@ -69,12 +74,215 @@ class GaoDePoiDataProcess extends BaseBatch{ // synchMapPoi(); - synchCompany() +// synchCompany() // poiCatProcess() // addressAndCatProcess() // synchNetwork() // println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime() // sysnchPoi() +// sysnchVillage() + + addTownship() + } + + def addTownship(){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d-> + if(d['E']){ +// def gaodeMap = getTownshipInfo(d['F']+","+d['E']) +// if(gaodeMap && gaodeMap.township){ +// if(gaodeMap.city && gaodeMap.city!="[]"){ +// d['H'] = gaodeMap.city +// } +// if( gaodeMap.district && gaodeMap.district!='[]'){ +// d['I'] = gaodeMap.district +// } +// if(gaodeMap.township && gaodeMap.township!='[]'){ +// d['J'] = gaodeMap.township +// } +// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){ +// d['K'] = gaodeMap.format_addr +// } +// +// } + + def q = new BasicDBObject() + q.put("city", d['H']) + q.put("district", d['I']) + q.put("street",d['J']) + def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q) + if(obj){ + d['L'] = "是" + } + } + },0) + + } + + def sysnchVillage(){ + def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"] + for(def city : buducitys){ + def pois = findAllVillagePoiMap(city) + def rows = createVillageData(city, pois) + + save(city, rows) + saveToMG(city, rows) + println city +" "+pois.size()+" add:"+rows.size() + } + + } + + def createVillageData(city, pois){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(pois.size()) + def townMap = findTownRange(city) + + for(def i=0;i + if(row['Q'] && row['Q']=='是'){ + def udata = new BasicDBObject() + udata.put("township", row['J']) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata)) + } + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index ac85c450c..907e2a572 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -98,6 +98,23 @@ class InsidePolygon { } return list; } + + public static List> createPoint4(def pointList){ + List> alllist = new ArrayList<>(); + for(def pointArr : pointList){ + List list = new ArrayList<>(); + for(def arr : pointArr){ + Point point = new Point(); + double x = arr[0]; + double y = arr[1]; + point.setLon(x); + point.setLat(y); + list.add(point); + } + alllist << list + } + return alllist + } /** * 判定一个经纬度点与区域的关系 @@ -284,7 +301,7 @@ class InsidePolygon { if(flag){ def arrs = key.split("##") name = arrs[1]; - break; +// break; } } if(flag){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 18fdb2d2f..0e5692cb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -46,7 +46,7 @@ import java.util.regex.Pattern * 8:地址及地址别名查找 * 9:地址及地址别名(不加区域限制)查找 * 10:拼音匹配 - *11:open接口匹配 + * 11:open接口匹配 * 14:拼音匹配,不加地址限制 */ @BatchOwner(Watcher.GaoJuan) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 9c6952a70..aeda5fc28 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.* public class DbUtils{ public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com"; - public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log'); + //public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log'); public static final String COMMON_DB = "fangjia"; public static final int COMMON_PORT = 27017; - public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater") + //public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater") public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index f14bedadd..6aba12207 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -76,7 +76,6 @@ class ElasticUtils { connection.close() } - static RestHighLevelClient getClient(String host) { RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { @@ -101,6 +100,30 @@ class ElasticUtils { } + static RestHighLevelClient getClient89(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456 + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + + /** * 创建一个ES批处理器 * @param host diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy index ab5c90233..b3b119384 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/amap/GaoDeApiUtils.groovy @@ -12,7 +12,7 @@ import org.jsoup.Jsoup import java.util.zip.GZIPInputStream class GaoDeApiUtils { - def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" + def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995 /** * 经纬度转地址
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java index 8cd9fc3f9..04e5d5de2 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictService.java @@ -6,6 +6,7 @@ import com.fangjia.dal.SdQuery; import com.fangjia.dal.SortParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.model.AgencyDistrictTradeRank; +import com.fangjia.dal.mongo.model.AliasFang; import com.fangjia.dal.mongo.model.DistrictFacilitys; import com.fangjia.dal.seaweed.model.District; import com.fangjia.service.bo.DistrictInfo; @@ -540,10 +541,10 @@ public interface DistrictService extends ModeService { * * @param city * @param region - * @param masterDistrictNname + * @param masterDistrictName * @return */ - List findSlaves(String city, String region, String masterDistrictNname); + List findSlaves(String city, String region, String masterDistrictName); /** * 分期操作时设置此小区为人工操作标识 diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java index 31b588538..dd625b469 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java @@ -2717,12 +2717,12 @@ public class DistrictServiceImpl implements DistrictService { } @Override - public List findSlaves(String city, String region, String masterDistrictNname) { + public List findSlaves(String city, String region, String masterDistrictName) { List params = new ArrayList(); params.add(new QueryParam("visible", 0)); params.add(new QueryParam("city", city)); params.add(new QueryParam("region", region)); - params.add(new QueryParam("masterDistrictNname", masterDistrictNname)); + params.add(new QueryParam("masterDistrictName", masterDistrictName)); return districtDao.findAllByParams(params); } @@ -2930,5 +2930,4 @@ public class DistrictServiceImpl implements DistrictService { params.add(new QueryParam("$sr", new SrQuery(query))); return districtDao.findAllByParams(params); } - } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DevelopmentZoneService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DevelopmentZoneService.java index fe9313f52..3b3209dad 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DevelopmentZoneService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DevelopmentZoneService.java @@ -33,7 +33,7 @@ public interface DevelopmentZoneService { * @param id * @param udata */ - public void updateDevelopmentZone(DevelopmentZone developmentZone); + public void updateDevelopmentZone(Map udata, String id); /** * 根据ID查询 @@ -51,7 +51,9 @@ public interface DevelopmentZoneService { * @param pager * @return */ - public List findDevelopmentZone(String province,String name, Pager pager); + public List findDevelopmentZone(String province,String name,String level, Pager pager); + + public long countDev(String province,String name,String level); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DevelopmentZoneServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DevelopmentZoneServiceImpl.java index a35b49f5f..f6ddb1ce3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DevelopmentZoneServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DevelopmentZoneServiceImpl.java @@ -10,6 +10,7 @@ import org.sothis.core.util.Pager; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; public class DevelopmentZoneServiceImpl implements DevelopmentZoneService { private DevelopmentZoneDao developmentZoneDao; @@ -26,8 +27,8 @@ public class DevelopmentZoneServiceImpl implements DevelopmentZoneService { } @Override - public void updateDevelopmentZone(DevelopmentZone developmentZone) { - developmentZoneDao.update(developmentZone); + public void updateDevelopmentZone(Map map,String id) { + developmentZoneDao.updateById(map,id); } @Override @@ -37,19 +38,44 @@ public class DevelopmentZoneServiceImpl implements DevelopmentZoneService { @Override - public List findDevelopmentZone(String province, String name, Pager pager) { + public List findDevelopmentZone(String province,String name,String level, Pager pager) { List params = new ArrayList(); if (StringUtils.isNotBlank(province)) { - params.add(new QueryParam("province", province)); + Pattern pattern = Pattern.compile(".*" + province + ".*"); + params.add(new QueryParam("province", pattern)); } if (StringUtils.isNotBlank(name)) { - params.add(new QueryParam("name", name)); + Pattern pattern = Pattern.compile(".*" + name + ".*"); + params.add(new QueryParam("name", pattern)); + } + if (StringUtils.isNotBlank(level)) { + Pattern pattern = Pattern.compile(".*" + level + ".*"); + params.add(new QueryParam("level", pattern)); } return developmentZoneDao.findAllByParams(params, pager.getStartRow(), pager.getPageSize()); } + @Override + public long countDev(String province, String name,String level) { + List params = new ArrayList(); + if (StringUtils.isNotBlank(province)) { + Pattern pattern = Pattern.compile(".*" + province + ".*"); + params.add(new QueryParam("province", pattern)); + } + + if (StringUtils.isNotBlank(name)) { + Pattern pattern = Pattern.compile(".*" + name + ".*"); + params.add(new QueryParam("name", pattern)); + } + if (StringUtils.isNotBlank(level)) { + Pattern pattern = Pattern.compile(".*" + level + ".*"); + params.add(new QueryParam("level", pattern)); + } + return developmentZoneDao.findCountByParams(params); + } + public void setDevelopmentZoneDao(DevelopmentZoneDao developmentZoneDao) { this.developmentZoneDao = developmentZoneDao; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java new file mode 100644 index 000000000..c3136da96 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangService.java @@ -0,0 +1,23 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.model.AliasFang; + +import java.util.List; +import java.util.Set; + +/** + * @author shenjian + * @since 2024/6/11 + */ +public interface AliasFangService { + + /** + * 查询搜房别名与名称关系 + * + * @param city + * @param region + * @param names + * @return + */ + List findFangByNameAndAlias(String city, String region, Set names); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java new file mode 100644 index 000000000..2437795cd --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/AliasFangServiceImpl.java @@ -0,0 +1,27 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.mongo.dao.AliasFangDao; +import com.fangjia.dal.mongo.model.AliasFang; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * @author shenjian + * @since 2024/6/11 + */ +public class AliasFangServiceImpl implements AliasFangService{ + + private AliasFangDao aliasFangDao; + + @Override + public List findFangByNameAndAlias(String city, String region, Set names) { + return aliasFangDao.findFangByNameAndAlias(city, region, names); + } + + public void setAliasFangDao(AliasFangDao aliasFangDao) { + this.aliasFangDao = aliasFangDao; + } +}