This commit is contained in:
mjt
2024-03-04 09:32:06 +08:00
37 changed files with 649 additions and 209 deletions
@@ -206,7 +206,7 @@ public class AddrSvcController {
//新匹配接口 //新匹配接口
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaNlpService/match", params, "UTF-8"); json = request.getJsonContent("http://192.168.0.100:32080/FangJiaNlpService/match", params, "UTF-8");
BasicModel basicModel = new BasicModel(200, "ok"); BasicModel basicModel = new BasicModel(200, "ok");
if (json != null && json.size()>0) { if (json != null && json.size() > 0) {
basicModel.setResult(CommonDtoUtils.transformV(MatchedBuildingDto.class, json)); basicModel.setResult(CommonDtoUtils.transformV(MatchedBuildingDto.class, json));
} }
mav.setModel(basicModel); mav.setModel(basicModel);
@@ -284,16 +284,16 @@ public class AddrSvcController {
//新匹配接口 //新匹配接口
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaNlpService/match", params, "UTF-8"); json = request.getJsonContent("http://192.168.0.100:32080/FangJiaNlpService/match", params, "UTF-8");
BasicModel basicModel = new BasicModel(200, "ok"); BasicModel basicModel = new BasicModel(200, "ok");
if (json != null && json.size()>0) { if (json != null && json.size() > 0) {
MatchedBuildingDto dto = CommonDtoUtils.transformV(MatchedBuildingDto.class, json); MatchedBuildingDto dto = CommonDtoUtils.transformV(MatchedBuildingDto.class, json);
District district = districtService.findByNameAndAlias(dto.getCity(), dto.getDistrict(), dto.getName(),"district"); District district = districtService.findByNameAndAlias(dto.getCity(), dto.getDistrict(), dto.getName(), "district");
if (null != district) { if (null != district) {
DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(dto.getCity(), dto.getDistrict(), dto.getName())); DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(dto.getCity(), dto.getDistrict(), dto.getName()));
if(null != statBo){ if (null != statBo) {
if(statBo.getDistrictStat()!=null){ if (statBo.getDistrictStat() != null) {
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice()); dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
}else if(statBo.getVillaDistrictStat()!=null){ } else if (statBo.getVillaDistrictStat() != null) {
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice()); dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
} }
@@ -309,8 +309,9 @@ public class AddrSvcController {
return mav; return mav;
} }
/** /**
*五级地址标准化 * 五级地址标准化
* *
* @param model * @param model
* @return * @return
@@ -357,6 +358,54 @@ public class AddrSvcController {
return mav; return mav;
} }
/**
* 涉农识别-新规则
*
* @param model
* @return
*/
public Object villageIdentifyAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
params.put("address", model.getAddress());
params.put("type", StringUtils.isNotBlank(model.getType()) ? model.getType() : null);
params.put("showFullValue", StringUtils.isNotBlank(model.getShowFullValue()) ? model.getShowFullValue() : null);
params.put("showTestValue", StringUtils.isNotBlank(model.getShowTestValue()) ? model.getShowTestValue() : null);
Integer radius = model.getRadius();
params.put("radius", (radius != null && radius != 0) ? radius.toString() : null);
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
} catch (Exception e) {
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("msgCode");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
public void setDistrictStatService(DistrictStatService districtStatService) { public void setDistrictStatService(DistrictStatService districtStatService) {
this.districtStatService = districtStatService; this.districtStatService = districtStatService;
} }
@@ -42,6 +42,10 @@ public class AddressModel extends FangjiaApiModel {
//是否使用地图poi数据 //是否使用地图poi数据
private boolean useMap; private boolean useMap;
//涉农到区之后,是否显式后面的字段值
private String showFullValue;
//是否显示涉农测试用字段值
private String showTestValue;
public boolean isContainsLocation() { public boolean isContainsLocation() {
return containsLocation; return containsLocation;
} }
@@ -258,4 +262,20 @@ public class AddressModel extends FangjiaApiModel {
public void setUseMap(boolean useMap) { public void setUseMap(boolean useMap) {
this.useMap = useMap; this.useMap = useMap;
} }
public String getShowFullValue() {
return showFullValue;
}
public void setShowFullValue(String showFullValue) {
this.showFullValue = showFullValue;
}
public String getShowTestValue() {
return showTestValue;
}
public void setShowTestValue(String showTestValue) {
this.showTestValue = showTestValue;
}
} }
+15 -15
View File
@@ -537,21 +537,21 @@
<!-- <scope>compile</scope>--> <!-- <scope>compile</scope>-->
<!-- </dependency>--> <!-- </dependency>-->
<dependency> <!-- <dependency>-->
<groupId>com.aliyun</groupId> <!-- <groupId>com.aliyun</groupId>-->
<artifactId>aliyun-java-sdk-address-purification</artifactId> <!-- <artifactId>aliyun-java-sdk-address-purification</artifactId>-->
<version>1.0.1</version> <!-- <version>1.0.1</version>-->
</dependency> <!-- </dependency>-->
<dependency> <!-- <dependency>-->
<groupId>com.aliyun</groupId> <!-- <groupId>com.aliyun</groupId>-->
<artifactId>aliyun-java-sdk-core</artifactId> <!-- <artifactId>aliyun-java-sdk-core</artifactId>-->
<version>4.5.9</version> <!-- <version>4.5.9</version>-->
</dependency> <!-- </dependency>-->
<dependency> <!-- <dependency>-->
<groupId>com.alibaba</groupId> <!-- <groupId>com.alibaba</groupId>-->
<artifactId>fastjson</artifactId> <!-- <artifactId>fastjson</artifactId>-->
<version>1.2.83</version> <!-- <version>1.2.83</version>-->
</dependency> <!-- </dependency>-->
</dependencies> </dependencies>
@@ -2,38 +2,36 @@ package com.fangjia.script
public enum Watcher { public enum Watcher {
//ÀëְͬÊ //ÀëְͬÊÂ
LiuPeiPei('13621626685', 'duxiaoyu@fangjia.com'), LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'),
YanZhou('13621626685','duxiaoyu@fangjia.com'), YanZhou('13621626685','wangwenyan@fangjia.com'),
HuangXiaoHu('13621626685', 'duxiaoyu@fangjia.com'), HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'),
SongYingXia('13621626685', 'duxiaoyu@fangjia.com'), SongYingXia('13621626685', 'wangwenyan@fangjia.com'),
YuKai('13621626685', 'duxiaoyu@fangjia.com'), YuKai('13621626685', 'wangwenyan@fangjia.com'),
RuanYaDong('13621626685', 'duxiaoyu@fangjia.com'), RuanYaDong('13621626685', 'wangwenyan@fangjia.com'),
Reda('13621626685', 'duxiaoyu@fangjia.com'), Reda('13621626685', 'wangwenyan@fangjia.com'),
Tomzhao('13621626685','duxiaoyu@fangjia.com'), Tomzhao('13621626685','wangwenyan@fangjia.com'),
YuHai('13621626685','duxiaoyu@fangjia.com'), YuHai('13621626685','wangwenyan@fangjia.com'),
Shibb('13621626685','duxiaoyu@fangjia.com'), Shibb('13621626685','wangwenyan@fangjia.com'),
HeLvYuan('13621626685','duxiaoyu@fangjia.com'), HeLvYuan('13621626685','wangwenyan@fangjia.com'),
WangYuHui('13621626685','duxiaoyu@fangjia.com'), WangYuHui('13621626685','wangwenyan@fangjia.com'),
LinXiaoShuai('15801810393','linxiaoshuai@fangjia.com'), LinXiaoShuai('15801810393','wangwenyan@fangjia.com'),
FangRui('17717680589','fangrui@fangjia.com'), FangRui('17717680589','wangwenyan@fangjia.com'),
//DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'), DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
//ÔÚְͬÊ //ÔÚְͬÊÂ
DuXiaoYu('13916076378', 'wangwenyan@fangjia.com'), LiXue('13311649646','wangwenyan@fangjia.com'),
LiXue('13311649646','lixue@fangjia.com'),
WangWenYan('13916076378','wangwenyan@fangjia.com'), WangWenYan('13916076378','wangwenyan@fangjia.com'),
GaoJuan('15021384995','gaojuan@fangjia.com'), GaoJuan('15021384995','gaojuan@fangjia.com'),
Hubingliang('15179856208','hubingliang@126.com'), Hubingliang('15179856208','wangwenyan@fangjia.com'),
ZhouTongYun('13585834921','zhoutongyun@fangjia.com'), ZhouTongYun('13585834921','wangwenyan@fangjia.com'),
YinJiHuan('18217642009','yinjihuan@fangjia.com'), YinJiHuan('18217642009','wangwenyan@fangjia.com'),
Xuhe('13862416422','xuhe@fangjia.com'), Xuhe('13862416422','wangwenyan@fangjia.com'),
ZongZhiMin('18260046486','zongzhimin@fangjia.com'), ZongZhiMin('18260046486','wangwenyan@fangjia.com'),
RaoYanHui('18516539752','raoyanhui@fangjia.com'), RaoYanHui('18516539752','wangwenyan@fangjia.com'),
BaYi('15300424133','bayi@fangjia.com'), BaYi('15300424133','wangwenyan@fangjia.com'),
WeiLei('18101751850','weilei@fangjia.com'), WeiLei('18101751850','wangwenyan@fangjia.com'),
JiangYaHui('13684913780','jiangyahui@fangjia.com'), JiangYaHui('13684913780','wangwenyan@fangjia.com'),
ShenJian('13764369683', 'shenjian@fangjia.com'), ShenJian('13764369683', 'shenjian@fangjia.com')
MaJinTao('13607600062','majintao@fangjia.com')
private final String mobile; private final String mobile;
private final String email; private final String email;
@@ -124,13 +124,13 @@ public class ManualOperationDataStat extends BaseBatch{
println(content) println(content)
sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content) sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
} }
} }
@@ -29,7 +29,7 @@ class HouseChangingMonitorBatch extends BaseBatch {
DBCollection COLLECTION = MongoUtils.getCollectionByName("fangjia_log", "house_changing_monitor", "log.mongo.db.fangjia.com") DBCollection COLLECTION = MongoUtils.getCollectionByName("fangjia_log", "house_changing_monitor", "log.mongo.db.fangjia.com")
Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
def EMAILS = [ def EMAILS = [
'duxiaoyu@fangjia.com' 'gaojuan@fangjia.com'
] ]
public static void main(String[] args){ public static void main(String[] args){
@@ -39,7 +39,7 @@ class WebAverageLabor extends BaseBatch{
public static final def CITIES = ["上海", "成都", "北京", "广州", "深圳"] public static final def CITIES = ["上海", "成都", "北京", "广州", "深圳"]
public static final def EMAILS = ["duxiaoyu@fangjia.com"] public static final def EMAILS = ["wangwenyan@fangjia.com"]
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
@@ -24,16 +24,16 @@ class DistrictDataCoverBatch extends BaseBatch {
public DSCollection dsCollection = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public DSCollection dsCollection = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "district_facilitys", "mongo.db.fangjia.com") public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "district_facilitys", "mongo.db.fangjia.com")
public static final def EMAILS =[ public static final def EMAILS =[
"duxiaoyu@fangjia.com", // "duxiaoyu@fangjia.com",
"ruanyadong@fangjia.com", // "ruanyadong@fangjia.com",
"xiamin@fangjia.com", // "xiamin@fangjia.com",
"zhutianli@fangjia.com", // "zhutianli@fangjia.com",
"joe@fangjia.com", // "joe@fangjia.com",
"yukai@fangjia.com", // "yukai@fangjia.com",
"jiangxiaochun@fangjia.com", // "jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com", "gaojuan@fangjia.com",
"liuxingwang@fangjia.com", // "liuxingwang@fangjia.com",
"xiaohai@fangjia.com", // "xiaohai@fangjia.com",
] ]
static def FIELDS_MAP = [ static def FIELDS_MAP = [
@@ -33,10 +33,7 @@ class DistrictFieldCoverBatch extends BaseBatch {
final static String coverDomain = "mongo.db.fangjia.com" final static String coverDomain = "mongo.db.fangjia.com"
final static String datastoreDomain = "seaweed.db.fangjia.com" final static String datastoreDomain = "seaweed.db.fangjia.com"
final static def emails = [ final static def emails = [
"duxiaoyu@fangjia.com", "gaojuan@fangjia.com"
"joe@fangjia.com", "yukai@fangjia.com", "yanzhou@fangjia.com",
"gaojuan@fangjia.com", "yuanyadong@fangjia.com", "liuxingwang@fangjia.com",
"xiamin@fangjia.com", "zhutianli@fangjia.com"
] ]
// final static def emails = [ // final static def emails = [
@@ -24,16 +24,16 @@ class LoudongDataCoverBatch extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com") public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
public static final def EMAILS = [ public static final def EMAILS = [
"duxiaoyu@fangjia.com", // "duxiaoyu@fangjia.com",
// "ruanyadong@fangjia.com", // "ruanyadong@fangjia.com",
// "xiamin@fangjia.com", // "xiamin@fangjia.com",
// "zhutianli@fangjia.com", // "zhutianli@fangjia.com",
"joe@fangjia.com", // "joe@fangjia.com",
"yukai@fangjia.com", // "yukai@fangjia.com",
"jiangxiaochun@fangjia.com", // "jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com", "gaojuan@fangjia.com"
"shijing@fangjia.com", // "shijing@fangjia.com",
"xiaohai@fangjia.com", // "xiaohai@fangjia.com",
] ]
static def FIELDS_MAP = [ static def FIELDS_MAP = [
@@ -27,8 +27,8 @@ class ApiDailyCount extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
def static emails = [ def static emails = [
'xuqi@fangjia.com', 'wangwenyan@fangjia.com',
'zongzhimin@fangjia.com' 'gaojuan@fangjia.com'
] ]
static void main(String[] args){ static void main(String[] args){
@@ -1,8 +1,10 @@
package com.fangjia.script.district package com.fangjia.script.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCursor import com.mongodb.DBCursor
import org.bson.types.ObjectId
import java.util.List import java.util.List
@@ -40,6 +42,19 @@ class DistrictHouseBatch extends BaseBatch {
@Override @Override
public void process(Object args) { public void process(Object args) {
// DBObject q = new BasicDBObject();
// q.put("_id", new ObjectId("6501174f6f5638447b264189"));
// def obj = CommonDatautils.getHouseDataStoreConnection().findOne(q)
// obj.put("c_date", new Date())
// CommonDatautils.getHouseDataStoreConnection().save(obj)
// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","新城帝景南区_武进_常州")).each{
// println(it)
// }
def cityList = cityList(); def cityList = cityList();
int c = 0 int c = 0
@@ -52,9 +67,28 @@ class DistrictHouseBatch extends BaseBatch {
Map<String, Long> rentCountMap = facetDistrictInHouse("rent", city); Map<String, Long> rentCountMap = facetDistrictInHouse("rent", city);
println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()) println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
for(DBObject district : districts) { for(DBObject district : districts) {
String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city") def names = []
long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 names << district.name
long rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 if(district.alias && districts.size()>0){
for(def name : district.alias){
if(!names.contains(name)){
names << name
}
}
}
long sellCount = 0
long rentCount = 0
for(def name : names){
String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
if(!sellCount){
sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0
}
if(!rentCount){
rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0
}
}
saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn) saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn)
} }
@@ -75,6 +109,7 @@ class DistrictHouseBatch extends BaseBatch {
sellQuery.put('$facet', facet) sellQuery.put('$facet', facet)
sellQuery.put('city', city) sellQuery.put('city', city)
sellQuery.put('cat',cat) sellQuery.put('cat',cat)
// sellQuery.put("district_name","新城帝景南区")
// sellQuery.put('visible',0) // sellQuery.put('visible',0)
DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0) DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0)
List<FacetResult> sellFacetResult = sellDSCursor.getFacetResults(); List<FacetResult> sellFacetResult = sellDSCursor.getFacetResults();
@@ -109,13 +144,14 @@ class DistrictHouseBatch extends BaseBatch {
fields.put("city", 1) fields.put("city", 1)
fields.put("region", 1) fields.put("region", 1)
fields.put("name", 1) fields.put("name", 1)
fields.put("alias", 1)
def ret = [] def ret = []
DBObject q = new BasicDBObject() DBObject q = new BasicDBObject()
q.put('city', city) q.put('city', city)
q.put('cat', 'district') q.put('cat', 'district')
q.put('visible', 0) q.put('visible', 0)
// q.put("name","新城帝景")
DSCursor cur = SEAWEED.find(q, fields).batchSize(2000) DSCursor cur = SEAWEED.find(q, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator() Iterator<DBObject> iter = cur.iterator()
@@ -29,9 +29,7 @@ class EvaluationFieldProcent extends BaseBatch{
public static final DBCollection EVALUA_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com") public static final DBCollection EVALUA_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com")
public static final DBCollection EVALUA_COVER_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation_field_coverage", "mongo.db.fangjia.com") public static final DBCollection EVALUA_COVER_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation_field_coverage", "mongo.db.fangjia.com")
final static def emails = ["xiaohai@fangjia.com","jiangxiaochun@fangjia.com","limingmin@fangjia.com","duxiaoyu@fangjia.com", final static def emails = [ "gaojuan@fangjia.com"]
"joe@fangjia.com", "yukai@fangjia.com", "gaojuan@fangjia.com", "yuanyadong@fangjia.com",
"xiamin@fangjia.com", "zhutianli@fangjia.com"]
@@ -30,7 +30,7 @@ class FjtAnalyCrawlerIp extends BaseBatch{
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection addressMatchLogColl = MongoUtils.getCollectionByName2("fangjia", "address_match_log", lineUrl) public static final DBCollection addressMatchLogColl = MongoUtils.getCollectionByName2("fangjia", "address_match_log", lineUrl)
final static def emails = ["gaojuan@fangjia.com","jiangxiaochun@fangjia.com","duxiaoyu@fangjia.com","wangwenyan@fangjia.com","luxuping@fangjia.com","yaojiayi@fangjia.com"] final static def emails = ["gaojuan@fangjia.com"]
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
static main(args) { static main(args) {
@@ -82,7 +82,7 @@ class LngLatEachWeek extends BaseBatch {
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>" href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href) jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href)
jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href) jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href)
jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href) // jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href)
} }
public static Map<String, Object> getLngLatByAddress(String city, String address){ public static Map<String, Object> getLngLatByAddress(String city, String address){
@@ -52,9 +52,9 @@ class YouDaDataParser {
static main(args) { static main(args) {
def task = new YouDaDataParser(); def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo // task.newHouseparse();//友达一手房存入mongo
task.esfParser("D:\\数据\\友达\\2024年1月\\查询1.DBF");//友达二手房存入mongo // task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es // task.newhouseToSaveEs();//新房保存到es
// task.esfhouseToSaveEs();//二手房保存到es task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
} }
@@ -18,7 +18,44 @@ DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia",
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
def filePath = "D:\\数据\\光大\\全量.xlsx" def filePath = "D:\\数据\\南京\\南京银行测试-新跑.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
def communityObj = null
if(row['E']){
communityObj = findCommunityTown(community_CONN, row['C'], row['D'], row['E'])
}
if(communityObj){
row['M'] = communityObj.province_code
row['N'] = communityObj.city_code
row['O'] = communityObj.area_code
row['P'] = communityObj.street_code
}
if(row['D']){
if(row['D'].toString().endsWith("区")){
row['F'] = "非农"
}else{
if(communityObj){
if(communityObj.has_gov && communityObj.has_gov=="是"){
row['I'] = "是"
row['F'] = "非农"
}else{
row['I'] = "否"
if(row['E']){
def townObj = isTown(row['C'], row['D'], row['E'])
if(townObj){
row['J'] = "是"
row['F'] = "非农"
}else{
row['F'] = "农"
}
}
}
}
}
}
});
//def filePath = "D:\\数据\\光大\\全量.xlsx"
//ExcelUtils.modify(filePath, { row -> //ExcelUtils.modify(filePath, { row ->
// if(!row['O']){ // if(!row['O']){
// def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A']) // def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A'])
@@ -62,19 +99,19 @@ def filePath = "D:\\
// } // }
// } // }
def retMap = SignatureUtils.getAliAddressStrc(row['A']) // def retMap = SignatureUtils.getAliAddressStrc(row['A'])
println row['A'] +" " + retMap.toString() // println row['A'] +" " + retMap.toString()
if(retMap){ // if(retMap){
row['Y'] = retMap.prov // row['Y'] = retMap.prov
row['Z'] = retMap.city // row['Z'] = retMap.city
row['AA'] = retMap.district // row['AA'] = retMap.district
row['AB'] = retMap.town // row['AB'] = retMap.town
row['AC'] = retMap.devzone // row['AC'] = retMap.devzone
row['AD'] = retMap.road // row['AD'] = retMap.road
row['AE'] = retMap.roadno // row['AE'] = retMap.roadno
row['AF'] = retMap.poi // row['AF'] = retMap.poi
row['AG'] = retMap.subpoi // row['AG'] = retMap.subpoi
} // }
// }) // })
@@ -66,8 +66,10 @@ class GaoDePoiDataProcess extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
// println ProvinceData.getProvince("成都") // println ProvinceData.getProvince("成都")
synchMapPoi(); // synchMapPoi();
// synchCompany()
synchCompany()
// poiCatProcess() // poiCatProcess()
// addressAndCatProcess() // addressAndCatProcess()
// synchNetwork() // synchNetwork()
@@ -99,10 +101,10 @@ class GaoDePoiDataProcess extends BaseBatch{
def synchCompany(){ def synchCompany(){
fullCityMap = CommonDatautils.queryFullNameCity() fullCityMap = CommonDatautils.queryFullNameCity()
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市", //"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
def citys = ["湖州市"] def citys = ["杭州","宁波","温州","嘉兴","台州","绍兴","金华","舟山","衢州","丽水"]
for(def sampleCity : citys){ for(def sampleCity : citys){
def cityObj = fullCityMap.get("湖州"); def cityObj = fullCityMap.get(sampleCity);
def city = null def city = null
if(cityObj){ if(cityObj){
city = cityObj.city city = cityObj.city
@@ -112,6 +114,7 @@ class GaoDePoiDataProcess extends BaseBatch{
println city+ " 加载公司数据:"+companyDatas.size() println city+ " 加载公司数据:"+companyDatas.size()
def rows = running(city, companyDatas, "company", townMap) def rows = running(city, companyDatas, "company", townMap)
save(city, rows) save(city, rows)
// saveToMG(city, rows)
println "完成" println "完成"
} }
} }
@@ -182,7 +185,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def createCompanyData(map, townMap){ def createCompanyData(map, townMap){
def retMap = [:] def retMap = [:]
retMap._id = map._id retMap.data_id = map._id.toString()
retMap.fjw_province = map.province_name retMap.fjw_province = map.province_name
retMap.fjw_city = map.city_name retMap.fjw_city = map.city_name
@@ -45,10 +45,10 @@ class PdsUseStatisticBatch extends BaseBatch{
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [ def static emails = [
"duweiliang@fangjia.com", // "duweiliang@fangjia.com",
"yaojingmeng@fangjia.com", // "yaojingmeng@fangjia.com",
"xuqi@fangjia.com", // "xuqi@fangjia.com",
"wangpeili@fangjia.com" "gaojuan@fangjia.com"
] ]
static main(args) { static main(args) {
@@ -163,9 +163,9 @@ class AnalyticsApiLogPerformanceBatch extends BaseBatch {
} }
static void sendEmail(String sub, String msg) { static void sendEmail(String sub, String msg) {
jms.sendMail("duxiaoyu@fangjia.com", sub, msg) // jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
jms.sendMail("wangwenyan@fangjia.com", sub, msg) jms.sendMail("wangwenyan@fangjia.com", sub, msg)
jms.sendMail("helvyuan@fangjia.com", sub, msg) // jms.sendMail("helvyuan@fangjia.com", sub, msg)
jms.sendMail("gaojuan@fangjia.com", sub, msg) jms.sendMail("gaojuan@fangjia.com", sub, msg)
} }
@@ -32,11 +32,11 @@ class CommonLogAnaBatch extends BaseBatch {
def static bjEvaReport = []; def static bjEvaReport = [];
public static final def EMAILS = [ public static final def EMAILS = [
"jiangxiaochun@fangjia.com", "jiangxiaochun@fangjia.com"
"limingmin@fangjia.com", // "limingmin@fangjia.com",
"xiaohai@fangjia.com", // "xiaohai@fangjia.com",
"luxuping@fangjia.com", // "luxuping@fangjia.com",
"yukai@fangjia.com" // "yukai@fangjia.com"
]; ];
static main(args) { static main(args) {
@@ -26,12 +26,12 @@ class DaikuanLogAnaBatch extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public static final def EMAILS = [ public static final def EMAILS = [
"jiangxiaochun@fangjia.com", "jiangxiaochun@fangjia.com"
"limingmin@fangjia.com", // "limingmin@fangjia.com",
"zhongjingyi@fangjia.com", // "zhongjingyi@fangjia.com",
"xiaohai@fangjia.com", // "xiaohai@fangjia.com",
"luxuping@fangjia.com", // "luxuping@fangjia.com",
"yukai@fangjia.com" // "yukai@fangjia.com"
]; ];
static main(args) { static main(args) {
@@ -148,14 +148,14 @@ class ReadBkoffSafeLogBatch extends BaseBatch{
} }
static void sendEmail(String sub, String msg) { static void sendEmail(String sub, String msg) {
jms.sendMail("duxiaoyu@fangjia.com", sub, msg) // jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
jms.sendMail("joe@fangjia.com", sub, msg) // jms.sendMail("joe@fangjia.com", sub, msg)
jms.sendMail("xiaohai@fangjia.com", sub, msg) // jms.sendMail("xiaohai@fangjia.com", sub, msg)
jms.sendMail("caoyuyan@fangjia.com", sub, msg) jms.sendMail("caoyuyan@fangjia.com", sub, msg)
jms.sendMail("wangchencen@fangjia.com", sub, msg) // jms.sendMail("wangchencen@fangjia.com", sub, msg)
jms.sendMail("yaojingmeng@fangjia.com", sub, msg) // jms.sendMail("yaojingmeng@fangjia.com", sub, msg)
jms.sendMail("chichilook@qq.com", sub, msg) // jms.sendMail("chichilook@qq.com", sub, msg)
jms.sendMail("sunjian@fangjia.com", sub, msg) // jms.sendMail("sunjian@fangjia.com", sub, msg)
} }
static public String getCatById(String id) { static public String getCatById(String id) {
@@ -0,0 +1,72 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.ExcelUtils
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count = count + 1
if (count % 10 == 0) {
println("已处理:" + count)
}
sleep(50)
def address = items['A']
def map = SignatureUtils.getAliAddressStrc(address)
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
if ("prov" in map) {
def province = map.get("prov")
items['D'] = province
}
if ("city" in map) {
def city = map.get("city")
items['H'] = city
}
if ("district" in map) {
def district = map.get("district")
items['L'] = district
}
if ("town" in map) {
def town = map.get("town")
items['P'] = town
}
if ("community" in map) {
def community = map.get("community")
items['T'] = community
}
// if ("road" in map) {
// def road = map.get("road")
// items['G'] = road
//
// }
// if ("poi" in map) {
//
// def poi = map.get("poi")
// items['H'] = poi
//
// }
// if ("roomno" in map) {
// def roomno = map.get("roomno")
// items['I'] = roomno
//
// }
// if ("devzone" in map) {
// def devzone = map.get("devzone")
// items['J'] = devzone
//
// }
// if ("roadno" in map) {
// def roadno = map.get("roadno")
// items['K'] = roadno
// }
})
@@ -0,0 +1,76 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.ExcelUtils
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count = count + 1
if (count % 10 == 0) {
println("已处理:" + count)
}
sleep(50)
def address = items['A']
def pro = items['D']
if (pro == null) {
def map = SignatureUtils.getAliAddressStrc(address)
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
if ("prov" in map) {
def province = map.get("prov")
items['D'] = province
}
if ("city" in map) {
def city = map.get("city")
items['H'] = city
}
if ("district" in map) {
def district = map.get("district")
items['L'] = district
}
if ("town" in map) {
def town = map.get("town")
items['P'] = town
}
if ("community" in map) {
def community = map.get("community")
items['T'] = community
}
// if ("road" in map) {
// def road = map.get("road")
// items['G'] = road
//
// }
// if ("poi" in map) {
//
// def poi = map.get("poi")
// items['H'] = poi
//
// }
// if ("roomno" in map) {
// def roomno = map.get("roomno")
// items['I'] = roomno
//
// }
// if ("devzone" in map) {
// def devzone = map.get("devzone")
// items['J'] = devzone
//
// }
// if ("roadno" in map) {
// def roadno = map.get("roadno")
// items['K'] = roadno
// }
}
})
@@ -0,0 +1,44 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据 - 副本.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count += 1
if (count % 10 == 0) {
println("已处理:" + count)
}
String flag = items['X']
String address = items['A']
String tt = items['Y']
if (flag == null&&"地址有效".equals(tt)) {
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
def map = [:]
map.put("address", address)
map.put("complete", "true")
map.put("addressAssessment", "true")
map.put("type", "1")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if (result != null) {
if ("verifyFlag" in result) {
items['X'] = result.get("verifyFlag")
}
}
}
})
@@ -0,0 +1,121 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\ÄϾ©ÒøÐвâÊÔ.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count+=1
if (count%10==0){
println("ÒÑ´¦Àí£º"+count)
}
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
String address = items['A']
def map = [:]
map.put("address", address)
map.put("complete", "true")
map.put("addressAssessment", "true")
map.put("type", "2")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if ("province" in result) {
items['B'] = result.get("province")
}
if ("provinceCode" in result) {
items['C'] = result.get("provinceCode")
}
if ("city" in result){
items['D'] = result.get("city")
}
if("cityCode" in result){
items['E'] = result.get("cityCode")
}
if ("district" in result){
items['F'] = result.get("district")
}
if ("districtCode" in result){
items['G'] = result.get("districtCode")
}
if ("town" in result){
items['H'] = result.get("town")
}
if ("townCode" in result){
items['I'] = result.get("townCode")
}
if ("committee" in result){
items['J'] = result.get("committee")
}
if ("committeeCode" in result){
items['K'] = result.get("committeeCode")
}
if ("typeCode" in result){
items['L'] = result.get("typeCode")
}
if ("typeName"){
items['M'] = result.get("typeName")
}
if ("verifyFlag"){
items['N'] = result.get("verifyFlag")
}
if ("addrVerify"){
items['O'] = result.get("addrVerify")
}
if ("countyLevelCity"){
items['P'] = result.get("countyLevelCity")
}
if ("countyGov"){
items['Q'] = result.get("countyGov")
}
// if ("communitySource"){
// items['R'] = result.get("communitySource")
// }
//
// if ("administrativeTowns" in result){
// items['T'] = result.get("administrativeTowns")
// }
// if ("devDistrict" in result){
// items['V'] = result.get("devDistrict")
// }
// if ("devDistrictName" in result){
// items['W'] = result.get("devDistrictName")
// }
// if ("splitCommittee" in result){
// items['X'] = result.get("splitCommittee")
// }
// if ("splitStreet" in result){
// items['Y'] = result.get("splitStreet")
// }
// if ("poiName" in result){
// items['Z'] = result.get("poiName")
// }
// if ("splitStreetNum" in result){
// items['AA'] = result.get("splitStreetNum")
// }
// if ("dev" in result){
// items['AB'] = result.get("dev")
// }
})
@@ -26,8 +26,8 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] // def esSources = ["搜房":"搜房"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产","幸福里APP":"幸福里"] def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
static main(args) { static main(args) {
runBatch(new SynchDistrictPrice(), args); runBatch(new SynchDistrictPrice(), args);
@@ -36,10 +36,10 @@ class DBHelper {
static final DB offline61PoseidonDb = new Mongo("192.168.0.61", 27037).getDB("poseidon"); static final DB offline61PoseidonDb = new Mongo("192.168.0.61", 27037).getDB("poseidon");
static final online42GoojiaFangjiaDbConn = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?useUnicode=true&characterEncoding=utf-8", "sa", "G00jia", "com.mysql.jdbc.Driver"); static final online42GoojiaFangjiaDbConn = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?useUnicode=true&characterEncoding=utf-8", "sa", "G00jia", "com.mysql.jdbc.Driver");
static final offline89GoojiaFangjiaStatDbConn = Sql.newInstance("jdbc:mysql://192.168.0.89:3306/goojia_fangjia_stat?useUnicode=true&characterEncoding=utf-8", "root", "123456", "com.mysql.jdbc.Driver"); static final offline89GoojiaFangjiaStatDbConn = Sql.newInstance("jdbc:mysql://192.168.0.89:3306/goojia_fangjia_stat?useUnicode=true&characterEncoding=utf-8", "root", "123456", "com.mysql.jdbc.Driver");
static final emailsGroup0 = ['gaojuan@fangjia.com','yukai@fangjia.com','joe@fangjia.com','duxiaoyu@fangjia.com','tangye@fangjia.com','ruanyadong@fangjia.com','xiamin@fangjia.com','zhutianli@fangjia.com']; static final emailsGroup0 = ['gaojuan@fangjia.com'];
static final emailsFangjiaWang = ['all.list@fangjia.com']; static final emailsFangjiaWang = ['all.list@fangjia.com'];
static final mobilesYanzhou = ['13621626685']; static final mobilesYanzhou = ['13621626685'];
static final emailsYanzhou = ['duxiaoyu@fangjia.com']; static final emailsYanzhou = ['wangwenyan@fangjia.com'];
static List<String> getCityNames(){ static List<String> getCityNames(){
DBObject dbObject = new BasicDBObject("status",0); DBObject dbObject = new BasicDBObject("status",0);
@@ -454,7 +454,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size()) def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 3 def nThread = 10
println nThread println nThread
ExecutorService executorService = Executors.newFixedThreadPool(3) ExecutorService executorService = Executors.newFixedThreadPool(3)
final AtomicInteger counter = new AtomicInteger(0) final AtomicInteger counter = new AtomicInteger(0)
@@ -23,7 +23,7 @@ public class ComparisonDataStatistics extends com.fangjia.script.BaseBatch{
static final DATE_NUMBER = new Date().format("yyyyMMdd").toInteger() static final DATE_NUMBER = new Date().format("yyyyMMdd").toInteger()
static final DBCollection MESSENGER_COLL = MongoUtils.getCollectionByName("fangjia", "messenger", MongoUtils.FANGJIA_ONLINE_MONGO ,27017) static final DBCollection MESSENGER_COLL = MongoUtils.getCollectionByName("fangjia", "messenger", MongoUtils.FANGJIA_ONLINE_MONGO ,27017)
static final def categories = ["district":"סլ", "office":"д×ÖÂ¥" , "shop":"ÉÌÆÌ"] static final def categories = ["district":"סլ", "office":"д×ÖÂ¥" , "shop":"ÉÌÆÌ"]
def emails = ["wangwenyan@fangjia.com","duxiaoyu@fangjia.com","joe@fangjia.com","caoyuyan@fangjia.com"] def emails = ["wangwenyan@fangjia.com","caoyuyan@fangjia.com"]
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
def static main(args){ def static main(args){
@@ -14,7 +14,7 @@ class EvaluationSourceAnalyze extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public static final DBCollection STAT_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com") public static final DBCollection STAT_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com")
def emails = ["wangwenyan@fangjia.com","duxiaoyu@fangjia.com","joe@fangjia.com","xiaohai@fangjia.com"] def emails = ["wangwenyan@fangjia.com","xiaohai@fangjia.com"]
static main(args) { static main(args) {
new EvaluationSourceAnalyze().process(null); new EvaluationSourceAnalyze().process(null);
@@ -1,7 +1,16 @@
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils import com.fangjia.core.util.Mongo3Utils
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document import org.bson.Document
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
@@ -11,67 +20,43 @@ import java.util.function.Consumer
import static com.mongodb.client.model.Filters.and import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq import static com.mongodb.client.model.Filters.eq
//REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
//TOWNSHIP_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
//TOWNSHIP_CONN.find().each {BasicDBObject township->
// township.area_station = township.central
// township.removeField("central")
// TOWNSHIP_CONN.save(township)
//
//}
//REGION_CONN.find(new BasicDBObject("category", "region")).each {BasicDBObject regionBlock->
// def lng = regionBlock.lng/10000000000l
// def lat = regionBlock.lat/10000000000l
// HttpRequest request = new HttpRequest()
// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8")
// if(resp.result){
// TOWNSHIP_CONN.update(new BasicDBObject("_id", new ObjectId(resp.result.id)), new BasicDBObject('$set', new BasicDBObject('central', 1)))
// println(resp.result)
// }
//}
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
MongoCollection<Document> DISTRICT_STAT = Mongo3Utils.getDbCollection("fangjia", "district_stat_rent")
MongoCollection<Document> DEAL_PRICE = Mongo3Utils.getDbCollection("fangjia", "deal_price")
def data = [["城市", "区域", "小区名", "版块", "装修", "朝向", "室", "厅", "卫", '标题', '总价', '面积', '单价', '发布时间', '来源']] //CommonUtils.findAllCity().each {def city->
cityList.each { def city -> // DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city, null, null, "district"),{BasicDBObject office->
def statMap = [:] // if(!office.township){
DISTRICT_STAT.find(and(eq("city", city), eq("end_month_date", Date.parse("yyyyMMdd", "20190228")))).forEach(new Consumer<Document>() { // def lng = office.lng/10000000000l
@Override // def lat = office.lat/10000000000l
void accept(Document t) { // HttpRequest request = new HttpRequest()
def key = "${t.city}_${t.region}_${t.district_name}_${t.house_type}".toString() // JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8")
statMap[key] = t.avg_price // if(resp.result){
} // println(office._id.toString() + "\t" + office.township + "\t-->\t" + resp.result.streetName)
}) // DbUtils.seaweed.update(new BasicDBObject("_id", office._id), new BasicDBObject('$set', new BasicDBObject('township', resp.result.streetName)))
roots.each { def root -> // }
EsDataSource dataSource = new EsDataSource(); // }
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" // })
dataSource.setUrl(address) //}
Properties connectionProperties = new Properties() ExcelUtils.multiThreadingModify("C:\\Users\\goojia\\Desktop\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{List item->
connectionProperties.put("user", "developer") def lng = item["AB"].toString().split(",")[0].toDouble()
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") def lat = item["AB"].toString().split(",")[1].toDouble()
dataSource.setProperties(connectionProperties) item["AD"] = GeoUtils.calculateDistance(121.526672,31.142637,lng, lat)
Connection connection = dataSource.getConnection() println(item)
Statement statement = connection.createStatement() })
ResultSet rs = statement.executeQuery("""select * from "seawater-2019" where city='${city}' and cat = 'rent' order by p_date""")
int count = 0
while (rs.next()) {
String region = rs.getString("region")
String district_name = rs.getString("district_name")
int room = rs.getInt("room")
def key = "${city}_${region}_${district_name}_${room}".toString()
if (!statMap.containsKey(key)) {
continue
}
int total_price = rs.getInt("total_price")
double area = rs.getDouble("area")
int avg_price = (int) (total_price / area)
if (total_price * 1.3 || total_price < statMap[key] * 0.7) {
continue
}
String block = rs.getString("block")
String name = rs.getString("name")
String fitment = rs.getString("fitment")
String direction = rs.getString("direction")
int hall = rs.getInt("hall")
int toilet = rs.getInt("toilet")
int p_date = rs.getInt("p_date")
if (!block || !name || !fitment || !direction || room == 0 || hall == 0 || toilet == 0) {
continue
}
println(rs.getString('url'))
def item = [city, region, district_name, block, fitment, direction, room, hall, toilet, name, total_price, area, avg_price, p_date, root]
println(item)
data << item
count++
if(count > 10){
break
}
}
}
}
ExcelUtils.write("D:\\建设银行\\租金样本.xlsx", data, "结果")
@@ -3,9 +3,9 @@ package com.fangjia.script.wenyan
public enum Watcher { public enum Watcher {
//在职同事 //在职同事
DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'), DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
WangWenYan('13916076378','wangwenyan@fangjia.com'), WangWenYan('13916076378','wangwenyan@fangjia.com'),
Tomzhao('15021021464','zhaoshumao@fangjia.com'), Tomzhao('15021021464','wangwenyan@fangjia.com'),
GaoJuan('15021384995','gaojuan@fangjia.com'); GaoJuan('15021384995','gaojuan@fangjia.com');
private final String mobile; private final String mobile;
@@ -27,8 +27,7 @@ class ApiDailyCountOfficialAccount extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
def static emails = [ def static emails = [
'xuqi@fangjia.com', 'wangwenyan@fangjia.com'
'zongzhimin@fangjia.com'
] ]
static void main(String[] args){ static void main(String[] args){
@@ -299,6 +299,8 @@ public class CzbankSampleServiceImpl implements CzbankSampleService {
selectedSample.add(querySamples.get(i)); selectedSample.add(querySamples.get(i));
} }
fillOthersInfo(selectedSample); fillOthersInfo(selectedSample);
// 将最终样本进行排序
sortForSample(selectedSample, target);
List<Sample> sampleList = new ArrayList<Sample>(selectedSample.size()); List<Sample> sampleList = new ArrayList<Sample>(selectedSample.size());
for (SampleWrapper sampleWrapper : selectedSample) { for (SampleWrapper sampleWrapper : selectedSample) {
@@ -165,8 +165,11 @@ public class SellTriggerServiceImpl implements TriggerService {
newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city"));
newObj.put("district_name", district.getName()); newObj.put("district_name", district.getName());
} }
}else{
if(object.containsField("city") && object.containsField("district_name") && object.containsField("region")){
newObj.put("districtname_key", object.get("district_name") + "_" + object.get("region") + "_" + object.get("city"));
}
} }
// newObj.put("if_school", 0); // newObj.put("if_school", 0);
// if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) { // if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) {
// List<SchoolKey> skList = districtSchoolMap.get(districtKey); // List<SchoolKey> skList = districtSchoolMap.get(districtKey);