This commit is contained in:
mjt
2024-04-08 23:06:12 +08:00
69 changed files with 2337 additions and 324 deletions
@@ -0,0 +1,71 @@
package com.fangjia.open.controllers;
import com.fangjia.core.util.HttpRequest;
import com.fangjia.open.interceptors.Authority;
import com.fangjia.open.interceptors.Role;
import com.fangjia.open.model.AddressModel;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import java.util.HashMap;
import java.util.Map;
/**
* 光大接口
*/
@Authority(role = Role.USER)
public class GdBankController {
/**
* 涉农识别-新规
*
* @param model
* @return
*/
public Object villageIdentifyAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
params.put("type", StringUtils.isNotBlank(model.getType()) ? model.getType() : null);
params.put("redis", (model.getRadius() != null && model.getRadius() != 0) ? model.getRadius().toString() : null);
params.put("address", model.getAddress());
params.put("showFullValue", "2");
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
} catch (Exception e) {
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("msgCode");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
JSONObject result = json.getJSONObject("result");
if (("是").equals(result.get("countyLevelCity"))){
result.put("prefectureLevelCity", "否");
} else if(("否").equals(result.get("countyLevelCity"))){
result.put("prefectureLevelCity", "是");
}
basicModel.setResult(result);
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
}
@@ -133,7 +133,7 @@
<div class="item-inner"> <div class="item-inner">
<div class="item-title label">产证建筑面积:</div> <div class="item-title label">产证建筑面积:</div>
<div class="item-input"> <div class="item-input">
<input type="number" placeholder="请输入建筑面积" id="houseArea" min="0" max="1000" <input type="number" placeholder="请输入建筑面积" id="houseArea" min="0" max="3000"
name="fjtEvaluation.houseArea" required> name="fjtEvaluation.houseArea" required>
</div> </div>
</div> </div>
+15 -15
View File
@@ -537,21 +537,21 @@
<!-- <scope>compile</scope>--> <!-- <scope>compile</scope>-->
<!-- </dependency>--> <!-- </dependency>-->
<dependency> <!-- <dependency>-->
<groupId>com.aliyun</groupId> <!-- <groupId>com.aliyun</groupId>-->
<artifactId>aliyun-java-sdk-address-purification</artifactId> <!-- <artifactId>aliyun-java-sdk-address-purification</artifactId>-->
<version>1.0.1</version> <!-- <version>1.0.1</version>-->
</dependency> <!-- </dependency>-->
<dependency> <!-- <dependency>-->
<groupId>com.aliyun</groupId> <!-- <groupId>com.aliyun</groupId>-->
<artifactId>aliyun-java-sdk-core</artifactId> <!-- <artifactId>aliyun-java-sdk-core</artifactId>-->
<version>4.5.9</version> <!-- <version>4.5.9</version>-->
</dependency> <!-- </dependency>-->
<dependency> <!-- <dependency>-->
<groupId>com.alibaba</groupId> <!-- <groupId>com.alibaba</groupId>-->
<artifactId>fastjson</artifactId> <!-- <artifactId>fastjson</artifactId>-->
<version>1.2.83</version> <!-- <version>1.2.83</version>-->
</dependency> <!-- </dependency>-->
</dependencies> </dependencies>
@@ -2,38 +2,36 @@ package com.fangjia.script
public enum Watcher { public enum Watcher {
//ÀëְͬÊ //ÀëְͬÊÂ
LiuPeiPei('13621626685', 'duxiaoyu@fangjia.com'), LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'),
YanZhou('13621626685','duxiaoyu@fangjia.com'), YanZhou('13621626685','wangwenyan@fangjia.com'),
HuangXiaoHu('13621626685', 'duxiaoyu@fangjia.com'), HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'),
SongYingXia('13621626685', 'duxiaoyu@fangjia.com'), SongYingXia('13621626685', 'wangwenyan@fangjia.com'),
YuKai('13621626685', 'duxiaoyu@fangjia.com'), YuKai('13621626685', 'wangwenyan@fangjia.com'),
RuanYaDong('13621626685', 'duxiaoyu@fangjia.com'), RuanYaDong('13621626685', 'wangwenyan@fangjia.com'),
Reda('13621626685', 'duxiaoyu@fangjia.com'), Reda('13621626685', 'wangwenyan@fangjia.com'),
Tomzhao('13621626685','duxiaoyu@fangjia.com'), Tomzhao('13621626685','wangwenyan@fangjia.com'),
YuHai('13621626685','duxiaoyu@fangjia.com'), YuHai('13621626685','wangwenyan@fangjia.com'),
Shibb('13621626685','duxiaoyu@fangjia.com'), Shibb('13621626685','wangwenyan@fangjia.com'),
HeLvYuan('13621626685','duxiaoyu@fangjia.com'), HeLvYuan('13621626685','wangwenyan@fangjia.com'),
WangYuHui('13621626685','duxiaoyu@fangjia.com'), WangYuHui('13621626685','wangwenyan@fangjia.com'),
LinXiaoShuai('15801810393','linxiaoshuai@fangjia.com'), LinXiaoShuai('15801810393','wangwenyan@fangjia.com'),
FangRui('17717680589','fangrui@fangjia.com'), FangRui('17717680589','wangwenyan@fangjia.com'),
//DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'), DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
//ÔÚְͬÊ //ÔÚְͬÊÂ
DuXiaoYu('13916076378', 'wangwenyan@fangjia.com'), LiXue('13311649646','wangwenyan@fangjia.com'),
LiXue('13311649646','lixue@fangjia.com'),
WangWenYan('13916076378','wangwenyan@fangjia.com'), WangWenYan('13916076378','wangwenyan@fangjia.com'),
GaoJuan('15021384995','gaojuan@fangjia.com'), GaoJuan('15021384995','gaojuan@fangjia.com'),
Hubingliang('15179856208','hubingliang@126.com'), Hubingliang('15179856208','wangwenyan@fangjia.com'),
ZhouTongYun('13585834921','zhoutongyun@fangjia.com'), ZhouTongYun('13585834921','wangwenyan@fangjia.com'),
YinJiHuan('18217642009','yinjihuan@fangjia.com'), YinJiHuan('18217642009','wangwenyan@fangjia.com'),
Xuhe('13862416422','xuhe@fangjia.com'), Xuhe('13862416422','wangwenyan@fangjia.com'),
ZongZhiMin('18260046486','zongzhimin@fangjia.com'), ZongZhiMin('18260046486','wangwenyan@fangjia.com'),
RaoYanHui('18516539752','raoyanhui@fangjia.com'), RaoYanHui('18516539752','wangwenyan@fangjia.com'),
BaYi('15300424133','bayi@fangjia.com'), BaYi('15300424133','wangwenyan@fangjia.com'),
WeiLei('18101751850','weilei@fangjia.com'), WeiLei('18101751850','wangwenyan@fangjia.com'),
JiangYaHui('13684913780','jiangyahui@fangjia.com'), JiangYaHui('13684913780','wangwenyan@fangjia.com'),
ShenJian('13764369683', 'shenjian@fangjia.com'), ShenJian('13764369683', 'shenjian@fangjia.com')
MaJinTao('13607600062','majintao@fangjia.com')
private final String mobile; private final String mobile;
private final String email; private final String email;
@@ -124,13 +124,13 @@ public class ManualOperationDataStat extends BaseBatch{
println(content) println(content)
sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content) sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content) // sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
} }
} }
@@ -29,7 +29,7 @@ class HouseChangingMonitorBatch extends BaseBatch {
DBCollection COLLECTION = MongoUtils.getCollectionByName("fangjia_log", "house_changing_monitor", "log.mongo.db.fangjia.com") DBCollection COLLECTION = MongoUtils.getCollectionByName("fangjia_log", "house_changing_monitor", "log.mongo.db.fangjia.com")
Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
def EMAILS = [ def EMAILS = [
'duxiaoyu@fangjia.com' 'gaojuan@fangjia.com'
] ]
public static void main(String[] args){ public static void main(String[] args){
@@ -21,6 +21,7 @@ import org.bson.types.ObjectId
/** /**
* 通知计算二手房均价 * 通知计算二手房均价
* 发送平米租金命令
* @author duxiaoyu * @author duxiaoyu
* *
*/ */
@@ -39,7 +39,7 @@ class WebAverageLabor extends BaseBatch{
public static final def CITIES = ["上海", "成都", "北京", "广州", "深圳"] public static final def CITIES = ["上海", "成都", "北京", "广州", "深圳"]
public static final def EMAILS = ["duxiaoyu@fangjia.com"] public static final def EMAILS = ["wangwenyan@fangjia.com"]
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
@@ -24,15 +24,16 @@ class BatchAddPersonGuiPrice {
def run(){ def run(){
def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx" def path = "D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx"
def datas = ExcelUtils.read(path, 0, 1) def datas = ExcelUtils.read(path, 0, 1)
for(def d : datas){ for(def d : datas){
def row = findGuidingPrice(d['B'], d['C'], d['D']) def row = findGuidingPrice(d['B'], d['C'], d['C'])
if(!row){ if(!row){
def obj = CommonDatautils.findDistrictById(d['A']) // def obj = CommonDatautils.findDistrictById(d['A'])
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2) def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
save(d['A'], d['B'], d['C'], d['Q'], obj.estate_type2, d['P'])
//发重算均价命令 //发重算均价命令
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date()) addPrice(d['A'], d['B'], d['C'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),new Date())
} }
} }
} }
@@ -43,13 +44,13 @@ class BatchAddPersonGuiPrice {
q.put("city", city) q.put("city", city)
q.put("region", region) q.put("region", region)
q.put("name", name) q.put("name", name)
q.put("guiding_person" , "付贵生") // q.put("guiding_person" , person)
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd"))) q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2024-03-20", "yyyy-MM-dd")))
def dbo = GUIDING_PRICE_COLL.findOne(q) def dbo = GUIDING_PRICE_COLL.findOne(q)
return dbo return dbo
} }
def save(city, region, name, price, estate_type){ def save(city, region, name, price, estate_type, person){
DBObject obj = new BasicDBObject(); DBObject obj = new BasicDBObject();
obj.put("city", city); obj.put("city", city);
obj.put("region", region); obj.put("region", region);
@@ -62,7 +63,7 @@ class BatchAddPersonGuiPrice {
obj.put("m_date", new Date()); obj.put("m_date", new Date());
obj.put("estate_type" , estate_type); obj.put("estate_type" , estate_type);
obj.put("house_type" , 0) obj.put("house_type" , 0)
obj.put("guiding_person" , "陈杰") obj.put("guiding_person" , person)
GUIDING_PRICE_COLL.save(obj); GUIDING_PRICE_COLL.save(obj);
} }
@@ -387,26 +387,43 @@ class DealPriceDataProcess {
} }
def deleteThirdPartyData(){ def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all // ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
def citys = ["469005"]//CcbCommonUtils.citys_all
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期 def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = [] def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet() def set = new HashSet()
for(def city : citys) { for(def cityid : citys) {
def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期 def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
def total = 0 println cityid +" "+rets.size()
println city +" "+rets.size()
if(rets){ if(rets){
set.add(city) set.add(cityid)
} }
for(def m : rets){ for(def m : rets){
def cityId = cityCodeDatas.get(city) // def cityId = cityCodeDatas.get(city)
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}" // def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql) // sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
def total = 0
pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println cityid +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}else{
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityId}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}" querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql) pgsqlInstance.eachRow(querySql, { r ->
total = r.count
})
if(total>0){
println cityId +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql)
}
}
} }
} }
println set.toString() println set.toString()
@@ -432,8 +449,8 @@ class DealPriceDataProcess {
return rets return rets
} }
def findPeGteThirdParty2(cityName, yearmonth){ def findPeGteThirdParty2(cityid, yearmonth){
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_nm='${cityName}' and gujia_mns_rto>20" def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20"
def rets = [] def rets = []
pgsqlInstance.eachRow(sql,{r-> pgsqlInstance.eachRow(sql,{r->
def map = [:] def map = [:]
@@ -27,4 +27,13 @@ class PgsqlCommUtils {
}); });
return count return count
} }
public static def findDistrictMap(){
def sql = "select urbn_id,gujia_admndiv_id,gujia_admndiv_nm from base.t0831_bigdata_district_gj_h"
def map = [:]
sqlInstance.eachRow(sql,{r->
map.put(r.urbn_id+"#"+r.gujia_admndiv_id, r.gujia_admndiv_nm)
});
return map
}
} }
@@ -69,6 +69,12 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataCityPrice(queryDate) exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate) exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate) exportBigdataCommPrice(queryDate)
// 新增两张数据表
// avm_evaluate_rate_city_gj_h
// avm_evaluate_rate_comm_gj_h
exportEvaluateRateCity(queryDate)
exportEvaluateRateComm(queryDate)
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
// exportCommunityBasePrice(lastMonth) // exportCommunityBasePrice(lastMonth)
// exportCommunityPdescribeGj(lastMonth) // exportCommunityPdescribeGj(lastMonth)
@@ -228,6 +234,79 @@ class ExportDistrictRongHe extends BaseBatch{
CcbFtpUtils.ftpUpload("rh",fileName, curDate) CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "GUIDED_PRICE_GJ 上传Ftp完成......" println "GUIDED_PRICE_GJ 上传Ftp完成......"
} }
// avm_evaluate_rate_city_gj_h gujia_clc_yrmo_val =202402 and gujia_clc_yrmo_val<=202403 gujia_clc_yrmo_val>=to_date('${startDateStr}','yyyy-MM-dd') and gujia_clc_yrmo_val<=to_date('${endDateStr}','yyyy-MM-dd')
def exportEvaluateRateCity(queryDate){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_bld_tp_val
data << r.gujia_bld_tp_val_coef
data << r.gujia_clc_yrmo_val
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "EVALUATE_RATE_CITY_FJW_ADD 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "EVALUATE_RATE_CITY_FJW_ADD 上传Ftp完成......"
}
def exportEvaluateRateComm(queryDate){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' ";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujia_clc_yrmo_val
data << r.basic_price
data << r.area_avg
data << r.unitprc_avg
data << r.comm_type
data << r.area_avg_coef
data << r.basic_price_coef
data << r.buildarea_coef
data << r.type_coef
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){ def exportBigdataCommbuildstruGJ(){
@@ -40,6 +40,8 @@ import java.util.zip.GZIPInputStream
class CommonDatautils { class CommonDatautils {
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
@@ -1175,6 +1177,28 @@ class CommonDatautils {
return null; return null;
} }
public static def findAllStatusDistrict(String city,String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
q.put("region", region);
q.put("name", name);
q.put("cat", cat);
def rets = []
def cursor = SEAWEED_MONGO_COLL.find(q);
if(cursor){
for(def obj : cursor){
rets << obj;
}
}
return rets;
}
public static def findDistrictsByName(String city, String name, String cat){ public static def findDistrictsByName(String city, String name, String cat){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
@@ -1537,6 +1561,32 @@ class CommonDatautils {
} }
static def findAllStatusDistrictByAlias(def city, def region, def name, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
def names = [];
names << name;
BasicDBObject query = new BasicDBObject();
query.putAt(QueryOperators.IN, names);
q.put("alias", query);
def dsCursor = getDataStoreConnection().find(q);
def ret = [];
for(def obj : dsCursor){
ret << obj;
}
return ret;
}
static def findDistrictByAddress(def city, def region, def address, def cat){ static def findDistrictByAddress(def city, def region, def address, def cat){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
if(region){ if(region){
@@ -3764,6 +3814,135 @@ class CommonDatautils {
return rets; return rets;
} }
static def findHouses2(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(true);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city))
.must(QueryBuilders.termQuery("cat", cat)).must(date_range).must(area_range)
if(region){
queryBuilder.must(QueryBuilders.termQuery("region", region))
}
if(name || alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
def avgPrices = []
def areas = []
def totalPrices = []
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(cat == "rent"){
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
avgPrices << seawater.avg_price
areas << seawater.area
totalPrices << seawater.total_price
}
}
def retMap = [:]
if(areas){
retMap.avg_price = avgPrices
retMap.area = areas
retMap.total_price = totalPrices
}
return retMap;
}
/**
* 是否是别墅
* @return
*/
public static boolean isVilla(Double area, String name, Integer height, Integer room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
}else if(height != null){
if (height <= 3) {
if (area > 200){
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
static def parseFloor(def floorDesc){ static def parseFloor(def floorDesc){
if(floorDesc){ if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "") floorDesc = floorDesc.replaceAll("[\\s ]+", "")
@@ -3919,6 +4098,29 @@ class CommonDatautils {
return avg_price return avg_price
} }
static def findMatchTrades(city, region, name, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
/** /**
@@ -3993,6 +4195,32 @@ class CommonDatautils {
return total; return total;
} }
static def findLogs(user_id, requestUri, startDate, endDate){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", user_id)).must(QueryBuilders.termQuery("request_uri", requestUri))
if(startDate){
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
queryBuilder.must(date_range_cdate);
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
if(map.parameter_map){
def pMap = map.parameter_map;
if((pMap.keyword && pMap.keyword.contains("紫金山蘭园")) || (pMap.name && pMap.name.contains("紫金山蘭园"))){
println map
}
}
}
}
return null;
}
static def findNewAmapDetailBySql(city){ static def findNewAmapDetailBySql(city){
String sql = "select * from amap_poi where cityname='${city}' and status=0" String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = [] def rets = []
@@ -4219,6 +4447,23 @@ class CommonDatautils {
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price) dataMap.put("beike_esf_price", db.beike_esf_price)
} }
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("xingfuli_esf_price") && db.xingfuli_esf_price && db.xingfuli_esf_price>3000){
dataMap.put("xingfuli_esf_price", db.xingfuli_esf_price)
}
if(!dataMap.containsKey("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
} }
return map return map
} }
@@ -4337,6 +4582,23 @@ class CommonDatautils {
} }
static def findHouseAvgPrice(city, region , start, end){
def prices = []
def sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date > ${start} and p_date<${end} and (root='lianjia.com' or root='ke.com')"
if(region){
sql = sql + " and region='${region}'"
}
ElasticUtils.executeSqlQuery(sql, {
prices << it.avg_price
})
if(prices && prices.size()>3){
def avg_price = (int)(prices.sum()/prices.size())
println region + " "+ prices.size() +" "+avg_price +" "+start+" "+end
return avg_price
}
return null
}
} }
@@ -24,16 +24,16 @@ class DistrictDataCoverBatch extends BaseBatch {
public DSCollection dsCollection = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed") public DSCollection dsCollection = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "district_facilitys", "mongo.db.fangjia.com") public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "district_facilitys", "mongo.db.fangjia.com")
public static final def EMAILS =[ public static final def EMAILS =[
"duxiaoyu@fangjia.com", // "duxiaoyu@fangjia.com",
"ruanyadong@fangjia.com", // "ruanyadong@fangjia.com",
"xiamin@fangjia.com", // "xiamin@fangjia.com",
"zhutianli@fangjia.com", // "zhutianli@fangjia.com",
"joe@fangjia.com", // "joe@fangjia.com",
"yukai@fangjia.com", // "yukai@fangjia.com",
"jiangxiaochun@fangjia.com", // "jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com", "gaojuan@fangjia.com",
"liuxingwang@fangjia.com", // "liuxingwang@fangjia.com",
"xiaohai@fangjia.com", // "xiaohai@fangjia.com",
] ]
static def FIELDS_MAP = [ static def FIELDS_MAP = [
@@ -33,10 +33,7 @@ class DistrictFieldCoverBatch extends BaseBatch {
final static String coverDomain = "mongo.db.fangjia.com" final static String coverDomain = "mongo.db.fangjia.com"
final static String datastoreDomain = "seaweed.db.fangjia.com" final static String datastoreDomain = "seaweed.db.fangjia.com"
final static def emails = [ final static def emails = [
"duxiaoyu@fangjia.com", "gaojuan@fangjia.com"
"joe@fangjia.com", "yukai@fangjia.com", "yanzhou@fangjia.com",
"gaojuan@fangjia.com", "yuanyadong@fangjia.com", "liuxingwang@fangjia.com",
"xiamin@fangjia.com", "zhutianli@fangjia.com"
] ]
// final static def emails = [ // final static def emails = [
@@ -24,16 +24,16 @@ class LoudongDataCoverBatch extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com") public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
public static final def EMAILS = [ public static final def EMAILS = [
"duxiaoyu@fangjia.com", // "duxiaoyu@fangjia.com",
// "ruanyadong@fangjia.com", // "ruanyadong@fangjia.com",
// "xiamin@fangjia.com", // "xiamin@fangjia.com",
// "zhutianli@fangjia.com", // "zhutianli@fangjia.com",
"joe@fangjia.com", // "joe@fangjia.com",
"yukai@fangjia.com", // "yukai@fangjia.com",
"jiangxiaochun@fangjia.com", // "jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com", "gaojuan@fangjia.com"
"shijing@fangjia.com", // "shijing@fangjia.com",
"xiaohai@fangjia.com", // "xiaohai@fangjia.com",
] ]
static def FIELDS_MAP = [ static def FIELDS_MAP = [
@@ -27,8 +27,8 @@ class ApiDailyCount extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
def static emails = [ def static emails = [
'xuqi@fangjia.com', 'wangwenyan@fangjia.com',
'zongzhimin@fangjia.com' 'gaojuan@fangjia.com'
] ]
static void main(String[] args){ static void main(String[] args){
@@ -1,8 +1,10 @@
package com.fangjia.script.district package com.fangjia.script.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCursor import com.mongodb.DBCursor
import org.bson.types.ObjectId
import java.util.List import java.util.List
@@ -40,6 +42,24 @@ class DistrictHouseBatch extends BaseBatch {
@Override @Override
public void process(Object args) { public void process(Object args) {
// DBObject q = new BasicDBObject();
// q.put("cat", "sell");
// q.put("city","常州")
// q.put("district_name","清河苑一区")
// def dbcursor = CommonDatautils.getHouseDataStoreConnection().find(q)
// for(def obj : dbcursor){
// obj.put("c_date", new Date())
// CommonDatautils.getHouseDataStoreConnection().save(obj)
// }
// CommonDatautils.getHouseDataStoreConnection().find(new BasicDBObject("districtname_key","清河苑一区_溧阳_常州")).each{
// println(it)
// }
def cityList = cityList(); def cityList = cityList();
int c = 0 int c = 0
@@ -47,18 +67,38 @@ class DistrictHouseBatch extends BaseBatch {
c++ c++
try { try {
long sn = Long.parseLong("${new Date().format('yyyyMMddHHmmssSSS')}") long sn = Long.parseLong("${new Date().format('yyyyMMddHHmmssSSS')}")
List<DBObject> districts = findDistrictByCity(city) // List<DBObject> districts = findDistrictByCity(city)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias"])
Map<String, Long> sellCountMap = facetDistrictInHouse("sell", city); Map<String, Long> sellCountMap = facetDistrictInHouse("sell", city);
Map<String, Long> rentCountMap = facetDistrictInHouse("rent", city); Map<String, Long> rentCountMap = facetDistrictInHouse("rent", city);
println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date()) println c + "\t" + city + "\t" + "\tsn:" + sn + "\t" + districts.size() + "\t" + DateUtil.formatToTightYYYYMMDDhhmmss(new Date())
for(DBObject district : districts) { for(DBObject district : districts) {
String key = district.get("name").toString().trim() + "_" + district.get("region") + "_" + district.get("city") def names = []
long sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0 names << district.name
long rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0 if(district.alias && districts.alias.size()>0){
for(def name : district.alias){
if(!names.contains(name)){
names << name
}
}
}
long sellCount = 0
long rentCount = 0
for(def name : names){
String key = name.toString().trim() + "_" + district.get("region") + "_" + district.get("city")
if(!sellCount){
sellCount = sellCountMap.get(key) ? sellCountMap.get(key) : 0
}
if(!rentCount){
rentCount = rentCountMap.get(key) ? rentCountMap.get(key) : 0
}
}
saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn) saveToDistrictHouse(district.get('city'), district.get("region"), district.get("name"), sellCount, rentCount, sn)
} }
HOUSE_COLL.remove(new BasicDBObject("city", city).append('sn', new BasicDBObject('$ne', sn))) HOUSE_COLL.remove(new BasicDBObject("city", city).append('sn', new BasicDBObject('$ne', sn)))
println c + " " + city
}catch (Exception e) { }catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
} }
@@ -75,6 +115,7 @@ class DistrictHouseBatch extends BaseBatch {
sellQuery.put('$facet', facet) sellQuery.put('$facet', facet)
sellQuery.put('city', city) sellQuery.put('city', city)
sellQuery.put('cat',cat) sellQuery.put('cat',cat)
// sellQuery.put("district_name","新城帝景南区")
// sellQuery.put('visible',0) // sellQuery.put('visible',0)
DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0) DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0)
List<FacetResult> sellFacetResult = sellDSCursor.getFacetResults(); List<FacetResult> sellFacetResult = sellDSCursor.getFacetResults();
@@ -109,13 +150,14 @@ class DistrictHouseBatch extends BaseBatch {
fields.put("city", 1) fields.put("city", 1)
fields.put("region", 1) fields.put("region", 1)
fields.put("name", 1) fields.put("name", 1)
fields.put("alias", 1)
def ret = [] def ret = []
DBObject q = new BasicDBObject() DBObject q = new BasicDBObject()
q.put('city', city) q.put('city', city)
q.put('cat', 'district') q.put('cat', 'district')
q.put('visible', 0) q.put('visible', 0)
q.put("name","清河苑")
DSCursor cur = SEAWEED.find(q, fields).batchSize(2000) DSCursor cur = SEAWEED.find(q, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator() Iterator<DBObject> iter = cur.iterator()
@@ -29,9 +29,7 @@ class EvaluationFieldProcent extends BaseBatch{
public static final DBCollection EVALUA_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com") public static final DBCollection EVALUA_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com")
public static final DBCollection EVALUA_COVER_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation_field_coverage", "mongo.db.fangjia.com") public static final DBCollection EVALUA_COVER_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation_field_coverage", "mongo.db.fangjia.com")
final static def emails = ["xiaohai@fangjia.com","jiangxiaochun@fangjia.com","limingmin@fangjia.com","duxiaoyu@fangjia.com", final static def emails = [ "gaojuan@fangjia.com"]
"joe@fangjia.com", "yukai@fangjia.com", "gaojuan@fangjia.com", "yuanyadong@fangjia.com",
"xiamin@fangjia.com", "zhutianli@fangjia.com"]
@@ -30,7 +30,7 @@ class FjtAnalyCrawlerIp extends BaseBatch{
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection addressMatchLogColl = MongoUtils.getCollectionByName2("fangjia", "address_match_log", lineUrl) public static final DBCollection addressMatchLogColl = MongoUtils.getCollectionByName2("fangjia", "address_match_log", lineUrl)
final static def emails = ["gaojuan@fangjia.com","jiangxiaochun@fangjia.com","duxiaoyu@fangjia.com","wangwenyan@fangjia.com","luxuping@fangjia.com","yaojiayi@fangjia.com"] final static def emails = ["gaojuan@fangjia.com"]
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
static main(args) { static main(args) {
@@ -82,7 +82,7 @@ class LngLatEachWeek extends BaseBatch {
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>" href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href) jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href)
jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href) jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href)
jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href) // jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href)
} }
public static Map<String, Object> getLngLatByAddress(String city, String address){ public static Map<String, Object> getLngLatByAddress(String city, String address){
@@ -7,6 +7,7 @@ import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
@@ -74,7 +75,7 @@ class DistrictSellStatBatch extends BaseBatch {
def day = new Date().getDate() def day = new Date().getDate()
def startMDate = DateUtils.truncateDay(new Date()-1) def startMDate = DateUtils.truncateDay(new Date()-7)
def endMDate = DateUtils.truncateDay(new Date()) def endMDate = DateUtils.truncateDay(new Date())
def index = 0 def index = 0
@@ -144,7 +145,8 @@ class DistrictSellStatBatch extends BaseBatch {
void run() { void run() {
try { try {
totalCount.addAndGet(1) totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name def type = dist.estate_type2=="±ðÊû"?"±ðÊû":"Ð¡Çø"
def key = dist.region +"##" + dist.name+"##"+ type
def weekendPriceObj = weekendPriceMap.get(key) def weekendPriceObj = weekendPriceMap.get(key)
if(weekendPriceObj){ if(weekendPriceObj){
@@ -153,8 +155,13 @@ class DistrictSellStatBatch extends BaseBatch {
def lastYearPriceObj = lastYearPriceMap.get(key) def lastYearPriceObj = lastYearPriceMap.get(key)
double mom = 0d double mom = 0d
if(lastMonthPriceObj) { if(lastMonthPriceObj && lastMonthPriceObj.avg_price>0) {
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4) try{
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
}catch(Exception e){
println key+" ³ö´í................."
}
} }
def curPrice = weekendPriceObj.avg_price def curPrice = weekendPriceObj.avg_price
@@ -192,7 +199,7 @@ class DistrictSellStatBatch extends BaseBatch {
curPrice = 0 curPrice = 0
} }
if(curPriceObj){ if(curPriceObj){
if(curPrice!=weekendPriceObj.avg_price){ if(curPrice!=curPriceObj.avg_price){
def data = new BasicDBObject() def data = new BasicDBObject()
data.put("m_date", new Date()) data.put("m_date", new Date())
data.put("commid", dist._id.toString()) data.put("commid", dist._id.toString())
@@ -263,7 +270,7 @@ class DistrictSellStatBatch extends BaseBatch {
} }
} }
if(!priceMap.containsKey(d.region +"##"+d.district_name)){ if(!priceMap.containsKey(d.region +"##"+d.district_name)){
priceMap.put(d.region +"##"+d.district_name, d) priceMap.put(d.region +"##"+d.district_name+"##"+d.estate_type, d)
} }
} }
return priceMap return priceMap
@@ -355,4 +362,5 @@ class DistrictSellStatBatch extends BaseBatch {
// } // }
} }
@@ -9,6 +9,7 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBCursor import com.mongodb.DBCursor
import com.mongodb.DBObject import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl import com.rabbitmq.client.impl.AMQImpl
import org.apache.commons.io.FileUtils import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId import org.bson.types.ObjectId
@@ -28,10 +29,11 @@ class FzDataResultStat {
static main(args) { static main(args) {
def task = new FzDataResultStat() def task = new FzDataResultStat()
task.priceMarkupRun(); // task.priceMarkupRun();
task.regionPriceTrendRun() // task.regionPriceTrendRun()
//// task.margeExcel() //// task.margeExcel()
// task.saveCsvData() // task.saveCsvData()
task.testNewPrice()
} }
def margeExcel(){ def margeExcel(){
@@ -163,7 +165,7 @@ class FzDataResultStat {
def exportRegion(city, months){ def exportRegion(city, months){
def regions = ["뮬짜","꾑�","憩쉭","쐈갛","쯩棺","낀있"]; def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"];
def datas =[] def datas =[]
def header = ["³ÇÊÐ", "ÇøÓò"] def header = ["³ÇÊÐ", "ÇøÓò"]
for(def m : months){ for(def m : months){
@@ -206,8 +208,12 @@ class FzDataResultStat {
DBObject dbo = cur.next() DBObject dbo = cur.next()
prices << dbo.get("avg_price"); prices << dbo.get("avg_price");
} }
def avg_price = (int)(prices.sum()/prices.size()) if(prices){
return avg_price; def avg_price = (int)(prices.sum()/prices.size())
return avg_price;
}
return null;
} }
@@ -293,4 +299,20 @@ class FzDataResultStat {
} }
return rets return rets
} }
def testNewPrice(){
ExcelUtils.modify("D:\\数据\\福州\\价格比对_20240315 -价格入库.xlsx",{d->
if(d['A']){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region",d['B'])
q.put("name",d['C'])
q.put("month", 202402)
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){
d['S'] = obj.avg_price
}
}
})
}
} }
@@ -1,6 +1,7 @@
package com.fangjia.script.fuzhou package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
@@ -19,7 +20,72 @@ class FzDistrictMatch {
def task = new FzDistrictMatch() def task = new FzDistrictMatch()
// task.run(); // task.run();
// task.addPrice() // task.addPrice()
task.addThirdPrice() // task.addThirdPrice()
// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
task.longyanPrice()
}
def longyanPrice(){
// def dists = CommonDatautils.findAllDistricts("龙岩","district", ["city","region","name","estate_type2"])
// def rets = []
// rets << ["Id", "城市","区域", "名称", "物业类型", "上个月价格toB","上个月价格","最新价格toB","最新价格"]
// for(def d : dists){
// def ret = []
// ret << d._id.toString()
// ret << d.city
// ret << d.region
// ret << d.name
// ret << d.estate_type2
// def obj2 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-03-10"), 1)
// if(obj2){
// ret << obj2.avg_price_2b
// ret << obj2.avg_price
// }
//
// def obj1 = CommonDatautils.queryDistrictPriceByWeekend(d.city, d.region, d.name, DateUtil.parseFormatDate("2024-04-07"), 1)
// if(obj1){
// ret << obj1.avg_price_2b
// ret << obj1.avg_price
// }
// rets << ret
// }
// ExcelUtils.write("D:\\数据\\福州\\龙岩价格对比.xlsx", rets, "龙岩")
addThirdPrice("D:\\数据\\福州\\龙岩价格对比.xlsx")
}
def addThirdPrice2(path){
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = CommonDatautils.findThirdPartyPrice(row['A'], row['B'], row['C'], 202312, 202402)
def priceMap = cityMap.get(row['A']+"##"+row['B']+"##"+row['C'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['J'] = priceMap.get("lianjia_esf_price")
}
if(!row['J'] && priceMap.containsKey("beike_esf_price")){
row['J'] = priceMap.get("beike_esf_price")
}
if(priceMap.containsKey("maitian_esf_price")){
row['K'] = priceMap.get("maitian_esf_price")
}
}
def tradeAvgPrice = CommonDatautils.findMatchTrades(row['A'], row['B'], row['C'], "2023-08-01", "2024-01-31");
if(tradeAvgPrice){
row['L'] = (int)tradeAvgPrice
}
def priceObj = CommonDatautils.findDistrictPrice(row['A'], row['B'], row['C'])
if(priceObj){
row['M'] = priceObj.avg_price_2b
}
})
} }
def addThirdPrice(){ def addThirdPrice(){
@@ -173,4 +239,57 @@ class FzDistrictMatch {
} }
},0) },0)
} }
def addThirdPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['C'] && d['C']=="新罗"){
def cityMap = CommonDatautils.findThirdPartyPrice(d['B'], d['C'], d['D'], 202312, 202403)
def priceMap = cityMap.get(d['B']+"##"+d['C']+"##"+d['D'])
if(priceMap && priceMap.zhugezhaofang_esf_price){
d['J'] = priceMap.zhugezhaofang_esf_price
}
if(priceMap && priceMap.anjuke_esf_price){
d['K'] = priceMap.anjuke_esf_price
}
if(priceMap && priceMap.soufang_esf_price) {
d['L'] = priceMap.soufang_esf_price
}
if(priceMap && priceMap.lianjia_esf_price){
d['M'] = priceMap.lianjia_esf_price
}
if(priceMap && priceMap.cityhouse_esf_price){
d['N'] = priceMap.cityhouse_esf_price
}
if(priceMap && priceMap.zhongyuan_esf_price){
d['O'] = priceMap.zhongyuan_esf_price
}
if(priceMap && priceMap.xingfuli_esf_price){
d['P'] = priceMap.xingfuli_esf_price
}
if(priceMap && priceMap.tuboshi_esf_price){
d['Q'] = priceMap.tuboshi_esf_price
}
if(priceMap && priceMap.get("5i5j_esf_price")){
d['R'] = priceMap.get("5i5j_esf_price")
}
def sellMap = CommonDatautils.findHouses2(d['B'], d['C'], "sell", 20231001, 20240407, d['D'], null, d['E'])
if(sellMap && sellMap.avg_price){
if(sellMap.avg_price.size()>=3){
d['S'] = Math.round(sellMap.avg_price.sum() / sellMap.avg_price.size());
}
}
}
},0)
}
} }
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息 //福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv") def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv")
def count0 = 0 def count0 = 0
def count1 = 0 def count1 = 0
def count2 = 0 def count2 = 0
@@ -20,16 +20,16 @@ class AddAvgpriceBatch {
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static { static {
citys = ["ÐìÖÝ"]//CommonDatautils.findAllCity() citys = ["Ò˲ý"]//CommonDatautils.findAllCity()
println "加载城市完成:"+citys.size() println "加载城市完成:"+citys.size()
} }
static main(args) { static main(args) {
def task = new AddAvgpriceBatch(); def task = new AddAvgpriceBatch();
// task.run1(); task.run1();
// task.test() // task.test()
// task.sendPriceCommand() // task.sendPriceCommand()
task.sendNewDistPrice() // task.sendNewDistPrice()
} }
def sendNewDistPrice(){ def sendNewDistPrice(){
@@ -107,7 +107,7 @@ class AddAvgpriceBatch {
println "------------------------------------------------------------" println "------------------------------------------------------------"
println "${city}小区量:"+districtKeysMap.size() println "${city}小区量:"+districtKeysMap.size()
if(districtKeysMap){ if(districtKeysMap){
def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2023-03-26")) def lastWeekendStats = queryAllDistrictPriceByWeekend(city, DateUtils.parseFromTightYYYYMMDD("2024-04-07"))
println "【2023-03-26】共计均价小区:"+lastWeekendStats.size() println "【2023-03-26】共计均价小区:"+lastWeekendStats.size()
//库中小区没有计算过均价的小区 //库中小区没有计算过均价的小区
def districtKeys = districtKeysMap.keySet(); def districtKeys = districtKeysMap.keySet();
@@ -118,7 +118,7 @@ class AddAvgpriceBatch {
def array = disKey.split("##"); def array = disKey.split("##");
def estateType2 = districtKeysMap.get(disKey) def estateType2 = districtKeysMap.get(disKey)
//发重算均价命令 //发重算均价命令
addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2023-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-04-03','yyyy-MM-dd')) addPrice(array[0], array[1], array[2], estateType2, DateUtils.parseFromTightByPattern('2024-03-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2024-04-07','yyyy-MM-dd'))
} }
} }
} }
@@ -1,5 +1,7 @@
package com.fangjia.script.gj package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.poi.entity.Community
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
@@ -12,7 +14,41 @@ class CitySellStatTest {
static main(args) { static main(args) {
def task = new CitySellStatTest(); def task = new CitySellStatTest();
task.run() // task.run()
task.cityPriceRun()
}
def cityPriceRun(){
def cityObjs = CommonDatautils.findCitys();
def rets = []
rets << ["序号","城市","城市级别","城市均价起始月","起始均价","城市均价最新月","最新均价"]
def index = 1
for(def o : cityObjs){
def ret = []
ret << index
ret << o.display
ret << o.new_city_grade
def row1 = findCityMonthPrice(o.display, 1)
if(row1){
ret << row1.date
ret << row1.avg_price
}else{
ret << ""
ret << ""
}
def row2 = findCityMonthPrice(o.display, -1)
if(row2){
ret << row2.date
ret << row2.avg_price
}else{
ret << ""
ret << ""
}
rets << ret
index++
}
ExcelUtils.write("D:\\数据\\其它\\全国城市均价.xlsx", rets, "城市均价")
} }
def run(){ def run(){
@@ -25,18 +61,20 @@ class CitySellStatTest {
},0) },0)
} }
def findFirstMonth(def city){ def findCityMonthPrice(def city, def asc){
def q = new BasicDBObject(); def q = new BasicDBObject();
q.put("city", city) q.put("city", city)
def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("_id",1)).limit(1) def dbcursor = CITY_STAT_COLL.find(q).sort(new BasicDBObject("date", asc)).limit(1)
def month; def obj = null
for(def o : dbcursor){ for(def o : dbcursor){
month = o.date obj = o
} }
return month return obj
} }
def findTotalMonth(def city, def month){ def findTotalMonth(def city, def month){
def q = new BasicDBObject(); def q = new BasicDBObject();
q.put("city", city) q.put("city", city)
@@ -23,18 +23,22 @@ class EsSearchUtils {
ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString()) ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString())
def rets = [] def rets = []
ElasticUtils.executeSqlQuery(sql,{ data-> ElasticUtils.executeSqlQuery(sql,{ data->
def m = [:] def mapName = data.name.replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
m.city = data.cityname def newName = name.replaceAll('[·\\(\\)()]|·', '')
m.region = data.adname if(newName == mapName){
m.name = data.name def m = [:]
m.address = data.address m.city = data.cityname
m.type = data.type m.region = data.adname
m.code = data.typecode m.name = data.name
m.address = data.address
m.type = data.type
m.code = data.typecode
def location = data.location def location = data.location
m.lng = location.lon m.lng = location.lon
m.lat = location.lat m.lat = location.lat
rets << m rets << m
}
}) })
return rets; return rets;
@@ -52,9 +52,9 @@ class YouDaDataParser {
static main(args) { static main(args) {
def task = new YouDaDataParser(); def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo // task.newHouseparse();//友达一手房存入mongo
task.esfParser("D:\\数据\\友达\\2024年1月\\查询1.DBF");//友达二手房存入mongo task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es // task.newhouseToSaveEs();//新房保存到es
// task.esfhouseToSaveEs();//二手房保存到es task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
} }
@@ -31,7 +31,34 @@ class CityStatExport {
// task.regionStat() // task.regionStat()
// task.stat() // task.stat()
// task.houseStat() // task.houseStat()
task.addFieldStat() // task.addFieldStat()
task.exportRegionAvgPrice()
}
def exportRegionAvgPrice(){
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐","福清","闽侯","连江","罗源","闽清","永泰","平潭"];
def add = -9;
def months = []
while(add<=0){
months << DateUtils.addMonth(new Date(), add)
add++
}
def rets = []
rets << months
for(def region : regions){
def row = []
row << region
for(def i=0; i<months.size()-1;i++){
def start = months[i].replace("-","").toInteger()
def end = months[i+1].replace("-","").toInteger()
def avgprice = CommonDatautils.findHouseAvgPrice("福州", region, start, end)
row << avgprice
}
rets.add(row)
}
ExcelUtils.write("D:\\数据\\福州\\福州区域及城市走势_2024-03.xlsx", rets, "房源均价")
} }
def addFieldStat(){ def addFieldStat(){
@@ -33,7 +33,16 @@ class CityStat {
// task.run6() // task.run6()
// task.run7() // task.run7()
// task.run8() // task.run8()
task.dealPriceStat() // task.dealPriceStat()
task.countDist()
}
def countDist(){
ExcelUtils.modify("D:\\数据\\其它\\小区统计.xlsx", { d ->
if(d['B']){
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["city", "region", "name"])
d['D'] = districts.size()
}
},0);
} }
def dealPriceStat(){ def dealPriceStat(){
@@ -0,0 +1,112 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
//郑州银行数据导出
class ZhengzhouBlankStat {
public static DBCollection cityInfoColl = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "mongo.db.fangjia.com")
public static final def DISTRICT_FIELDS_MAP = [
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',build_date:'buildDate',
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',
start_building_date :'startBuildingDate',loop_line_info :'loopLineInfo',property_life :'propertyLife',house_property :'houseProperty',build_structure :'buildStructure',
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', street_community:'streetCommunity',committee:'committee',schools:'schools',house_structure:'houseStructure',
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
]
static main(args) {
def task = new ZhengzhouBlankStat();
task.run()
}
def run(){
def citys = ["郑州","洛阳","平顶山","安阳","许昌","南阳","新乡","焦作","开封","信阳","驻马店","漯河","鹤壁","周口","濮阳","三门峡","商丘","济源"]
//statRegionDistrictCount(citys)
// statCityProcent(citys)
statCityHouseloudongProcent(citys)
}
def statRegionDistrictCount(citys){
def rets = []
rets << ["城市", "区县", "楼盘数量"]
for(def c : citys){
def regions = CommonDatautils.getCityRegions(c)
for(def r : regions){
def count = CommonDatautils.findRegionDistrictCount("district", c, r)
def ret = []
ret << c
ret << r
ret << count
rets << ret
}
}
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省区县小区量统计")
}
def statCityProcent(citys){
def rets = []
rets << ["城市","数据量","地址","地址别名","别名","楼栋总数","户数","产权年限","产权性质","建筑结构","竣工时间","小区当前均价"]
for(def c : citys){
def dbo = CommonDatautils.findCityInfo(c)
if(dbo){
def detail = dbo.district_detail
def ret = []
ret << c
if(detail){
ret << dbo.existing_district_count//数据量
ret << detail.address*100// 地址
ret << detail.addrAlias*100// 地址别名
ret << detail.alias*100// 别名
ret << detail.loudongCount*100// 楼栋总数
ret << detail.householdCount*100// 户数
ret << detail.propertyLife*100// 产权年限
ret << detail.houseProperty*100// 产权性质
ret << detail.buildStructure*100// 建筑结构(按材料划分)
ret << detail.finishBuildingDate*100// 竣工时间
ret << dbo.price_coverage*100// 小区当前均价
}
rets << ret
}
}
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省城市字段覆盖率统计")
}
def statCityHouseloudongProcent(citys){
def rets = []
rets << ["城市","楼栋","单元","所在层数","房号","面积","套型","楼层"]
for(def c : citys){
def dbo = CommonDatautils.findCityInfo(c)
if(dbo){
def detail = dbo.house_loudong_detail
def ret = []
ret << c
if(detail){
ret << detail.house_num_rate*100//楼栋
ret << detail.unit_num_rate*100// 单元
ret << detail.room_num_rate*100// 所在层数
ret << detail.room_num_rate*100// 房号
ret << detail.area_rate*100// 面积
ret << detail.room_info_rate*100// 套型
}
def loudong_detail = dbo.loudong_detail
if(loudong_detail){
ret << loudong_detail.loudong_height_rate*100// 楼层
}
rets << ret
}
}
ExcelUtils.write("D:\\数据\\郑州银行\\郑州银行统计.xlsx", rets, "河南省房产信息字段覆盖率统计")
}
}
@@ -0,0 +1,86 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
class DevProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
static main(args) {
def task = new DevProcess();
task.run()
}
def run(){
def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls")
def index = 0
def mcaDistMap = CommonDatautils.queryDistNameCity()
for(def r : rets){
index ++
if(index >2){
def udata = new BasicDBObject()
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(null, r['E']);
if (jsonObject.get("district") && jsonObject.get("district") == "null") {
jsonObject = FangjiaClientUtils.formatFivelevel(null, r['C']+r['E']);
}
if (jsonObject.get("district") && jsonObject.get("district") == "null") {
jsonObject = FangjiaClientUtils.formatFivelevel(null, r['B']+r['C']+r['E']);
}
udata.address = r['E']
udata.create_time = new Date()
udata.industry = r['F']
udata.level = "国家级"
udata.name = r['D']
udata.province = r['B']
udata.type = "农村创业园区"
if(jsonObject) {
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
udata.city = jsonObject.get("city")
}
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
udata.district = jsonObject.get("district")
}
if(!udata.district && udata.city){
def city = RegionData.formatCity(udata.city)
def region = RegionData.formatRegion(udata.city, r['C'])
if(region){
def regionObj = mcaDistMap.get(city+"#"+region)
if(regionObj){
udata.district = regionObj.district
}
}
}
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
udata.township = jsonObject.get("township")
}
if(!udata.district){
println index+" "+ udata
}
}
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
if(!obj){
development_zone_CONN.save(udata)
}else{
println udata.name +" 已存在"
}
}
}
}
}
@@ -0,0 +1,467 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.PgsqlCommUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class GXDdistrictMatch {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def regionCodeDataMap = PgsqlCommUtils.findDistrictMap();
def fullCityMap = CommonDatautils.queryFullNameCity();
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
// task.addOldMatch()
// task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
task.restoreDist()//恢复小区status=0
}
def addOldMatch(){
//老匹配
def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
d['AL'] = jsonObject.get("district")
d['AM'] = jsonObject.get("name")
d['AN'] = jsonObject.get("address")
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['AO'] = dbo.status
d['AP'] = dbo.name_cat
d['AQ'] = dbo.estate_type2
}
}
}
});
}
def runDb(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d ->
if(!d['U']){
def fullCityObj = fullCityMap.get(d['AD'])
if(fullCityObj){
def fullRegion = regionCodeDataMap.get(d['A']+"#"+d['E'])
def obj = findPoiFromDB(fullCityObj.city, fullRegion, d['C'])
if(obj){
d['U'] = "地图库"
d['V'] = obj.city
d['W'] = obj.region
d['X'] = obj.name
d['Y'] = obj.type
d['Z'] = obj.address
d['AA'] = obj.lng
d['AB'] = obj.lat
}else{
obj = findDatas(d['AD'], d['AE'], d['C'], d['C'].replaceAll('[·\\(\\)()]|·', ''))
if(obj){
d['U'] = "楼盘抓取库"
d['V'] = obj.city
d['W'] = obj.region
d['X'] = obj.name
d['Y'] = obj.property_type
d['Z'] = obj.address
def loc = obj.location
if(loc){
d['AA'] = loc.x
d['AB'] = loc.y
}
}
}
}
}
})
}
def findPoiFromDB(city, region, name){
def rets = EsSearchUtils.findPois(city, region, name)
def obj = null
if(rets && rets.size()==1){
obj = rets[0]
}
return obj
}
def run(){
def fileName = "D:\\数据\\国信达\\接口匹配.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['V'] && d['A']){
def city = getCityFromCityCode(d['A'])
def region = regionCodeDataMap.get(d['A'] + "#" + d['E'])
if (city && region) {
region = RegionData.formatRegion(city, region)
d['AD'] = city
d['AE'] = region
if (region) {
d = match(city, region, d)
} else {
println d['A'] + "#" + d['E'] + " 区域无法格式化"
}
} else {
println d['A'] + "#" + d['E'] + " 无法格式化"
}
}
})
}
def updateDistStatus(){
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
def index = 0
ExcelUtils.modify(fileName, { d ->
if(d['U'] && ["名称匹配","名称匹配(Update)","名称在别名中","名称在别名中(Update)","别名匹配"].contains(d['U'])){
index++
def status = d['Z']
def name_cat = d['AA']
println d["V"]+" "+d["X"]+name_cat
if(status!=null && status.toString().toDouble().toInteger()==0){
if(name_cat==null){//&& name_cat.toString().toDouble().toInteger()!=0
def dist = CommonDatautils.findDistrict(d["V"],d["W"],d["X"], "district")
if(dist){
def delUdata = new BasicDBObject()
delUdata.put("name_cat", 0)
delUdata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
saveOptLog(dist.city, dist.region, dist.name, "update")
}
}
}
}
})
}
def restoreDist(){
def fileName = "D:\\数据\\国信达\\11城小区需恢复数据.xlsx"//国信达11城(全).xlsx
def index = 0
ExcelUtils.modify(fileName, { d ->
if(d['U']){
index++
def status = d['Z']
println d["V"]+" "+d["X"]
if(status && status.toString().toDouble().toInteger()==1){
def dists = CommonDatautils.findAllStatusDistrict(d["V"],d["W"],d["X"], "district")
if(dists && dists.size()==1){
def dist = dists[0]
def delUdata = new BasicDBObject()
delUdata.put("name_cat", 0)
delUdata.put("status", 0)
delUdata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
saveOptLog(dist.city, dist.region, dist.name, "update-status")
}
}
}
})
}
def saveToDB(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
ExcelUtils.modify(fileName, { d ->
if(d['U'] && d['AG']!='过滤'){
def udata = new BasicDBObject()
udata.put("city", d['AD'])
udata.put("region", d['AE'])
def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def alias = []
if(name!=d['C']){
alias << d['C']
}
if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
alias << d['AI']
}
if(alias){
udata.put("alias", alias)
}
udata.put("address", d['AF'])
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
def estateType2 = "普通住宅"
def type = ""
if(d['AH']){
type = d['AH']
}else{
type = d['Y']
}
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
udata.put("estateType2", estateType2)
if(d['AA']){
Long lng = d['AA']*10000000000
udata.put("lng", lng)
}
if(d['AB']){
Long lat = d['AB'] * 10000000000
udata.put("lat", lat)
}
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
def opt = "add"
if(s && s=='成功'){
saveOptLog(udata.city, udata.region, udata.name, opt)
}
}
})
}
def match(city, region, d){
d = matching(city, region, d, d['C'].replaceAll('[·\\(\\)()]|·', ''), d['D'], "")
if(!d['U']){
d = matching(city, region, d, d['C'].replace("小区","").replace("号楼","号院").replace("号","号院"), d['D'], "(Update)")
if(!d['U']){
d = matching(city, region, d, d['C']+"小区", d['D'], "(Update)")
}
if(!d['U']){
if(d['F'] && d['G']){
d = matchAddr(city, region, d, "")
}
}
if(!d['U']){
d = matchApi(city, region, d, "property", false)
if(!d['U']){
d = matchApi(city, region, d, null, true)
}
}
if(!d['U']){
d = matching2(city, region, d, d['C'], d['D'], "")
}
if(!d['U']){
if(d['C'].contains("小区")){
d = matching2(city, region, d, d['C'].replace("小区",""), d['D'], "(Update)")
}else{
d = matching2(city, region, d, d['C']+"小区", d['D'], "(Update)")
}
}
}
return d
}
def matchApi(city, region, d, cat, useMap){
def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
if(!jsonObject){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
if(!jsonObject && d['D']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
}
}
if(jsonObject != null && jsonObject.credit>0) {
d['U'] = "接口匹配"
d['V'] = jsonObject.get("city")
d['W'] = jsonObject.get("district")
d['X'] = jsonObject.get("name")
def type = jsonObject.get("category")//propertyType
if(type && type!='null'){
d['Y'] = type
}
def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
if(dbo){
d['Z'] = dbo.status
d['AA'] = dbo.name_cat
}
d['AC'] = jsonObject.get("credit")
if(jsonObject.get("address") && jsonObject.get("address")!='null'){
d['AH'] = jsonObject.get("address")
}
}
return d
}
def matching(city, region, d, name, alias, str){
//正常状态下有没有
def rets = CommonDatautils.findDistrictList(city, region, name, "district")
if(rets){
if(rets.size()>=1){
d = setRowValue(rets[0], d, "名称匹配"+str)
}
}else{
rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if(rets && rets.size()>=1){
d = setRowValue(rets[0], d, "名称在别名中"+str)
}else{
rets = CommonDatautils.findDistrictByAddress(city, region, name, "district")
if(rets && rets.size()>=1){
d = setRowValue(rets[0], d, "名称在地址中"+str)
}else{
if (d['D']) {
rets = CommonDatautils.findDistrictByAlias(city, region, alias.replaceAll('[·\\(\\)()]|·', ''), "district")
if (rets) {
if (rets.size() >= 1) {
d = setRowValue(rets[0], d, "别名匹配"+str)
}
}
}
}
}
}
if(rets){
d['AI'] = rets.size()
}
return d;
}
def matching2(city, region, d, name, alias, str){
def rets = CommonDatautils.findAllStatusDistrict(city, region, name, "district")//包含删除的
if(rets){
if (rets.size() >= 1) {
d = setRowValue(rets[0], d, "名称匹配(非正常)"+str)
}
}else{
rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, name, "district")
if (rets && rets.size() >= 1) {
d = setRowValue(rets[0], d, "别名匹配(非正常)"+str)
}else{
rets = CommonDatautils.findAllStatusDistrictByAlias(city, region, alias, "district")
if (rets.size() >= 1) {
d = setRowValue(rets[0], d, "别名匹配(非正常)"+str)
}
}
}
if(rets){
d['AI'] = rets.size()
}
return d;
}
def matchAddr(city, region, d, str){
def rets = CommonDatautils.findDistrictByAddress(city, region, d['F']+d['G'],"district")
if(rets){
if (rets.size() >= 1) {
d = setRowValue(rets[0], d, "地址匹配")
}
}
if(rets){
d['AI'] = rets.size()
}
return d
}
def setRowValue(dbo, d, remark){
if (dbo) {
d['V'] = dbo.city
d['W'] = dbo.region
d['X'] = dbo.name
d['Y'] = dbo.estate_type2
d['Z'] = dbo.status
d['AA'] = dbo.name_cat
d['U'] = remark
d['AH'] = dbo.address
}
return d;
}
def getCityFromCityCode(cityCode){
def city = null
for(def m : cityCodeDataMap){
if(m.value == cityCode){
city = m.key
break;
}
}
return city;
}
def filterSpacialChar(def s){
def pattern = ~/[()!;\·]\[【】()]/
def matcher = pattern.matcher(s)
def count = matcher.getCount()
if(count && count>0){
return matcher.replaceAll('_')
}
return null;
}
def findDatas(def city, def region, def name1, def name2){
def sql = "select * from community where city='${city}' and region='${region}' and (name='${name1}' or name='${name2}')"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
println d.property_type
})
if(rets){
def obj = null
for(def r : rets){
if(r.property_type && (r.property_type.contains("住宅") || r.property_type.contains("别墅") || r.property_type.contains("公寓"))){
obj = r
break
}
}
if(obj){
return obj;
}else{
return rets[0];
}
}
return null;
}
def saveOptLog(city, region, name, type){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category","district");
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", type)
obj.put("user_name", "高娟")
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -0,0 +1,51 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.regex.Pattern
class TownshipPrcoess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
static main(args) {
def task = new TownshipPrcoess();
task.run()
}
def run() {
def fileName = "D:\\数据\\数据建设\\乡镇别名问题.xlsx"
ExcelUtils.modify(fileName, { d ->
if(!d['G']){
// def name1 = d['E'].replace("街道","").replace("镇","").replace("乡","")
// def name2 = d['F'].replace("街道","").replace("镇","").replace("乡","")
// if(name1 == name2){
// d['G'] = "不处理"
// }
def lists1 = findCommunityTown(community_CONN, d['C'], d['D'], d['E'])
def lists2 = findCommunityTown(community_CONN, d['C'], d['D'], d['F'])
if(lists1 && lists2){
d['G'] = "需删除"
}
}
})
}
def findCommunityTown(community_CONN, city, region, town){
def q = new BasicDBObject()
q.put("city_name", city)
q.put("area_name", region)
q.put("street_name", town)
def rets = community_CONN.find(q).toList()
return rets;
}
}
@@ -0,0 +1,13 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
class TradeDatasDistBatch {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
static main(args) {
// def
}
}
@@ -228,9 +228,9 @@ class FangjiaClientUtils {
def url = null; def url = null;
try { try {
address = address.replace("{","").replace("\\","") address = address.replace("{","").replace("\\","")
url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}" // url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){ if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8") url = url + "&city="+URLEncoder.encode(city,"UTF-8")
} }
@@ -314,13 +314,13 @@ class FangjiaClientUtils {
def content = null def content = null
try { try {
address = address.replace("{","") address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") // def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:32080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
if(city){ if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8") url = url +"&city="+URLEncoder.encode(city,"UTF-8")
} }
content = new HttpRequest().newGet(url, null, "utf-8") content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content) jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) { } catch (Exception e) {
println address +" "+content println address +" "+content
e.printStackTrace(); e.printStackTrace();
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.es package com.fangjia.script.gj.es
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
@@ -16,7 +17,13 @@ class EsSelectTest {
static main(args) { static main(args) {
def task = new EsSelectTest() def task = new EsSelectTest()
task.run() // task.run()
task.test()
}
def test(){
def ret = CommonDatautils.findLogs("1007741", "/property/evaluate", "2024-03-01", "2024-03-28")
} }
def run(){ def run(){
@@ -18,7 +18,44 @@ DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia",
DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL) DBCollection community_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
def filePath = "D:\\数据\\光大\\全量.xlsx" def filePath = "D:\\数据\\南京\\南京银行测试-新跑.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
def communityObj = null
if(row['E']){
communityObj = findCommunityTown(community_CONN, row['C'], row['D'], row['E'])
}
if(communityObj){
row['M'] = communityObj.province_code
row['N'] = communityObj.city_code
row['O'] = communityObj.area_code
row['P'] = communityObj.street_code
}
if(row['D']){
if(row['D'].toString().endsWith("区")){
row['F'] = "非农"
}else{
if(communityObj){
if(communityObj.has_gov && communityObj.has_gov=="是"){
row['I'] = "是"
row['F'] = "非农"
}else{
row['I'] = "否"
if(row['E']){
def townObj = isTown(row['C'], row['D'], row['E'])
if(townObj){
row['J'] = "是"
row['F'] = "非农"
}else{
row['F'] = "农"
}
}
}
}
}
}
});
//def filePath = "D:\\数据\\光大\\全量.xlsx"
//ExcelUtils.modify(filePath, { row -> //ExcelUtils.modify(filePath, { row ->
// if(!row['O']){ // if(!row['O']){
// def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A']) // def jsonObj = FangjiaClientUtils.villageIdentify(null, row['A'])
@@ -62,19 +99,19 @@ def filePath = "D:\\
// } // }
// } // }
def retMap = SignatureUtils.getAliAddressStrc(row['A']) // def retMap = SignatureUtils.getAliAddressStrc(row['A'])
println row['A'] +" " + retMap.toString() // println row['A'] +" " + retMap.toString()
if(retMap){ // if(retMap){
row['Y'] = retMap.prov // row['Y'] = retMap.prov
row['Z'] = retMap.city // row['Z'] = retMap.city
row['AA'] = retMap.district // row['AA'] = retMap.district
row['AB'] = retMap.town // row['AB'] = retMap.town
row['AC'] = retMap.devzone // row['AC'] = retMap.devzone
row['AD'] = retMap.road // row['AD'] = retMap.road
row['AE'] = retMap.roadno // row['AE'] = retMap.roadno
row['AF'] = retMap.poi // row['AF'] = retMap.poi
row['AG'] = retMap.subpoi // row['AG'] = retMap.subpoi
} // }
// }) // })
@@ -0,0 +1,191 @@
package com.fangjia.script.gj.village
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import net.sf.json.JSONObject
import okhttp3.*;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder;
class QiliangTest {
// static main(args) {
// def task = new QiliangTest()
//// task.run()
// task.getHttpResult("扬州扬子江北路399号11041")
// }
def run(){//地址测试-2024.3.4-结果.xlsx
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
if(!row['N']){
def address = null
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
address = row['B']+row['C']
}else{
address = row['C']
}
if(address){
def result = getHttpResult(address)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['N'] = data.province
}
if(data.city && data.city!='null'){
row['O'] = data.city
}
if(data.district && data.district!='null'){
row['P'] = data.district
}
if(data.town && data.town!='null'){
row['Q'] = data.town
}
if(data.village && data.village!='null'){
row['R'] = data.village
}
if(data.areaCode && data.areaCode!='null'){
row['S'] = data.areaCode
}
if(data.farmer == "0"){
row['T'] = "非农"
}else{
row['T'] = "农"
}
if(data.isGovt == "0"){
row['U'] = "否"
}else{
row['U'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['V'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['W'] = data.govtAddr?data.govtAddr:''
}
}
}
}
}
})
}
def getHttpResult(address){
def data = new JSONObject()
data.put("address", address)
data.put("type","2")
Map headers = new HashMap();
headers.put("Accept", "application/json, text/plain, */*");
headers.put("Accept-Encoding", "gzip, deflate");
headers.put("Accept-Language","zh-CN,zh;q=0.9")
headers.put("Content-Type","application/json;charset=UTF-8")
headers.put("Host","146.56.217.44:9010")
headers.put("Origin","http://146.56.217.44:8088")
headers.put("Proxy-Connection","keep-alive")
headers.put("Referer","http://146.56.217.44:8088/")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
HttpRequest request1 = new HttpRequest()
try{
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
def response = JSONObject.fromObject(res)
println response
return response
}catch(Exception e){
e.printStackTrace()
}
return null;
}
final static String proxyIp = "tunnel2.qg.net";
final static Integer proxyPort = 14186;
final static String authKey = "113BBC06";
final static String password = "3B2FBD05B0AF";
public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8");
// public String request(params) throws IOException {
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort));
// OkHttpClient client = new OkHttpClient.Builder()
// .proxy(proxy)
// .proxyAuthenticator((route, response){
// String credential = Credentials.basic(authKey, password);
// return response.request().newBuilder().header("Proxy-Authorization", credential).build();
// }).
// build();
//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build();
//
// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params));
// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build();
//
// Response response = client.newCall(request).execute();
// return response.body().string();
//
//// return client.newCall(request).execute();
// }
public static void main(String[] args) {
def proxyHost = "tunnel2.qg.net"
def proxyPort = 14186
def proxyUsername = '113BBC06'
def proxyPassword = '3B2FBD05B0AF'
def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign'
def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2']
// 创建代理服务器的连接
def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}")
// 打开代理连接
def proxyConnection = proxy.openConnection()
// 创建目标URL的连接
def url = new URL(targetUrl)
def connection = url.openConnection(proxy)
// 设置请求方法
connection.setRequestMethod("GET")
// 添加请求头(可选)
connection.setRequestProperty("Content-Type", "application/json")
// 构建查询参数
def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&')
// 将查询参数附加到目标URL
def finalUrl = "${targetUrl}?${queryString}"
// 发送请求
connection.connect()
// 获取响应
def responseCode = connection.getResponseCode()
def responseMessage = connection.getResponseMessage()
// 读取响应内容
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
def responseContent = reader.readLine()
reader.close()
println "Response Code: ${responseCode}"
println "Response Message: ${responseMessage}"
println "Response Content: ${responseContent}"
}
}
@@ -200,4 +200,9 @@ public class SignatureUtils {
} }
return null; return null;
} }
public static void main(String[] args){
def a = getAliAddressStrc("上海市长宁区新泾三村")
println a
}
} }
@@ -66,8 +66,10 @@ class GaoDePoiDataProcess extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
// println ProvinceData.getProvince("成都") // println ProvinceData.getProvince("成都")
synchMapPoi(); // synchMapPoi();
// synchCompany()
synchCompany()
// poiCatProcess() // poiCatProcess()
// addressAndCatProcess() // addressAndCatProcess()
// synchNetwork() // synchNetwork()
@@ -99,10 +101,10 @@ class GaoDePoiDataProcess extends BaseBatch{
def synchCompany(){ def synchCompany(){
fullCityMap = CommonDatautils.queryFullNameCity() fullCityMap = CommonDatautils.queryFullNameCity()
//"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市", //"三明市","宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市",
def citys = ["湖州市"] def citys = ["º¼ÖÝ","Äþ²¨","ÎÂÖÝ","¼ÎÐË","̨ÖÝ","ÉÜÐË","½ð»ª","ÖÛɽ","áéÖÝ","ÀöË®"]
for(def sampleCity : citys){ for(def sampleCity : citys){
def cityObj = fullCityMap.get("湖州"); def cityObj = fullCityMap.get(sampleCity);
def city = null def city = null
if(cityObj){ if(cityObj){
city = cityObj.city city = cityObj.city
@@ -112,6 +114,7 @@ class GaoDePoiDataProcess extends BaseBatch{
println city+ " 加载公司数据:"+companyDatas.size() println city+ " 加载公司数据:"+companyDatas.size()
def rows = running(city, companyDatas, "company", townMap) def rows = running(city, companyDatas, "company", townMap)
save(city, rows) save(city, rows)
// saveToMG(city, rows)
println "完成" println "完成"
} }
} }
@@ -182,7 +185,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def createCompanyData(map, townMap){ def createCompanyData(map, townMap){
def retMap = [:] def retMap = [:]
retMap._id = map._id retMap.data_id = map._id.toString()
retMap.fjw_province = map.province_name retMap.fjw_province = map.province_name
retMap.fjw_city = map.city_name retMap.fjw_city = map.city_name
@@ -21,7 +21,7 @@ import java.util.concurrent.atomic.AtomicLong
class SuggestionDataSynch extends BaseBatch{ class SuggestionDataSynch extends BaseBatch{
static def targetUrl = "http://192.168.0.12:9200/seaweed/" static def targetUrl = "http://192.168.0.12:9200/seaweed/"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(100)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
@@ -45,10 +45,10 @@ class PdsUseStatisticBatch extends BaseBatch{
final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000) final static DBCollection fsDB = MongoUtils.getCollectionByName("fangjia_fs", "files", MongoUtils.MongoDB_Domain_fs_mongo, 27000)
def static emails = [ def static emails = [
"duweiliang@fangjia.com", // "duweiliang@fangjia.com",
"yaojingmeng@fangjia.com", // "yaojingmeng@fangjia.com",
"xuqi@fangjia.com", // "xuqi@fangjia.com",
"wangpeili@fangjia.com" "gaojuan@fangjia.com"
] ]
static main(args) { static main(args) {
@@ -163,9 +163,9 @@ class AnalyticsApiLogPerformanceBatch extends BaseBatch {
} }
static void sendEmail(String sub, String msg) { static void sendEmail(String sub, String msg) {
jms.sendMail("duxiaoyu@fangjia.com", sub, msg) // jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
jms.sendMail("wangwenyan@fangjia.com", sub, msg) jms.sendMail("wangwenyan@fangjia.com", sub, msg)
jms.sendMail("helvyuan@fangjia.com", sub, msg) // jms.sendMail("helvyuan@fangjia.com", sub, msg)
jms.sendMail("gaojuan@fangjia.com", sub, msg) jms.sendMail("gaojuan@fangjia.com", sub, msg)
} }
@@ -32,11 +32,11 @@ class CommonLogAnaBatch extends BaseBatch {
def static bjEvaReport = []; def static bjEvaReport = [];
public static final def EMAILS = [ public static final def EMAILS = [
"jiangxiaochun@fangjia.com", "jiangxiaochun@fangjia.com"
"limingmin@fangjia.com", // "limingmin@fangjia.com",
"xiaohai@fangjia.com", // "xiaohai@fangjia.com",
"luxuping@fangjia.com", // "luxuping@fangjia.com",
"yukai@fangjia.com" // "yukai@fangjia.com"
]; ];
static main(args) { static main(args) {
@@ -26,12 +26,12 @@ class DaikuanLogAnaBatch extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public static final def EMAILS = [ public static final def EMAILS = [
"jiangxiaochun@fangjia.com", "jiangxiaochun@fangjia.com"
"limingmin@fangjia.com", // "limingmin@fangjia.com",
"zhongjingyi@fangjia.com", // "zhongjingyi@fangjia.com",
"xiaohai@fangjia.com", // "xiaohai@fangjia.com",
"luxuping@fangjia.com", // "luxuping@fangjia.com",
"yukai@fangjia.com" // "yukai@fangjia.com"
]; ];
static main(args) { static main(args) {
@@ -148,14 +148,14 @@ class ReadBkoffSafeLogBatch extends BaseBatch{
} }
static void sendEmail(String sub, String msg) { static void sendEmail(String sub, String msg) {
jms.sendMail("duxiaoyu@fangjia.com", sub, msg) // jms.sendMail("duxiaoyu@fangjia.com", sub, msg)
jms.sendMail("joe@fangjia.com", sub, msg) // jms.sendMail("joe@fangjia.com", sub, msg)
jms.sendMail("xiaohai@fangjia.com", sub, msg) // jms.sendMail("xiaohai@fangjia.com", sub, msg)
jms.sendMail("caoyuyan@fangjia.com", sub, msg) jms.sendMail("caoyuyan@fangjia.com", sub, msg)
jms.sendMail("wangchencen@fangjia.com", sub, msg) // jms.sendMail("wangchencen@fangjia.com", sub, msg)
jms.sendMail("yaojingmeng@fangjia.com", sub, msg) // jms.sendMail("yaojingmeng@fangjia.com", sub, msg)
jms.sendMail("chichilook@qq.com", sub, msg) // jms.sendMail("chichilook@qq.com", sub, msg)
jms.sendMail("sunjian@fangjia.com", sub, msg) // jms.sendMail("sunjian@fangjia.com", sub, msg)
} }
static public String getCatById(String id) { static public String getCatById(String id) {
@@ -0,0 +1,50 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.ExcelUtils
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\1.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count = count + 1
if (count % 10 == 0) {
println("已处理:" + count)
}
sleep(80)
def address = items['A']
def map = SignatureUtils.getAliAddressStrc(address)
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
if ("prov" in map) {
def province = map.get("prov")
items['E'] = province
}
if ("city" in map) {
def city = map.get("city")
items['G'] = city
}
if ("district" in map) {
def district = map.get("district")
items['I'] = district
}
if ("town" in map) {
def town = map.get("town")
items['K'] = town
}
// if ("community" in map) {
// def community = map.get("community")
// items['S'] = community
// }
if ("devzone" in map) {
def devzone = map.get("devzone")
items['Q'] = devzone
}
})
@@ -0,0 +1,85 @@
package com.fangjia.script.majintao.JSBankTest;
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils;
import net.sf.json.JSONObject;
import java.util.Base64;
import net.sf.json.JSONArray;
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count += 1
if (count % 10 == 0) {
println("已处理:" + count)
}
def zx = ["北京", "天津", "上海", "重庆"]
// String province = "天津";
String province = items['E'];
if (zx.contains(province)) {
province = province + "市";
}
// String name = "天津经济技术开发区";
String name = items['Q'];
String elasticUsername = "elastic";
String elasticPassword = "elasticsearch";
String seawaterUrl = "http://192.168.0.89:9200/development_zone/_search";
String basicAuth = "Basic " + Base64.getEncoder().encodeToString((elasticUsername + ":" + elasticPassword).getBytes("UTF-8"));
String body = """
{
"query": {
"bool": {
"must": [
{
"match": {
"name": {
"query": "${name}",
"operator": "and"
}
}
},
{
"term": {"province": {"value": "${province}"}}
}
]
}
}
}
""";
HttpRequest request = new HttpRequest();
JSONObject jsonBody = JSONObject.fromObject(body);
JSONObject header = new JSONObject();
header.put("Authorization", basicAuth);
String res = request.postWithBody(seawaterUrl, jsonBody, header, "UTF-8");
JSONObject response = JSONObject.fromObject(res);
JSONObject obj = null;
if (response != null && response.containsKey("hits")) {
JSONObject hits = response.getJSONObject("hits");
if (hits != null && hits.size() > 0) {
JSONArray hitsArray = hits.getJSONArray("hits");
// 省略了之前的部分...
if (hitsArray != null && hitsArray.size() > 0) {
for (int i = 0; i < hitsArray.size(); i++) {
JSONObject hit = hitsArray.getJSONObject(i);
JSONObject source = hit.getJSONObject("_source");
String devName = source.optString("name"); // Assuming "name" is a String
String level = source.optString("level"); // Assuming "level" is a String
if (devName.equals(name)) {
items['O'] = "是";
items['P'] = level;
println("省份:" + province + " 名称:" + name + " 开发区名:" + devName + " 开发区级别:" + level);
break
}
}
}
}
}
});
@@ -0,0 +1,85 @@
package com.fangjia.script.majintao.JSBankTest
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import org.apache.tools.ant.types.resources.selectors.None
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\测试.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
count += 1
if (count % 10 == 0) {
println("已处理:" + count)
}
String address = items['B']
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
def map = [:]
map.put("address", address)
map.put("showFullValue", "2")
map.put("showTestValue", "2")
map.put("type", "2")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if (result != null) {
if ("addrVerify") {
items['C'] = result.get("addrVerify")
}
if ("province" in result) {
items['D'] = result.get("province")
}
if ("city" in result) {
items['F'] = result.get("city")
}
if ("district" in result) {
items['H'] = result.get("district")
}
if ("town" in result) {
items['J'] = result.get("town")
}
if ("committee" in result) {
items['L'] = result.get("committee")
}
if ("typeCode" in result) {
items['M'] = result.get("typeCode")
}
if ("countyGov") {
items['N'] = result.get("countyGov")
}
def devDistrict = ""
if ("devDistrict" in result) {
devDistrict = result.get("devDistrict")
}
def dev = ""
if ("dev" in result) {
dev = result.get("dev")
}
// 开发区级别暂时没有
// 暂时使用
def defGrade = ""
if (!dev.equals("") && devDistrict.equals("否")) {
defGrade = "非国际级或省级"
devDistrict = "是"
}
if (!devDistrict.equals("")) {
items['O'] = devDistrict
}
items["P"] = defGrade
}
})
@@ -5,7 +5,11 @@ import com.fangjia.script.gj.village.ailiyun.SignatureUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
//def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801" //def address = "江苏省徐州市贾汪区经济开发区大黄山镇孤山小区B3.801"
<<<<<<< HEAD
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx";
=======
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx"; String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
ExcelUtils.removeEmptyRow(excelFilePath) ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0 def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items -> ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
@@ -14,13 +18,43 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
if (count % 10 == 0) { if (count % 10 == 0) {
println("已处理:" + count) println("已处理:" + count)
} }
<<<<<<< HEAD
sleep(80)
=======
sleep(50) sleep(50)
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
def address = items['A'] def address = items['A']
def map = SignatureUtils.getAliAddressStrc(address) def map = SignatureUtils.getAliAddressStrc(address)
//prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012 //prov=河南省 city=信阳市 district=息县 town=孙庙乡 community=前赵楼村 road=前赵楼村 poi=爱婆店 roomno=012
if ("prov" in map) { if ("prov" in map) {
def province = map.get("prov") def province = map.get("prov")
<<<<<<< HEAD
items['C'] = province
}
if ("city" in map) {
def city = map.get("city")
items['G'] = city
}
if ("district" in map) {
def district = map.get("district")
items['K'] = district
}
if ("town" in map) {
def town = map.get("town")
items['O'] = town
}
if ("community" in map) {
def community = map.get("community")
items['S'] = community
}
if ("devzone" in map) {
def devzone = map.get("devzone")
items['AE'] = devzone
}
=======
items['D'] = province items['D'] = province
} }
if ("city" in map) { if ("city" in map) {
@@ -66,6 +100,7 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
// items['K'] = roadno // items['K'] = roadno
// } // }
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
}) })
@@ -4,10 +4,17 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
<<<<<<< HEAD
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\注册地址-太仓农商行POC测试.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0
ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
=======
String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx"; String excelFilePath = "C:\\Users\\Goojia\\Desktop\\涉农对私测试数据.xlsx";
ExcelUtils.removeEmptyRow(excelFilePath) ExcelUtils.removeEmptyRow(excelFilePath)
def count = 0 def count = 0
ExcelUtils.multiThreadingModify(excelFilePath, { List items -> ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
count += 1 count += 1
if (count % 10 == 0) { if (count % 10 == 0) {
println("已处理:" + count) println("已处理:" + count)
@@ -15,6 +22,112 @@ ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
String address = items['A'] String address = items['A']
<<<<<<< HEAD
// if (province == null) {
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
def map = [:]
map.put("address", address)
map.put("showFullValue", "2")
map.put("showTestValue", "2")
map.put("type", "2")
def content = httpRequest.getJsonContent(url, map, "UTF-8")
def result = content.get("result")
if (result != null) {
if ("province" in result) {
items['B'] = result.get("province")
}
if ("provinceCode" in result) {
items['E'] = result.get("provinceCode")
}
if ("city" in result) {
items['F'] = result.get("city")
}
if ("cityCode" in result) {
items['I'] = result.get("cityCode")
}
if ("district" in result) {
items['J'] = result.get("district")
}
if ("districtCode" in result) {
items['M'] = result.get("districtCode")
}
if ("town" in result) {
items['N'] = result.get("town")
}
if ("townCode" in result) {
items['Q'] = result.get("townCode")
}
if ("committee" in result) {
items['R'] = result.get("committee")
}
if ("committeeCode" in result) {
items['U'] = result.get("committeeCode")
}
if ("typeCode" in result) {
items['V'] = result.get("typeCode")
}
if ("typeName") {
items['W'] = result.get("typeName")
}
if ("verifyFlag") {
items['X'] = result.get("verifyFlag")
}
if ("addrVerify") {
items['Y'] = result.get("addrVerify")
}
if ("countyLevelCity") {
items['Z'] = result.get("countyLevelCity")
}
if ("countyGov") {
items['AA'] = result.get("countyGov")
}
if ("communitySource") {
items['AB'] = result.get("communitySource")
}
if ("administrativeTowns" in result) {
items['AD'] = result.get("administrativeTowns")
}
if ("devDistrict" in result) {
items['AF'] = result.get("devDistrict")
}
if ("devDistrictName" in result) {
items['AG'] = result.get("devDistrictName")
}
if ("splitCommittee" in result) {
items['AH'] = result.get("splitCommittee")
}
if ("splitStreet" in result) {
items['AI'] = result.get("splitStreet")
}
if ("poiName" in result) {
items['AJ'] = result.get("poiName")
}
if ("splitStreetNum" in result) {
items['AK'] = result.get("splitStreetNum")
}
if ("dev" in result) {
items['AL'] = result.get("dev")
}
}
// }
=======
String province = items['B'] String province = items['B']
if (province == null) { if (province == null) {
String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify"; String url = "http://192.168.0.100:32080/FangJiaAddressService/villageIdentify";
@@ -120,5 +233,6 @@ ExcelUtils.multiThreadingModify(excelFilePath, { List items ->
} }
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
}) })
@@ -24,8 +24,13 @@ ExcelUtils.singleThreadingModify(excelFilePath, { List items ->
def map = [:] def map = [:]
map.put("address", address) map.put("address", address)
<<<<<<< HEAD
map.put("complete", "2")
map.put("addressAssessment", "2")
=======
map.put("complete", "true") map.put("complete", "true")
map.put("addressAssessment", "true") map.put("addressAssessment", "true")
>>>>>>> f0061ade19d549f26467ab6be7e9eaaa66cd3011
map.put("type", "2") map.put("type", "2")
def content = httpRequest.getJsonContent(url, map, "UTF-8") def content = httpRequest.getJsonContent(url, map, "UTF-8")
@@ -0,0 +1,15 @@
package com.fangjia.script.majintao.test
import com.fangjia.core.util.HttpRequest
String url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify";
HttpRequest httpRequest = new HttpRequest();
map = new HashMap<String, String>()
def map = [:]
map.put("address", address)
map.put("showFullValue", "2")
map.put("showTestValue", "2")
map.put("type", "2")
@@ -23,8 +23,8 @@ class DataStat {
// for(def city :citys){ // for(def city :citys){
// println city // println city
// } // }
// task.run(); task.run();
task.stat() // task.stat()
// def citys = ["福州", "厦门", "泉州", "莆田", "宁德", "三明", "龙岩", "南平", "漳州"] // def citys = ["福州", "厦门", "泉州", "莆田", "宁德", "三明", "龙岩", "南平", "漳州"]
// def rets = [] // def rets = []
@@ -76,7 +76,7 @@ class DataStat {
} }
def stat(){ def stat(){
def dates = [202301,202302,202303,202304,202305,202306,202307,202308,202309,202310,202311,202401] def dates = [202303,202304,202305,202306,202307,202308,202309,202310,202311,202401,202402]
def sources = ["时间","我爱我家","兔博士","吉屋","贝壳找房","乐有家","365淘房","诸葛找房","链家","Q房","麦田","中原地产","安居客","搜房","C21","城市房产","幸福里"] def sources = ["时间","我爱我家","兔博士","吉屋","贝壳找房","乐有家","365淘房","诸葛找房","链家","Q房","麦田","中原地产","安居客","搜房","C21","城市房产","幸福里"]
// crawlCount(dates,sources); // crawlCount(dates,sources);
matchCount(dates,sources); matchCount(dates,sources);
@@ -26,8 +26,8 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田", // "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士" // "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产"] // def esSources = ["搜房":"搜房"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产":"城市房产","幸福里APP":"幸福里"] def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
static main(args) { static main(args) {
runBatch(new SynchDistrictPrice(), args); runBatch(new SynchDistrictPrice(), args);
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){ def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"] def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
for(def source :sources){ for(def source :sources){
def count1 = findTotalCount(source, 202307); def count1 = findTotalCount(source, 202401);
def count2 = findTotalCount(source, 202401); def count2 = findTotalCount(source, 202402);
println source +" "+count1 +" "+count2 println source +" "+count1 +" "+count2
} }
} }
@@ -36,10 +36,10 @@ class DBHelper {
static final DB offline61PoseidonDb = new Mongo("192.168.0.61", 27037).getDB("poseidon"); static final DB offline61PoseidonDb = new Mongo("192.168.0.61", 27037).getDB("poseidon");
static final online42GoojiaFangjiaDbConn = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?useUnicode=true&characterEncoding=utf-8", "sa", "G00jia", "com.mysql.jdbc.Driver"); static final online42GoojiaFangjiaDbConn = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?useUnicode=true&characterEncoding=utf-8", "sa", "G00jia", "com.mysql.jdbc.Driver");
static final offline89GoojiaFangjiaStatDbConn = Sql.newInstance("jdbc:mysql://192.168.0.89:3306/goojia_fangjia_stat?useUnicode=true&characterEncoding=utf-8", "root", "123456", "com.mysql.jdbc.Driver"); static final offline89GoojiaFangjiaStatDbConn = Sql.newInstance("jdbc:mysql://192.168.0.89:3306/goojia_fangjia_stat?useUnicode=true&characterEncoding=utf-8", "root", "123456", "com.mysql.jdbc.Driver");
static final emailsGroup0 = ['gaojuan@fangjia.com','yukai@fangjia.com','joe@fangjia.com','duxiaoyu@fangjia.com','tangye@fangjia.com','ruanyadong@fangjia.com','xiamin@fangjia.com','zhutianli@fangjia.com']; static final emailsGroup0 = ['gaojuan@fangjia.com'];
static final emailsFangjiaWang = ['all.list@fangjia.com']; static final emailsFangjiaWang = ['all.list@fangjia.com'];
static final mobilesYanzhou = ['13621626685']; static final mobilesYanzhou = ['13621626685'];
static final emailsYanzhou = ['duxiaoyu@fangjia.com']; static final emailsYanzhou = ['wangwenyan@fangjia.com'];
static List<String> getCityNames(){ static List<String> getCityNames(){
DBObject dbObject = new BasicDBObject("status",0); DBObject dbObject = new BasicDBObject("status",0);
@@ -414,7 +414,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size()) def body = data.subList(1, data.size())
def nThread = Runtime.getRuntime().availableProcessors() * 3 def nThread = Runtime.getRuntime().availableProcessors() * 4
println nThread println nThread
ExecutorService executorService = Executors.newFixedThreadPool(nThread) ExecutorService executorService = Executors.newFixedThreadPool(nThread)
final AtomicInteger counter = new AtomicInteger(0) final AtomicInteger counter = new AtomicInteger(0)
@@ -454,8 +454,7 @@ public class ExcelUtils{
def data = ExcelUtils.read(fileName, srcSheetIndex, 0) def data = ExcelUtils.read(fileName, srcSheetIndex, 0)
def body = data.subList(1, data.size()) def body = data.subList(1, data.size())
def nThread = 10
println nThread
ExecutorService executorService = Executors.newFixedThreadPool(3) ExecutorService executorService = Executors.newFixedThreadPool(3)
final AtomicInteger counter = new AtomicInteger(0) final AtomicInteger counter = new AtomicInteger(0)
final double total = body.size() final double total = body.size()
@@ -23,7 +23,7 @@ public class ComparisonDataStatistics extends com.fangjia.script.BaseBatch{
static final DATE_NUMBER = new Date().format("yyyyMMdd").toInteger() static final DATE_NUMBER = new Date().format("yyyyMMdd").toInteger()
static final DBCollection MESSENGER_COLL = MongoUtils.getCollectionByName("fangjia", "messenger", MongoUtils.FANGJIA_ONLINE_MONGO ,27017) static final DBCollection MESSENGER_COLL = MongoUtils.getCollectionByName("fangjia", "messenger", MongoUtils.FANGJIA_ONLINE_MONGO ,27017)
static final def categories = ["district":"סլ", "office":"д×ÖÂ¥" , "shop":"ÉÌÆÌ"] static final def categories = ["district":"סլ", "office":"д×ÖÂ¥" , "shop":"ÉÌÆÌ"]
def emails = ["wangwenyan@fangjia.com","duxiaoyu@fangjia.com","joe@fangjia.com","caoyuyan@fangjia.com"] def emails = ["wangwenyan@fangjia.com","caoyuyan@fangjia.com"]
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
def static main(args){ def static main(args){
@@ -14,7 +14,7 @@ class EvaluationSourceAnalyze extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616'); static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public static final DBCollection STAT_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com") public static final DBCollection STAT_COLL = MongoUtils.getCollectionByName("fangjia", "evaluation", "mongo.db.fangjia.com")
def emails = ["wangwenyan@fangjia.com","duxiaoyu@fangjia.com","joe@fangjia.com","xiaohai@fangjia.com"] def emails = ["wangwenyan@fangjia.com","xiaohai@fangjia.com"]
static main(args) { static main(args) {
new EvaluationSourceAnalyze().process(null); new EvaluationSourceAnalyze().process(null);
@@ -1,7 +1,16 @@
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils import com.fangjia.core.util.Mongo3Utils
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document import org.bson.Document
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
@@ -11,67 +20,43 @@ import java.util.function.Consumer
import static com.mongodb.client.model.Filters.and import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq import static com.mongodb.client.model.Filters.eq
//REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
//TOWNSHIP_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
//TOWNSHIP_CONN.find().each {BasicDBObject township->
// township.area_station = township.central
// township.removeField("central")
// TOWNSHIP_CONN.save(township)
//
//}
//REGION_CONN.find(new BasicDBObject("category", "region")).each {BasicDBObject regionBlock->
// def lng = regionBlock.lng/10000000000l
// def lat = regionBlock.lat/10000000000l
// HttpRequest request = new HttpRequest()
// JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8")
// if(resp.result){
// TOWNSHIP_CONN.update(new BasicDBObject("_id", new ObjectId(resp.result.id)), new BasicDBObject('$set', new BasicDBObject('central', 1)))
// println(resp.result)
// }
//}
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
MongoCollection<Document> DISTRICT_STAT = Mongo3Utils.getDbCollection("fangjia", "district_stat_rent")
MongoCollection<Document> DEAL_PRICE = Mongo3Utils.getDbCollection("fangjia", "deal_price")
def data = [["城市", "区域", "小区名", "版块", "装修", "朝向", "室", "厅", "卫", '标题', '总价', '面积', '单价', '发布时间', '来源']] //CommonUtils.findAllCity().each {def city->
cityList.each { def city -> // DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city, null, null, "district"),{BasicDBObject office->
def statMap = [:] // if(!office.township){
DISTRICT_STAT.find(and(eq("city", city), eq("end_month_date", Date.parse("yyyyMMdd", "20190228")))).forEach(new Consumer<Document>() { // def lng = office.lng/10000000000l
@Override // def lat = office.lat/10000000000l
void accept(Document t) { // HttpRequest request = new HttpRequest()
def key = "${t.city}_${t.region}_${t.district_name}_${t.house_type}".toString() // JSONObject resp = request.getJsonContent("http://localhost:8080/FangJiaAddressService/geocode/township", ['location':"$lng,$lat".toString()], "UTF-8")
statMap[key] = t.avg_price // if(resp.result){
} // println(office._id.toString() + "\t" + office.township + "\t-->\t" + resp.result.streetName)
}) // DbUtils.seaweed.update(new BasicDBObject("_id", office._id), new BasicDBObject('$set', new BasicDBObject('township', resp.result.streetName)))
roots.each { def root -> // }
EsDataSource dataSource = new EsDataSource(); // }
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" // })
dataSource.setUrl(address) //}
Properties connectionProperties = new Properties() ExcelUtils.multiThreadingModify("C:\\Users\\goojia\\Desktop\\地图查询数据导出-添加经纬度-兴趣点.xlsx",{List item->
connectionProperties.put("user", "developer") def lng = item["AB"].toString().split(",")[0].toDouble()
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") def lat = item["AB"].toString().split(",")[1].toDouble()
dataSource.setProperties(connectionProperties) item["AD"] = GeoUtils.calculateDistance(121.526672,31.142637,lng, lat)
Connection connection = dataSource.getConnection() println(item)
Statement statement = connection.createStatement() })
ResultSet rs = statement.executeQuery("""select * from "seawater-2019" where city='${city}' and cat = 'rent' order by p_date""")
int count = 0
while (rs.next()) {
String region = rs.getString("region")
String district_name = rs.getString("district_name")
int room = rs.getInt("room")
def key = "${city}_${region}_${district_name}_${room}".toString()
if (!statMap.containsKey(key)) {
continue
}
int total_price = rs.getInt("total_price")
double area = rs.getDouble("area")
int avg_price = (int) (total_price / area)
if (total_price * 1.3 || total_price < statMap[key] * 0.7) {
continue
}
String block = rs.getString("block")
String name = rs.getString("name")
String fitment = rs.getString("fitment")
String direction = rs.getString("direction")
int hall = rs.getInt("hall")
int toilet = rs.getInt("toilet")
int p_date = rs.getInt("p_date")
if (!block || !name || !fitment || !direction || room == 0 || hall == 0 || toilet == 0) {
continue
}
println(rs.getString('url'))
def item = [city, region, district_name, block, fitment, direction, room, hall, toilet, name, total_price, area, avg_price, p_date, root]
println(item)
data << item
count++
if(count > 10){
break
}
}
}
}
ExcelUtils.write("D:\\建设银行\\租金样本.xlsx", data, "结果")
@@ -3,9 +3,9 @@ package com.fangjia.script.wenyan
public enum Watcher { public enum Watcher {
//在职同事 //在职同事
DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'), DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
WangWenYan('13916076378','wangwenyan@fangjia.com'), WangWenYan('13916076378','wangwenyan@fangjia.com'),
Tomzhao('15021021464','zhaoshumao@fangjia.com'), Tomzhao('15021021464','wangwenyan@fangjia.com'),
GaoJuan('15021384995','gaojuan@fangjia.com'); GaoJuan('15021384995','gaojuan@fangjia.com');
private final String mobile; private final String mobile;
@@ -1,51 +1,15 @@
package com.fangjia.script.wenyan.test package com.fangjia.script.wenyan.test
import com.aliyuncs.DefaultAcsClient import com.fangjia.datastore.driver.DSCollection
import com.aliyuncs.IAcsClient import com.fangjia.datastore.driver.DataStore
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest import com.fangjia.script.util.DbUtils
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse import com.mongodb.BasicDBObject
import com.aliyuncs.profile.DefaultProfile
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
def strt(String addr){
}
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
def city = item["A"]
def address1 = item["B"]
def address2 = item["C"]
HttpRequest http = new HttpRequest()
def params = new HashMap()
params["city"] = city
params["address_1"] = address1
params["address_2"] = address2
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
JSONObject result = json.result
item["D"] = result?.level
item["E"] = result?.formattedAddress_1
item["F"] = result?.formattedAddress_2
item["G"] = result?.isSameAddress
DefaultProfile profile = DefaultProfile.getProfile(
"cn-hangzhou",
"LTAIdaj4eSO79gzz",
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
IAcsClient client = new DefaultAcsClient(profile);
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
request.setServiceCode("addrp")
request.setAppKey("Xcgtyhupey0h")
request.setText("${address1};${address2}")
GetAddressSimilarityResponse response = client.getAcsResponse(request)
String data = response.getData()
json = JSONObject.fromObject(data)
item["H"] = json["sim_level"]["level"]
item["I"] = json["sim_level"]["address_a"]
item["J"] = json["sim_level"]["address_b"]
item["K"] = json["sim_level"]["is_same"]
println(json)
})
//、DSCollection seabird = new DataStore("192.168.0.38", 27007).getCollection("fangjia.seaweed")
//seabird.find(DbUtils.buildSeaweedQuery("常州", "溧阳", "吾悦广场").append("if_sell", 1)).each {
// println(it)
//}
long l = 0
if(!l){
println(l)
}
@@ -27,8 +27,7 @@ class ApiDailyCountOfficialAccount extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616') private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
def static emails = [ def static emails = [
'xuqi@fangjia.com', 'wangwenyan@fangjia.com'
'zongzhimin@fangjia.com'
] ]
static void main(String[] args){ static void main(String[] args){
@@ -165,8 +165,11 @@ public class SellTriggerServiceImpl implements TriggerService {
newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city")); newObj.put("districtname_key", district.getName() + "_" + newObj.get("region") + "_" + object.get("city"));
newObj.put("district_name", district.getName()); newObj.put("district_name", district.getName());
} }
}else{
if(object.containsField("city") && object.containsField("district_name") && object.containsField("region")){
newObj.put("districtname_key", object.get("district_name") + "_" + object.get("region") + "_" + object.get("city"));
}
} }
// newObj.put("if_school", 0); // newObj.put("if_school", 0);
// if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) { // if (districtSchoolMap != null && districtSchoolMap.containsKey(districtKey)) {
// List<SchoolKey> skList = districtSchoolMap.get(districtKey); // List<SchoolKey> skList = districtSchoolMap.get(districtKey);