Merge branch 'script_gj' into 'master'

临时导出苏州房源

See merge request !523
This commit is contained in:
2023-02-08 10:58:24 +08:00
4 changed files with 426 additions and 1 deletions
@@ -0,0 +1,115 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
class TestData {
static main(args) {
def task = new TestData()
// task.run()
task.update()
}
def update(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\ÁúÑÒ1.xlsx", { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
if(d['E'] && d['G']=="È˹¤"){
def q = new BasicDBObject()
def baidu = BaiDuApiUtils.geoconv(d['E'].toString().toDouble(), d['F'].toString().toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
q.put("lat2", dLat2)
q.put("lng2", dLng2)
Long lng = d['E'].toString().toDouble() * 10000000000L
Long lat = d['F'].toString().toDouble() * 10000000000L
q.put("lng", lng)
q.put("lat", lat)
DbUtils.seaweed.update(new BasicDBObject('_id',d['A']),new BasicDBObject('$set',q))
}
}
});
}
def run(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\ÁúÑÒ.xlsx", { d ->
def dist = CommonDatautils.findDistrictById(d['A'])
if(dist){
def loc = getLocation(dist.city, dist.region, dist.name)
if(loc){
d['E'] = loc.x
d['F'] = loc.y
println d['C']+" "+loc.x+" "+loc.y
}
}
// if(!d['E']){
// def loc = getAmapLocation("ÁúÑÒÊÐ", d['B'], d['C'])
// if(loc){
// d['E'] = loc.x
// d['F'] = loc.y
// }
// }
// if(!d['E']){
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// def loc = getAmapOldLocation(dist.city, dist.region, d['C'])
// if(loc){
// d['E'] = loc.x
// d['F'] = loc.y
// }
// }
//
// }
});
}
def getLocation(city, region, name){
String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
loc = data.location
}
})
return loc;
}
def getAmapLocation(city, region, name){
String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
if(name == data.name){
loc = data.location
}
}
})
return loc;
}
def getAmapOldLocation(city, region, name){
String sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
def loc = null
ElasticUtils.executeSqlQuery(sql,{ data->
if(!loc && data.location){
if(name == data.name){
loc = data.location
}
}
})
return loc;
}
}
@@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
@@ -75,6 +75,10 @@ class ExportDistrictRongHe extends BaseBatch{
// exportBigdataModelingBase(lastMonth)
}
//2月8号 临时导出苏州一年的数据
exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
@@ -1304,6 +1308,130 @@ class ExportDistrictRongHe extends BaseBatch{
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportBigdataForsale(publishDate, new_dists){
//["北京","上海"]
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd");
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
def index = 0
citys = ["苏州"]
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " +
"where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')";
allRows = getHouseDatas(sql)
}
if(allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" +
" BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" +
" IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " +
"from bigdata_forsale_gj where distid in ${newstr}"
def rets2 = getHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true);
}
}
println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......"
// CcbFtpUtils.ftpUpload(null, fileName, curDate)
// println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def getHouseDatas(sql){
def allRows = []
def index = 0
sqlInstance.eachRow(sql,{r->
index = index+1
println r.CITYNAME +" "+index
def data = []
data << r.CITYID
data << r.FORSALEID
data << r.PROPCODE
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.UNITPRICE
data << r.TOTALPRICE
data << r.PAYTYPE
data << r.FLOORNUM
data << r.FLOORCNT
data << r.BUILDTYPE
data << r.HOUSESTRU
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEASPECT
data << r.FITMENT
data << r.IFELEVATOR
data << r.ELEVATORHOUSE
data << r.PUBLISHTIME
data << r.EXPIRETIME
data << r.IFFIRST
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.PUBLISHER
data << r.PHONE
data << ""
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.ATTACH
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
// allRows.clear()
// }
});
return allRows
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
@@ -0,0 +1,82 @@
package com.fangjia.script.chongqing
import com.sun.org.apache.xml.internal.security.utils.Base64;
import org.apache.commons.lang3.StringUtils;
import javax.crypto.Cipher;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.DESedeKeySpec;
import javax.crypto.spec.IvParameterSpec;
import java.security.Key;
class Des3 {
private final static String encoding = "UTF-8";
static main(args) {
def task = new Des3()
task.run()
}
def run(){
try {
String key = "111036369260679051122113";
String iv = "23235645";//IV length must be 8 bytes long
//加密
String encryptStr = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"//encode("test", key, iv);
System.out.println("密文: " + encryptStr);
//解密
String decryptStr = decode(encryptStr, key, iv);
System.out.println("明文: " + decryptStr);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 方法描述:3DES加密
* @author hyang
*
* @param plainText 明文
* @param secretKey 密钥
* @param iv 加密向量
* @return String 密文
* @throws Exception
*/
public static String encode(String plainText, String secretKey, String iv)
throws Exception {
Key deskey = null;
DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes());
SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede");
deskey = keyfactory.generateSecret(spec);
Cipher cipher = Cipher.getInstance("desede/CBC/PKCS5Padding");
IvParameterSpec ips = new IvParameterSpec(iv.getBytes());
cipher.init(Cipher.ENCRYPT_MODE, deskey, ips);
byte[] encryptData = cipher.doFinal(plainText.getBytes(encoding));
return Base64.encode(encryptData);
}
/**
* 方法描述: 3DES解密
* @author hyang
*
* @param encryptText 密文
* @param secretKey 密钥
* @param iv 加密向量
* @return String 明文
* @throws Exception
*/
public static String decode(String encryptText, String secretKey, String iv)
throws Exception {
if (StringUtils.isBlank(encryptText) || StringUtils.isBlank(secretKey)) {
return "";
}
Key deskey = null;
DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes());
SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede");
deskey = keyfactory.generateSecret(spec);
Cipher cipher = Cipher.getInstance("desede/CBC/NoPadding"); //desede/CBC/NoPadding 无填充 desede/CBC/PKCS5Padding 加密需要填充,解密一般加不加都结果一致
IvParameterSpec ips = new IvParameterSpec(iv.getBytes());
cipher.init(Cipher.DECRYPT_MODE, deskey, ips);
byte[] decryptData = cipher.doFinal(Base64.decode(encryptText));
return new String(decryptData, encoding).trim();
}
}
@@ -0,0 +1,100 @@
package com.fangjia.script.chongqing
import com.fangjia.SecretUtils
import javax.crypto.Cipher
import javax.crypto.SecretKey
import javax.crypto.spec.SecretKeySpec
class Des3Test {
//定义加密算法,有DES、DESede(即3DES)、Blowfish
private static final String Algorithm = "DESede";
private static final String PASSWORD_CRYPT_KEY = "111036369260679051122113";
/**
* 加密方法
* @param src 源数据的字节数组
* @return
*/
public static byte[] encryptMode(byte[] src) {
try {
SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); //生成密钥
Cipher c1 = Cipher.getInstance(Algorithm); //实例化负责加密/解密的Cipher工具类
c1.init(Cipher.ENCRYPT_MODE, deskey); //初始化为加密模式
return c1.doFinal(src);
} catch (java.security.NoSuchAlgorithmException e1) {
e1.printStackTrace();
} catch (javax.crypto.NoSuchPaddingException e2) {
e2.printStackTrace();
} catch (java.lang.Exception e3) {
e3.printStackTrace();
}
return null;
}
/**
* 解密函数
* @param src 密文的字节数组
* @return
*/
public static byte[] decryptMode(byte[] src) {
try {
SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm);
Cipher c1 = Cipher.getInstance(Algorithm);
c1.init(Cipher.DECRYPT_MODE, deskey); //初始化为解密模式
return c1.doFinal(src);
} catch (java.security.NoSuchAlgorithmException e1) {
e1.printStackTrace();
} catch (javax.crypto.NoSuchPaddingException e2) {
e2.printStackTrace();
} catch (java.lang.Exception e3) {
e3.printStackTrace();
}
return null;
}
/*
* 根据字符串生成密钥字节数组
* @param keyStr 密钥字符串
* @return
* @throws UnsupportedEncodingException
*/
public static byte[] build3DesKey(String keyStr) throws UnsupportedEncodingException{
byte[] key = new byte[24]; //声明一个24位的字节数组,默认里面都是0
byte[] temp = keyStr.getBytes("UTF-8"); //将字符串转成字节数组
/*
* 执行数组拷贝
* System.arraycopy(源数组,从源数组哪里开始拷贝,目标数组,拷贝多少位)
*/
if(key.length > temp.length){
//如果temp不够24位,则拷贝temp数组整个长度的内容到key数组中
System.arraycopy(temp, 0, key, 0, temp.length);
}else{
//如果temp大于24位,则拷贝temp数组24个长度的内容到key数组中
System.arraycopy(temp, 0, key, 0, key.length);
}
return key;
}
/**
* @param args
*/
public static void main(String[] args) {
String msg = "3DES加密解密案例";
System.out.println("【加密前】:" + msg);
//加密
// byte[] secretArr = Des3Test.encryptMode(msg.getBytes());
// System.out.println("【加密后】:" + new String(secretArr));
String s = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"
//解密
byte[] myMsgArr = Des3Test.decryptMode(s.getBytes("UTF-8"));
System.out.println("【解密后】:" + new String(myMsgArr));
}
}