diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy new file mode 100644 index 000000000..857b62715 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TestData.groovy @@ -0,0 +1,115 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +class TestData { + + static main(args) { + def task = new TestData() +// task.run() + task.update() + } + + def update(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\龙岩1.xlsx", { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + if(d['E'] && d['G']=="人工"){ + def q = new BasicDBObject() + def baidu = BaiDuApiUtils.geoconv(d['E'].toString().toDouble(), d['F'].toString().toDouble(), 3) + def lng2 = baidu.x + def lat2 = baidu.y + + Long dLat2 = lat2 * 10000000000L + Long dLng2 = lng2 * 10000000000L + + q.put("lat2", dLat2) + q.put("lng2", dLng2) + + Long lng = d['E'].toString().toDouble() * 10000000000L + Long lat = d['F'].toString().toDouble() * 10000000000L + q.put("lng", lng) + q.put("lat", lat) + DbUtils.seaweed.update(new BasicDBObject('_id',d['A']),new BasicDBObject('$set',q)) + } + } + }); + } + + def run(){ + ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\龙岩.xlsx", { d -> + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def loc = getLocation(dist.city, dist.region, dist.name) + if(loc){ + d['E'] = loc.x + d['F'] = loc.y + println d['C']+" "+loc.x+" "+loc.y + } + } + +// if(!d['E']){ +// def loc = getAmapLocation("龙岩市", d['B'], d['C']) +// if(loc){ +// d['E'] = loc.x +// d['F'] = loc.y +// } +// } + +// if(!d['E']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// def loc = getAmapOldLocation(dist.city, dist.region, d['C']) +// if(loc){ +// d['E'] = loc.x +// d['F'] = loc.y +// } +// } +// +// } + + }); + } + + def getLocation(city, region, name){ + String sql = "select * from community where city='${city}' and region='${region}' and name='${name}' " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + loc = data.location + } + }) + return loc; + } + + def getAmapLocation(city, region, name){ + String sql = "select * from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + if(name == data.name){ + loc = data.location + } + } + }) + return loc; + } + + def getAmapOldLocation(city, region, name){ + String sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') " + def loc = null + ElasticUtils.executeSqlQuery(sql,{ data-> + if(!loc && data.location){ + if(name == data.name){ + loc = data.location + } + } + }) + return loc; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 8fe64bdef..d65a4e181 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -38,7 +38,7 @@ class ExportDistrictRongHe extends BaseBatch{ def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) + // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists) if(curDate.toString().endsWith("10")){ @@ -75,6 +75,10 @@ class ExportDistrictRongHe extends BaseBatch{ // exportBigdataModelingBase(lastMonth) } + + //2月8号 临时导出苏州一年的数据 + exportBigdataForsale("2022-01-01", new_dists) //现在按updatetime导出 + SSHHelper jschCommand = new SSHHelper(); System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); @@ -1304,6 +1308,130 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } + def exportBigdataForsale(publishDate, new_dists){ + //["北京","上海"] + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"); + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] + def index = 0 + citys = ["苏州"] + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH " + + "from bigdata_forsale_gj where distid in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true); + } + } + println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......" +// CcbFtpUtils.ftpUpload(null, fileName, curDate) +// println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.CITYNAME +" "+index + def data = [] + data << r.CITYID + data << r.FORSALEID + data << r.PROPCODE + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.UNITPRICE + data << r.TOTALPRICE + data << r.PAYTYPE + data << r.FLOORNUM + data << r.FLOORCNT + data << r.BUILDTYPE + data << r.HOUSESTRU + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEASPECT + data << r.FITMENT + data << r.IFELEVATOR + data << r.ELEVATORHOUSE + data << r.PUBLISHTIME + data << r.EXPIRETIME + data << r.IFFIRST + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.PUBLISHER + data << r.PHONE + data << "" + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.ATTACH + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + def exportOk(curDate){ // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy new file mode 100644 index 000000000..012ddd921 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3.groovy @@ -0,0 +1,82 @@ +package com.fangjia.script.chongqing +import com.sun.org.apache.xml.internal.security.utils.Base64; +import org.apache.commons.lang3.StringUtils; + +import javax.crypto.Cipher; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.DESedeKeySpec; +import javax.crypto.spec.IvParameterSpec; +import java.security.Key; + +class Des3 { + private final static String encoding = "UTF-8"; + + static main(args) { + def task = new Des3() + task.run() + } + def run(){ + try { + String key = "111036369260679051122113"; + String iv = "23235645";//IV length must be 8 bytes long + //加密 + String encryptStr = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0"//encode("test", key, iv); + System.out.println("密文: " + encryptStr); + //解密 + String decryptStr = decode(encryptStr, key, iv); + System.out.println("明文: " + decryptStr); + } catch (Exception e) { + e.printStackTrace(); + } + } + + /** + * 方法描述:3DES加密 + * @author hyang + * + * @param plainText 明文 + * @param secretKey 密钥 + * @param iv 加密向量 + * @return String 密文 + * @throws Exception + */ + public static String encode(String plainText, String secretKey, String iv) + throws Exception { + Key deskey = null; + DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes()); + SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede"); + deskey = keyfactory.generateSecret(spec); + Cipher cipher = Cipher.getInstance("desede/CBC/PKCS5Padding"); + IvParameterSpec ips = new IvParameterSpec(iv.getBytes()); + cipher.init(Cipher.ENCRYPT_MODE, deskey, ips); + byte[] encryptData = cipher.doFinal(plainText.getBytes(encoding)); + return Base64.encode(encryptData); + } + + /** + * 方法描述: 3DES解密 + * @author hyang + * + * @param encryptText 密文 + * @param secretKey 密钥 + * @param iv 加密向量 + * @return String 明文 + * @throws Exception + */ + public static String decode(String encryptText, String secretKey, String iv) + throws Exception { + if (StringUtils.isBlank(encryptText) || StringUtils.isBlank(secretKey)) { + return ""; + } + Key deskey = null; + DESedeKeySpec spec = new DESedeKeySpec(secretKey.getBytes()); + SecretKeyFactory keyfactory = SecretKeyFactory.getInstance("desede"); + deskey = keyfactory.generateSecret(spec); + Cipher cipher = Cipher.getInstance("desede/CBC/NoPadding"); //desede/CBC/NoPadding 无填充 desede/CBC/PKCS5Padding 加密需要填充,解密一般加不加都结果一致 + IvParameterSpec ips = new IvParameterSpec(iv.getBytes()); + cipher.init(Cipher.DECRYPT_MODE, deskey, ips); + byte[] decryptData = cipher.doFinal(Base64.decode(encryptText)); + return new String(decryptData, encoding).trim(); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy new file mode 100644 index 000000000..a9a798124 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/Des3Test.groovy @@ -0,0 +1,100 @@ +package com.fangjia.script.chongqing + +import com.fangjia.SecretUtils + +import javax.crypto.Cipher +import javax.crypto.SecretKey +import javax.crypto.spec.SecretKeySpec + +class Des3Test { + //定义加密算法,有DES、DESede(即3DES)、Blowfish + private static final String Algorithm = "DESede"; + private static final String PASSWORD_CRYPT_KEY = "111036369260679051122113"; + + + /** + * 加密方法 + * @param src 源数据的字节数组 + * @return + */ + public static byte[] encryptMode(byte[] src) { + try { + SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); //生成密钥 + Cipher c1 = Cipher.getInstance(Algorithm); //实例化负责加密/解密的Cipher工具类 + c1.init(Cipher.ENCRYPT_MODE, deskey); //初始化为加密模式 + return c1.doFinal(src); + } catch (java.security.NoSuchAlgorithmException e1) { + e1.printStackTrace(); + } catch (javax.crypto.NoSuchPaddingException e2) { + e2.printStackTrace(); + } catch (java.lang.Exception e3) { + e3.printStackTrace(); + } + return null; + } + + + /** + * 解密函数 + * @param src 密文的字节数组 + * @return + */ + public static byte[] decryptMode(byte[] src) { + try { + SecretKey deskey = new SecretKeySpec(build3DesKey(PASSWORD_CRYPT_KEY), Algorithm); + Cipher c1 = Cipher.getInstance(Algorithm); + c1.init(Cipher.DECRYPT_MODE, deskey); //初始化为解密模式 + return c1.doFinal(src); + } catch (java.security.NoSuchAlgorithmException e1) { + e1.printStackTrace(); + } catch (javax.crypto.NoSuchPaddingException e2) { + e2.printStackTrace(); + } catch (java.lang.Exception e3) { + e3.printStackTrace(); + } + return null; + } + + + /* + * 根据字符串生成密钥字节数组 + * @param keyStr 密钥字符串 + * @return + * @throws UnsupportedEncodingException + */ + public static byte[] build3DesKey(String keyStr) throws UnsupportedEncodingException{ + byte[] key = new byte[24]; //声明一个24位的字节数组,默认里面都是0 + byte[] temp = keyStr.getBytes("UTF-8"); //将字符串转成字节数组 + + /* + * 执行数组拷贝 + * System.arraycopy(源数组,从源数组哪里开始拷贝,目标数组,拷贝多少位) + */ + if(key.length > temp.length){ + //如果temp不够24位,则拷贝temp数组整个长度的内容到key数组中 + System.arraycopy(temp, 0, key, 0, temp.length); + }else{ + //如果temp大于24位,则拷贝temp数组24个长度的内容到key数组中 + System.arraycopy(temp, 0, key, 0, key.length); + } + return key; + } + + /** + * @param args + */ + public static void main(String[] args) { + String msg = "3DES加密解密案例"; + System.out.println("【加密前】:" + msg); + + //加密 +// byte[] secretArr = Des3Test.encryptMode(msg.getBytes()); +// System.out.println("【加密后】:" + new String(secretArr)); + + String s = "Lk+AtbEWoNF0eQNrSsGoNJjRFuvnCaRKaypcAnaRvGpHtc2KmMOMbufut+JDWP5Rq/4qNAWsEch0iZgE5z2CTnAwCjoRTUl0" + + //解密 + byte[] myMsgArr = Des3Test.decryptMode(s.getBytes("UTF-8")); + System.out.println("【解密后】:" + new String(myMsgArr)); + } +}