Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj
# Conflicts: # projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy
This commit is contained in:
@@ -25,8 +25,6 @@ import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import com.hankcs.hanlp.HanLP;
|
||||
import com.hankcs.hanlp.dictionary.py.Pinyin;
|
||||
import com.sun.javafx.scene.transform.TransformUtils;
|
||||
import net.sf.json.JSON;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONNull;
|
||||
import net.sf.json.JSONObject;
|
||||
|
||||
+16
-30
@@ -8,6 +8,7 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
@@ -35,7 +36,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys_92;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
def rows = []
|
||||
|
||||
@@ -107,7 +108,6 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
@@ -175,38 +175,25 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
ret << "pa101"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
|
||||
ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
|
||||
def jsonObj = null
|
||||
if(dist.address){
|
||||
jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
|
||||
}
|
||||
if(jsonObj && jsonObj.result && jsonObj.result.street){
|
||||
if(jsonObj.result.street){
|
||||
ret << jsonObj.result.street
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(jsonObj.result.streetNumber){
|
||||
ret << jsonObj.result.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = null
|
||||
if(dist.address){
|
||||
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
|
||||
}
|
||||
if(jsonObj && jsonObj.street){
|
||||
if(jsonObj.street){
|
||||
ret << jsonObj.street
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(jsonObj.streetNumber){
|
||||
ret << jsonObj.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
@@ -225,7 +212,6 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
println dist.city + " "+dist.name+" error"
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
|
||||
-1
@@ -448,7 +448,6 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
|
||||
+17
-1
@@ -11,7 +11,7 @@ class CcbFtpTestUtils {
|
||||
static def ftpPort = 17495
|
||||
static def ftpUser = "ccb"
|
||||
static def ftpPassword = "ccb!@#"
|
||||
static def ftpPath = "/home/ccb/output/"
|
||||
static def ftpPath = "/home/ccb_evaluation/output/"
|
||||
|
||||
static def ftpPath_rh = "/home/ccb_rh/output/"
|
||||
|
||||
@@ -70,6 +70,22 @@ class CcbFtpTestUtils {
|
||||
}
|
||||
|
||||
|
||||
static def listFileNames(month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
|
||||
return files
|
||||
}
|
||||
|
||||
static def getFileLength(String path, String month){
|
||||
sftpChannel = reConnect();
|
||||
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
|
||||
return size
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
|
||||
+8
@@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jiangnanblank.SFTP
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import com.jcraft.jsch.SftpATTRS
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class CcbFtpUtils {
|
||||
@@ -64,6 +65,13 @@ class CcbFtpUtils {
|
||||
return files
|
||||
}
|
||||
|
||||
static def getFileLength(String path, String month){
|
||||
sftpChannel = reConnect();
|
||||
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
|
||||
return size
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
|
||||
+24
-24
@@ -29,28 +29,28 @@ class ExportOracleData {
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonthStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
def new_dists = []
|
||||
//不会变
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()//
|
||||
// exportBigdataDistrictData()//
|
||||
//
|
||||
// exportGuidedPrice()//随人工改变//
|
||||
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
|
||||
//
|
||||
//// //处理小区情况
|
||||
// exportBigdataCommunity()//
|
||||
// exportCommunityStaging()//没cityid
|
||||
// exportCommunityBuileType()//
|
||||
//不会变 10.25
|
||||
exportEvaluationRateData()
|
||||
exportBigdataPoiData()//
|
||||
exportBigdataDistrictData()//
|
||||
|
||||
// exportThirdPartyPrice2(lastMonth, new_dists)//
|
||||
// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
|
||||
exportGuidedPrice()//随人工改变//
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
|
||||
|
||||
// //处理小区情况
|
||||
exportBigdataCommunity()//
|
||||
exportCommunityStaging()//没cityid
|
||||
exportCommunityBuileType()//
|
||||
|
||||
exportThirdPartyPrice2(lastMonth, new_dists)//
|
||||
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
|
||||
exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
|
||||
|
||||
//第三方网站权重表
|
||||
// exportWebsiteRate();
|
||||
|
||||
|
||||
//计算结果相关
|
||||
exportWebsiteRate();
|
||||
//
|
||||
//
|
||||
// //计算结果相关
|
||||
// exportAvmEvaluationGJNB(curMonth)
|
||||
// exportAVMEvaluationItemGJNB(curMonth)
|
||||
//
|
||||
@@ -62,12 +62,12 @@ class ExportOracleData {
|
||||
//
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
//ok文件
|
||||
exportLastFile()
|
||||
//
|
||||
// def curDate = curMonth.toString()+"28"
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
//// ok文件
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
|
||||
|
||||
+1171
File diff suppressed because it is too large
Load Diff
+5
-4
@@ -12,13 +12,14 @@ class ExportOracleEvaluationData {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def citys = null
|
||||
static main(args) {
|
||||
def task = new ExportOracleEvaluationData()
|
||||
task.run()
|
||||
task.run(args)
|
||||
}
|
||||
|
||||
def run() {
|
||||
|
||||
def run(args) {
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
//计算结果相关
|
||||
@@ -369,7 +370,7 @@ class ExportOracleEvaluationData {
|
||||
|
||||
def exportCommCaseDetailGj(month){
|
||||
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
|
||||
+2
-2
@@ -36,7 +36,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.citys_252;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
for (def city : citys) {
|
||||
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2021-09-01", "2022-10-12")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-22", "2022-10-27")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+14
-14
@@ -1001,10 +1001,10 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入人工查价_0922_300.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
|
||||
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
// def region = RegionData.formatRegion(d['A'], d['B'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
@@ -1013,18 +1013,18 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
// d['F'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
//// if(!d['Q']){
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//////
|
||||
//// if(dist){
|
||||
//// d['H'] = dist.region
|
||||
////// d['G'] = dist.name
|
||||
//// d['I'] = dist.status
|
||||
//// }
|
||||
//// }
|
||||
if(!d['F']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
if(dist){
|
||||
// d['H'] = dist.region
|
||||
// d['G'] = dist.name
|
||||
d['F'] = dist.status
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
|
||||
|
||||
+146
@@ -0,0 +1,146 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AddressCut extends BaseBatch{
|
||||
static main(args) {
|
||||
runBatch(new AddressCut(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
sameAddr()
|
||||
}
|
||||
|
||||
def sameAddr() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def dataMap = [:]
|
||||
def index = 0
|
||||
for (def c : citys) {
|
||||
index++
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "address"])
|
||||
|
||||
def addrMap = [:]
|
||||
for (def dist : dists) {
|
||||
if (dist.address) {
|
||||
def rets = addrMap.get(dist.address)
|
||||
if (!rets) {
|
||||
rets = []
|
||||
}
|
||||
rets << dist
|
||||
addrMap.put(dist.address, rets)
|
||||
}
|
||||
}
|
||||
dataMap.put(c, addrMap)
|
||||
println index +" "+c
|
||||
}
|
||||
|
||||
|
||||
def sameAddrList = []
|
||||
for(def c : citys) {
|
||||
def cityMap = dataMap.get(c)
|
||||
if(cityMap){
|
||||
cityMap.each{addr,rets->
|
||||
if(rets && rets.size()>1){
|
||||
def keys = new HashSet()
|
||||
for(def d : rets){
|
||||
if(d.name){
|
||||
def name = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("七期","").replace("八期","").replace("九期","").replace("十期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("六区","").replace("七区","").replace("八区","").replace("九区","").replace("十区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
|
||||
.replace("十一街区","").replace("十二街区","").replace("十三街区","").replace("十四街区","").replace("十五街区","").replace("十六街区","").replace("十七街区","").replace("十八街区","").replace("十九街区","").replace("一街区","").replace("二街区","").replace("三街区","").replace("四街区","").replace("五街区","").replace("六街区","").replace("七街区","").replace("八街区","").replace("九街区","").replace("十街区","")
|
||||
.replace("公寓","").replace("别墅","").replace("商住楼","").replace("花园","").replace("南区","").replace("北区","").replace("东区","").replace("西区","")
|
||||
keys << d.region +"##"+ name
|
||||
}
|
||||
}
|
||||
if(keys && keys.size()>1){
|
||||
for(def d : rets) {
|
||||
if (d.name) {
|
||||
sameAddrList << [d.city, d.region, d.name, d.address]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
//ExcelUtils.write("D:\\数据\\小区\\同地址小区.xlsx", sameAddrList, "相同地址小区")
|
||||
FileUtils.batchCreateFile(sameAddrList, "D:\\数据\\小区\\同地址小区.txt", null, "utf-8", false);
|
||||
println "完成"
|
||||
}
|
||||
|
||||
|
||||
def addrCut(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
|
||||
def fileName = "/home/goojia/gj/address_split.txt"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\address_split.txt"
|
||||
}
|
||||
for(def c : citys) {
|
||||
index++
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name","address"])
|
||||
def rets = cut(dists)
|
||||
FileUtils.batchCreateFile(rets, fileName, null, "utf-8", true);
|
||||
println index +" "+ c +" "+ rets.size()
|
||||
}
|
||||
}
|
||||
|
||||
def cut(rets){
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
def totalCount = new AtomicLong()
|
||||
def rows = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def d = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(d.address){
|
||||
def jsonObj = FangjiaClientUtils.addressStructure(d.city, d.address)
|
||||
if(jsonObj && jsonObj.result){
|
||||
def udata = new BasicDBObject()
|
||||
def street = null
|
||||
def street_number = null
|
||||
if(jsonObj.result.street){
|
||||
street = jsonObj.result.street
|
||||
}
|
||||
if(jsonObj.result.streetNumber){
|
||||
street_number = jsonObj.result.streetNumber
|
||||
}
|
||||
if(street || street_number){
|
||||
|
||||
println totalCount.get() +" "+d.city +" "+d.address +" "+street+" "+street_number
|
||||
|
||||
rows << [d.city,d.region, d.name, d.address, street, street_number]
|
||||
// udata.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package com.fangjia.script.ccbevaluation.ronghe
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
|
||||
import com.jcraft.jsch.ChannelExec;
|
||||
import com.jcraft.jsch.JSch;
|
||||
import com.jcraft.jsch.Session
|
||||
|
||||
import com.jcraft.jsch.*;
|
||||
|
||||
|
||||
class SSHHelperTest {
|
||||
|
||||
private Session session = null;
|
||||
private Channel channel = null;
|
||||
|
||||
private String sftpHost = "192.168.0.86";
|
||||
private int sftpPort = 17495;
|
||||
private String sftpUserName = "ccb";
|
||||
private String sftpPassword = "ccb!@#";
|
||||
private int timeout = 30000;
|
||||
/**
|
||||
* 获取连接
|
||||
* @return
|
||||
*/
|
||||
private ChannelExec getChannelExec() {
|
||||
try {
|
||||
if (channel != null && channel.isConnected()) {
|
||||
return (ChannelExec) channel;
|
||||
}
|
||||
JSch jSch = new JSch();
|
||||
if (session == null || !session.isConnected()) {
|
||||
session = jSch.getSession(sftpUserName, sftpHost, sftpPort);
|
||||
session.setPassword(sftpPassword);
|
||||
Properties config = new Properties();
|
||||
config.put("StrictHostKeyChecking", "no");
|
||||
session.setConfig(config);
|
||||
session.setTimeout(timeout);
|
||||
session.connect();
|
||||
}
|
||||
channel = session.openChannel("exec");
|
||||
} catch (Exception e) {
|
||||
if (session != null) {
|
||||
session.disconnect();
|
||||
session = null;
|
||||
}
|
||||
channel = null;
|
||||
}
|
||||
return channel == null ? null : (ChannelExec) channel;
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭连接
|
||||
*/
|
||||
private void closeChannel() {
|
||||
try {
|
||||
if (channel != null) {
|
||||
channel.disconnect();
|
||||
channel = null;
|
||||
}
|
||||
if (session != null) {
|
||||
session.disconnect();
|
||||
session = null;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println(e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 执行服务器端命令
|
||||
*/
|
||||
public boolean executeCommand(String command) {
|
||||
boolean flag = false;
|
||||
ChannelExec channelExec = getChannelExec();
|
||||
if (channelExec == null) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
channelExec.setInputStream(null);
|
||||
channelExec.setErrStream(System.err);
|
||||
channelExec.setCommand(command);
|
||||
|
||||
InputStream ins = channelExec.getInputStream();
|
||||
channelExec.connect();
|
||||
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(ins, "utf-8"));
|
||||
String line = null;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
System.out.println(line);
|
||||
}
|
||||
|
||||
reader.close();
|
||||
closeChannel();
|
||||
|
||||
flag = true;
|
||||
} catch (Exception e) {
|
||||
System.out.println(e);
|
||||
flag = false;
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110"));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+24
-1
@@ -29,7 +29,30 @@ class DataMatch {
|
||||
// task.run3()
|
||||
// println task.findAddress("昌州街道棠香路")
|
||||
|
||||
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
task.fzWeiyuanshuMatch("D:\\数据\\福州\\连江楼盘.xlsx")
|
||||
}
|
||||
|
||||
def fzWeiyuanshuMatch(fileName){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
// if(!d['G']){
|
||||
// def arr = OpenApiClient.search("福州", "连江", d['C'])
|
||||
// if(arr){
|
||||
// d['E'] = arr.city
|
||||
// d['F'] = arr.district
|
||||
// d['G'] = arr.name
|
||||
// }
|
||||
// }
|
||||
|
||||
def results = null
|
||||
if(d['H'] && d['H']=="F" && d['C']){
|
||||
results = OpenApiClient.addressMatch("福州",d['C'], "property", false, "连江", null)
|
||||
}
|
||||
if(results && results.precise>=0){
|
||||
d['J'] = results.district
|
||||
d['K'] = results.name
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
|
||||
+17
@@ -67,6 +67,7 @@ class FangjiaClientUtils {
|
||||
|
||||
|
||||
|
||||
//分词 省,市,区
|
||||
public static def addressStand(def city, def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
@@ -84,6 +85,22 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
//分词基础上的标准化,complete=false 不不全 short=true 省市区返回简拼
|
||||
public static def addressStructure(def city, def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure/v2?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def arroundPrice(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
|
||||
+5
-5
@@ -38,8 +38,8 @@ class PaimaiDataSend {
|
||||
// task.jdRunByMongo()
|
||||
|
||||
// task.aliZcRun()
|
||||
task.aliZcRun2()
|
||||
// task.aliFpRun()
|
||||
// task.aliZcRun2()
|
||||
task.aliFpRun()
|
||||
// task.rmfysszcRun()
|
||||
// task.jdRun()
|
||||
// task.checkUrl()
|
||||
@@ -334,10 +334,10 @@ class PaimaiDataSend {
|
||||
def index = 0
|
||||
def q = new BasicDBObject()
|
||||
// q.put("city",new BasicDBObject('$in', ["南京","宿迁","苏州"]))
|
||||
q.put("province","江苏")
|
||||
q.put("sub_type","土地")
|
||||
// q.put("province","江苏")
|
||||
// q.put("sub_type","土地")
|
||||
// q.put("parser_status", new BasicDBObject('$ne',2))
|
||||
q.put("url","https://susong-item.taobao.com/auction/671320162557.htm")
|
||||
q.put("url","https://sf-item.taobao.com/sf_item/673646300374.htm")
|
||||
//https://sf-item.taobao.com/sf_item/657131607486.htm
|
||||
// q.put("parser_status", new BasicDBObject('$exists',false))
|
||||
// q.put("finishDate", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2022-09-01")))
|
||||
|
||||
@@ -209,4 +209,20 @@ class SFTP {
|
||||
return isDirExistFlag;
|
||||
}
|
||||
|
||||
public long getFileSize(String srcSftpFilePath, ChannelSftp sftp) {
|
||||
long fileSize;//文件大于等于0则存在
|
||||
try {
|
||||
SftpATTRS sftpATTRS = sftp.lstat(srcSftpFilePath);
|
||||
fileSize = sftpATTRS.getSize();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
fileSize = -1;//获取文件大小异常
|
||||
if (e.getMessage().toLowerCase().equals("no such file")) {
|
||||
fileSize = -2;//文件不存在
|
||||
}
|
||||
}
|
||||
return fileSize;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+11
-9
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.temp.shenjian.poi
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.script.temp.shenjian.util.BaiDuUtils
|
||||
import com.fangjia.script.temp.shenjian.util.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
@@ -15,25 +16,26 @@ import java.util.concurrent.Executors
|
||||
*/
|
||||
class FillLngLat {
|
||||
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(4)
|
||||
// static ExecutorService executorService = Executors.newFixedThreadPool(4)
|
||||
static void main(String[] args) {
|
||||
ElasticUtils.executeSqlQuery("select * from dws_poi where location is not null", { dataT ->
|
||||
ElasticUtils.executeSqlQuery("select * from dws_poi where location is null", { dataT ->
|
||||
final def data = dataT
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
// executorService.submit(new Runnable() {
|
||||
// @Override
|
||||
// void run() {
|
||||
println(data)
|
||||
def lngLat = GaoDeUtils.parserByGaode(data.city, data.region + data.name + data.address)
|
||||
def lngLat = BaiDuUtils.parserByBaidu(data.city, data.region + data.name + data.address)
|
||||
Thread.sleep(1000)
|
||||
if (lngLat != null && lngLat.size() == 2 ) {
|
||||
JSONObject jsonObject = new JSONObject()
|
||||
jsonObject.put("lon", Double.valueOf(lngLat['lng']))
|
||||
jsonObject.put("lat", Double.valueOf(lngLat['lat']))
|
||||
EsUtils.bulkUpdateFieldByIds("dws_poi", "location", jsonObject, Arrays.asList(data.source_id))
|
||||
}
|
||||
}
|
||||
})
|
||||
// }
|
||||
// })
|
||||
|
||||
})
|
||||
executorService.shutdown()
|
||||
//executorService.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package com.fangjia.script.temp.shenjian.util
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
/**
|
||||
* 百度工具类
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2022/10/28
|
||||
*/
|
||||
class BaiDuUtils {
|
||||
|
||||
def static final String KEY = "6w9SEHS3hnAFjmY2oEhqIDdgvLPKWXu5"
|
||||
|
||||
/**
|
||||
* 百度地址逆解析,返回高德坐标!!!!!!!!!!!!!!!
|
||||
* @param city
|
||||
* @param address
|
||||
*/
|
||||
def static parserByBaidu(def city, def address) {
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("city", city)
|
||||
if (StringUtils.isBlank(address)) {
|
||||
return []
|
||||
}
|
||||
params.put("address", address.replaceAll("null", ""))
|
||||
params.put("output", "json")
|
||||
params.put("ak", KEY)
|
||||
params.put("callback", "showLocation")
|
||||
|
||||
String content
|
||||
content = new HttpRequest().get("https://api.map.baidu.com/geocoding/v3/", params, "utf-8")
|
||||
//content = "showLocation&&showLocation({\"status\":0,\"result\":{\"location\":{\"lng\":131.04009309945693,\"lat\":45.78426770180344},\"precise\":1,\"confidence\":70,\"comprehension\":100,\"level\":\"地产小区\"}})"
|
||||
content = content.replaceAll("showLocation&&showLocation\\(", "").replaceAll("\\)", "")
|
||||
JSONObject json = JSONObject.parseObject(content)
|
||||
if (json.status == 0 && json.result && json.result.location) {
|
||||
def bd_lng = json.result.location.lng
|
||||
def bd_lat = json.result.location.lat
|
||||
Map<String, Double> locationMap = GeoUtils.bd09ToGcj02(bd_lat, bd_lng)
|
||||
return ["lng": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lng']), "lat": com.fangjia.core.util.DataUtil.getDoubleRound(6, locationMap['lat'])]
|
||||
}
|
||||
return []
|
||||
}
|
||||
}
|
||||
+16
-13
@@ -91,11 +91,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||
|
||||
cityMap.each{city,alias->
|
||||
// if(city == "北京"){
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
|
||||
while(flag){
|
||||
@@ -122,9 +120,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
def final AtomicInteger successIndex = new AtomicInteger(0)
|
||||
datas.each { dbo ->
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
||||
if(dbo.name){
|
||||
@@ -164,12 +162,12 @@ class DataStandardBatch extends BaseBatch{
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
}
|
||||
|
||||
latch.await()
|
||||
// latch.await()
|
||||
long end = System.currentTimeMillis()
|
||||
|
||||
println "第【${rowIndex}】次中匹配成功的:"+successCount+" 个,耗时:"+ (end - start)+" "+ DateUtils.format(new Date(),"yyyy-MM-dd HH:mm:ss")
|
||||
@@ -502,6 +500,8 @@ class DataStandardBatch extends BaseBatch{
|
||||
type = 12;//区域不同
|
||||
district = null;
|
||||
}else{
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(district.name, copyObj.name)
|
||||
district.score = score //此处用来做测试
|
||||
type=15
|
||||
}
|
||||
}
|
||||
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
// qdata.put("city","北京")
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("金隅澜湾.*"))
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
|
||||
def ret = []
|
||||
try{
|
||||
@@ -1065,6 +1065,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
udata2.put("match_region", district.region);
|
||||
udata2.put("match_name", district.name);
|
||||
udata2.put("match_address", district.address);
|
||||
if(district.score){
|
||||
udata2.put("score", district.score);
|
||||
}
|
||||
udata2.put("avg_price", data.avg_price);
|
||||
udata2.put("type", type);
|
||||
udata2.put("tid", data._id.toString());
|
||||
|
||||
+15
-14
@@ -81,8 +81,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
println "小区添加价格完成"
|
||||
districts = addOtherWebsitePrice(districts);
|
||||
println "小区添加第三方价格完成"
|
||||
districts = addLevel(districts);
|
||||
println "Ð¡ÇøÆÀ¼¶Íê³É"
|
||||
// districts = addLevel(districts);
|
||||
// println "Ð¡ÇøÆÀ¼¶Íê³É"
|
||||
saveOrUpdate(districts);
|
||||
println city + "保存修改完成"
|
||||
toLineDB(city)
|
||||
@@ -101,16 +101,16 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
def tempPrice = d.get(field+"_price")
|
||||
if(!tempPrice){
|
||||
d.put(field+"_price", m.avg_price.toInteger())
|
||||
if(d.fangjia_price && m.avg_price){
|
||||
if(d.fangjia_price){
|
||||
try{
|
||||
d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger()))
|
||||
}catch(Exception e){
|
||||
println "dddd"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
// if(d.fangjia_price && m.avg_price){
|
||||
// if(d.fangjia_price){
|
||||
// try{
|
||||
// d.put(field+"_diff", diffPercent(d.fangjia_price.toInteger(), m.avg_price.toInteger()))
|
||||
// }catch(Exception e){
|
||||
// println "dddd"
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
}else{
|
||||
@@ -208,8 +208,9 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
query.put("s_date", SHORT_DATE)
|
||||
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("level",d.level);
|
||||
udata.put("last_price", d.last_price);
|
||||
// udata.put("level",d.level);
|
||||
// udata.put("last_price", d.last_price);
|
||||
udata.put("seaweed_id", d._id.toString())
|
||||
udata.put("fangjia_price", d.fangjia_price);
|
||||
|
||||
fieldMap.each{k,v->
|
||||
|
||||
@@ -52,6 +52,22 @@ public class OpenApiClient {
|
||||
return false
|
||||
}
|
||||
|
||||
public static def search(def city, def district, def name){
|
||||
Map<String, String> params = new HashMap<String, String>()
|
||||
params.put("city", city);
|
||||
params.put("keyword", name);
|
||||
params.put("district", district)
|
||||
params.put("token", TOKEN);
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/property/search", params, "utf-8")
|
||||
JSONObject jsonObj = JSONObject.fromObject(content)
|
||||
JSONArray array = jsonObj.get("result")
|
||||
def matchObj = null
|
||||
if(array){
|
||||
matchObj = array[0]
|
||||
}
|
||||
return matchObj
|
||||
}
|
||||
|
||||
public static List searchDistrict(double lng, double lat, int r, String city) {
|
||||
DecimalFormat format = new DecimalFormat("#.0000000000");
|
||||
try {
|
||||
@@ -348,6 +364,7 @@ public class OpenApiClient {
|
||||
}
|
||||
|
||||
|
||||
//此接口地址标准化,带补全
|
||||
public static JSONObject standard1Action(String city, String address) {
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
|
||||
Reference in New Issue
Block a user