融合均价数据连接数据库修改

This commit is contained in:
2022-12-06 14:56:42 +08:00
parent b610026293
commit e0acf09b56
17 changed files with 730 additions and 161 deletions
@@ -17,8 +17,8 @@ class AddThirdPartyPrice {
} }
def run(){ def run(){
//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关" //
def citys = ["林芝"] def citys = ["林芝","海西州","天门","大兴安岭","儋州","鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"]
def rows = [] def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def c : citys){ for(def c : citys){
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) { public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys; // def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args) def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){ if(args && args.size()>=2){
curDate = args[1].toString().toInteger() curDate = args[1].toString().toInteger()
} }
@@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{
} }
def trades = getCityData(city, statMap, ppeDists) // def trades = getCityData(city, statMap, ppeDists)
if(trades){ // if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") // def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets) // rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size() // println "【${index}】"+city +" 成交共:"+rets.size()
//
def cityDatas = [] // def cityDatas = []
cityDatas << headers // cityDatas << headers
cityDatas.addAll(rows) // cityDatas.addAll(rows)
} // }
} }
@@ -52,7 +52,7 @@ class CcbCommonUtils {
def countStr = args[0] def countStr = args[0]
if(countStr=="344"){ if(countStr=="344"){
citys = CcbCommonUtils.citys_all citys = CcbCommonUtils.citys_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="92"){ }else if(countStr=="92"){
citys = CcbCommonUtils.citys_92 citys = CcbCommonUtils.citys_92
@@ -13,7 +13,7 @@ class CcbFtpTestUtils {
static def ftpPassword = "ccb!@#" static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb_evaluation/output/" static def ftpPath = "/home/ccb_evaluation/output/"
static def ftpPath_rh = "/home/ccb_rh/output/" static def ftpPath_rh = "/home/ccb_ronghe/output/"
static SFTP sftp = new SFTP(); static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
@@ -70,16 +70,26 @@ class CcbFtpTestUtils {
} }
static def listFileNames(month){ static def listFileNames(type, month){
//上传到ftp //上传到ftp
sftpChannel = reConnect(); sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); Vector files = null;
if(type && type=="rh"){
files = sftp.listFiles(ftpPath_rh+ month+"/", sftpChannel);
}else{
files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
}
return files return files
} }
static def getFileLength(String path, String month){ static def getFileLength(String type, String path, String month){
sftpChannel = reConnect(); sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) def size = 0;
if(type && type=="rh"){
size = sftp.getFileSize(ftpPath_rh+ month+"/"+path, sftpChannel)
}else{
size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
}
return size return size
} }
@@ -371,11 +371,11 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){ def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_92 def citys = CcbCommonUtils.citys_92
def month = 202209 def month = 202210
def rows = [] def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys) { for(def city : citys) {
def rets = findPeGteThirdParty(city, 202210) def rets = findPeGteThirdParty(city, 202211)
def total = 0 def total = 0
println city +" "+rets.size() println city +" "+rets.size()
for(def m : rets){ for(def m : rets){
@@ -54,7 +54,7 @@ class ExportOracleAllDataTest {
// exportCommunityStaging()//没cityid // exportCommunityStaging()//没cityid
// exportCommunityBuileType()// // exportCommunityBuileType()//
exportThirdPartyPrice2(new_dists)// // exportThirdPartyPrice2(new_dists)//
// exportBigdataForsale(new_dists) //2022-07-19 // exportBigdataForsale(new_dists) //2022-07-19
// exportBigdataDeal( new_dists) //2022-07-01 // exportBigdataDeal( new_dists) //2022-07-01
// //
@@ -325,7 +325,7 @@ class ExportOracleAllDataTest {
def allRows = [] def allRows = []
for(def city : citys) { for(def city : citys) {
def cityCode = cityCodeMap.get(city) def cityCode = cityCodeMap.get(city)
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' ";//and createtime>=to_date('${date}','yyyyMMdd')
sqlInstance.eachRow(sql, { r -> sqlInstance.eachRow(sql, { r ->
def data = [] def data = []
data << r.ID data << r.ID
@@ -1172,14 +1172,14 @@ class ExportOracleAllDataTest {
def exportInOk(){ def exportInOk(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
Vector files = CcbFtpTestUtils.listFileNames(curDate) Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
Iterator fileVec = files.iterator() Iterator fileVec = files.iterator()
def fileRows = [] def fileRows = []
while(fileVec.hasNext()){ while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next() ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
} }
} }
def fileName = "/ccb_evaluation/${curDate}.ok" def fileName = "/ccb_evaluation/${curDate}.ok"
@@ -38,16 +38,17 @@ class GuiDingPriceBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args) def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
def rows = [] def rows = []
CcbCommonUtils.deleteGuidedPrice(curDate) // CcbCommonUtils.deleteGuidedPrice(curDate)
for (def city : citys) { for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价 //添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18") def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){ if(prices){
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -45,7 +45,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def citys = CcbCommonUtils.getRunningCitys(args) def citys = CcbCommonUtils.getRunningCitys(args)
def rows = [] def rows = []
def index = 0 def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date()) def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){ for(def city : citys){
index++ index++
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
} }
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
} }
@@ -40,7 +40,7 @@ class TradeStatLog {
static main(args) { static main(args) {
def task = new TradeStatLog() def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量 // task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交 // task.exportTrade()//导出未入库成交
// task.thirdPartyStat() // task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR) // println Calendar.getInstance().get(Calendar.YEAR)
@@ -50,8 +50,25 @@ class TradeStatLog {
// task.deleteDealData2() // task.deleteDealData2()
// task.statTableCount() // task.statTableCount()
// task.filterTrade() // task.filterTrade()
task.updateDealPriceType()
}
def updateDealPriceType(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
DEAL_PRICE_COLL.find(new BasicDBObject("city", c)).sort(new BasicDBObject("trade_date",-1)).batchSize(5000).each {it->
if(it.total_price instanceof Double){
index ++
println index+" "+it.city+" "+it.total_price+" "+it._id.toString()
def udata = new BasicDBObject()
udata.put("total_price", (int)it.total_price)
DEAL_PRICE_COLL.update(new BasicDBObject("_id", it._id), new BasicDBObject('$set', udata))
}
}
}
} }
def statTableCount(){ def statTableCount(){
@@ -8,6 +8,7 @@ import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql import groovy.sql.Sql
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
@@ -25,7 +26,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args) citys = CcbCommonUtils.getRunningCitys(args)
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def curDate = "20221110"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"] def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹) //按天导(每天一个文件夹)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
@@ -48,17 +49,17 @@ class ExportDistrictRongHeTest extends BaseBatch{
// exportBigdataDistrictFJW()// // exportBigdataDistrictFJW()//
// exportBigdataAliasFJW()// // exportBigdataAliasFJW()//
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() // def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
exportBigdataCityPrice(queryDate) // exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate) // exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate) // exportBigdataCommPrice(queryDate)
exportInOk()
// } // }
SSHHelperTest jschCommand = new SSHHelperTest(); SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
exportOk(curDate)
} }
@@ -66,9 +67,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommunityGJRH(){ def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
} }
def index = 0 def index = 0
for(def city : citys){ for(def city : citys){
@@ -133,9 +134,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
} }
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -162,9 +163,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
} }
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -190,9 +191,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
} }
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
@@ -228,9 +229,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}); });
} }
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -258,9 +259,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
} }
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -291,9 +292,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
} }
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -323,9 +324,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
} }
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
@@ -358,9 +359,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
} }
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -382,9 +383,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
index++ index++
def curStartStr = curStartDateStr def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
} }
def allRows = [] def allRows = []
@@ -482,9 +483,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
} }
def allRows = [] def allRows = []
@@ -675,9 +676,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
allRows << data allRows << data
}); });
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......" println "BIGDATA_DEAL_FJW 导出完成......"
@@ -686,9 +687,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataBuildingFJW(){ def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
} }
def index = 0 def index = 0
for(def city : citys) { for(def city : citys) {
@@ -743,9 +744,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCityPrice(queryDate){ def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
} }
def allRows = [] def allRows = []
for(def city : citys) { for(def city : citys) {
@@ -789,9 +790,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataDistPrice(queryDate){ def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
} }
def allRows = [] def allRows = []
for(def city : citys) { for(def city : citys) {
@@ -836,9 +837,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommPrice(queryDate){ def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
} }
def allRows = [] def allRows = []
for(def city : citys) { for(def city : citys) {
@@ -899,9 +900,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
allRows << data allRows << data
}); });
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -910,9 +911,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataBuildprojFjw(){ def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile([], fileName, [], "utf-8", false); FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -921,23 +922,35 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommlayoutpicFjw(){ def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile([], fileName, [], "utf-8", false); FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
} }
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def exportInOk(){
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
if(CommonDatautils.isWindow()){ Vector files = CcbFtpTestUtils.listFileNames("rh", curDate)
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate)
}
} }
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); def fileName = "/ccb_ronghe/${curDate}/${curDate}.ok"
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate) if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\${curDate}.ok"
}
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "融合统计文件长度ok文件 上传Ftp完成......"
} }
} }
@@ -755,7 +755,7 @@ class CommonDatautils {
q.put("visible",0) q.put("visible",0)
q.put("cat", cat); q.put("cat", cat);
q.put("city", city); q.put("city", city);
// q.put("name","湖畔佳苑") // q.put("name","万达华府一期")
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
if(fields){ if(fields){
@@ -903,6 +903,36 @@ class CommonDatautils {
} }
return null; return null;
} }
public static def findSeaweedById(String id, def filelds){
if(!id){
return null;
}
try{
DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
DBObject keys = new BasicDBObject();
filelds.each{
keys.put(it, 1);
}
try{
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
}
}
}catch(Exception e){
println "error_id:"+id
e.printStackTrace()
}
}catch(Exception e){
}
return null;
}
public static def findDistrictBySid(String sid){ public static def findDistrictBySid(String sid){
File diff suppressed because one or more lines are too long
@@ -2,51 +2,172 @@ package com.fangjia.script.gj.crawl
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class AnjukeDistrictMatch { class AnjukeDistrictMatch {
static main(args) { static main(args) {
def task = new AnjukeDistrictMatch(); def task = new AnjukeDistrictMatch();
task.run(); // task.run();
// task.run1() // task.run1()
// task.updateField()
task.fenQiRun()
} }
def run1(){ def fenQiRun(){
def regionPointsMap = findRegionRange("重庆") def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) for(def c : citys){
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region","name","virescence_percent","build_area","all_area","cubage_percent","master_district_name"])
def distMap = findFiqiDatas(dists)
println c +" "+dists.size() +" "+ distMap.size()
addFieldValue(distMap);
}
} }
def addFieldValue(distMap){
def index = 0
distMap.each {k,v->
if(v.master_district_name!="master"){
def masterKey = v.city +"##"+v.region+"##"+v.master_district_name;
// def masterObj = distMap.get(masterKey)
def masterObj = CommonDatautils.findDistrict(v.city, v.region, v.master_district_name, "district");
if(masterObj){
def udata = new BasicDBObject()
if(!v.virescence_percent && masterObj.virescence_percent){
udata.put("virescence_percent", masterObj.virescence_percent)
}
if(!v.build_area && masterObj.build_area){
udata.put("build_area", masterObj.build_area)
}
if(!v.all_area && masterObj.all_area){
udata.put("all_area", masterObj.all_area)
}
if(!v.cubage_percent && masterObj.cubage_percent){
udata.put("cubage_percent", masterObj.cubage_percent)
}
if(udata){
index ++
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(v._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
println index;
}
def findFiqiDatas(dists){
def distMap = [:]
for(def d : dists){
def master_district_name = d.master_district_name
if(master_district_name && master_district_name!="null"){
distMap.put(d.city+"##"+d.region+"##"+d.name, d);
}
}
return distMap;
}
def updateField(){
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['Q'] && !row['Q'].contains("接口")){
def dist = CommonDatautils.findSeaweedById(row['R'], ["virescence_percent"])
if(dist && !dist.virescence_percent){
def val = row['K']
if(!val){
val = row['J']
}
if(!val){
val = row['M']
}
if(!val){
val = row['O']
}
if(!val){
val = row['G']
}
if(!val){
val = row['E']
}
if(!val){
val = row['F']
}
if(!val){
val = row['H']
}
if(!val){
val = row['I']
}
if(!val){
val = row['L']
}
if(!val){
val = row['N']
}
if(!val){
val = row['O']
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("virescence_percent", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
row['U'] = 1
}
}
}
})
}
// def run1(){
// def regionPointsMap = findRegionRange("重庆")
// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
// }
def run(){ def run(){
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
ExcelUtils.multiThreadingModify(filePath,{row-> ExcelUtils.multiThreadingModify(filePath,{row->
if(row['A'] && !row['J']){ if(row['A'] && !row['Q']){
def city = row['B'] def city = row['B']
def region = row['C'] def region = row['C']
def name = row['D'] def name = row['D']
def address = row['E']
// def flag = "别名匹配"
def flag = "名称匹配"
// def dist = matchDist(city, null, name) def dist = matchDist(city, region, name)
// if(!row['J'] && name.contains("(")){ if(!row['D'] && name.contains("(")){
// name = name.replace("(${region})","").replace("(","").replace(")","") name = name.replace("(${region})","").replace("(","").replace(")","")
// flag = "去括号匹配" flag = "去括号匹配"
// dist = matchDist(city, region, name) dist = matchDist(city, region, name)
// } }
// if(!dist){ if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
// flag = "映射别名匹配" flag = "映射别名匹配"
// } }
// if(!dist) { if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
// if(rets && rets.size()==1){ if(rets && rets.size()==1){
// dist = rets[0] dist = rets[0]
// } }
// } }
// //
// if(!dist){ // if(!dist){
@@ -61,23 +182,21 @@ class AnjukeDistrictMatch {
// } // }
// } // }
// if(dist){ if(dist){
// row['J'] = flag row['Q'] = flag
// row['K'] = dist._id.toString() row['R'] = dist._id.toString()
// row['L'] = dist.region row['S'] = dist.region
// row['M'] = dist.name row['T'] = dist.name
// row['N'] = dist.address }else{
// } def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name) row['Q'] = "匹配接口"
if(jsonObject != null && jsonObject.precise!=-1) { row['R'] = jsonObject.get("id")
// row['J'] = "匹配接口" row['S'] = jsonObject.get("district")
// row['K'] = jsonObject.get("id") row['T'] = jsonObject.get("name")
// row['L'] = jsonObject.get("district") }
// row['M'] = jsonObject.get("name")
// row['N'] = jsonObject.get("address")
row['M'] = jsonObject.get("name")
} }
} }
}) })
} }
@@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome {
def price = e.select(".totalPrice .number").text() def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) { if (price && price.isNumber()) {
detail.total_price = price.toDouble() * 10000 detail.total_price = (int)(price.toDouble() * 10000)
} }
if (detail.total_price && detail.area) { if (detail.total_price && detail.area) {
@@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome {
def price = e.select(".totalPrice .number").text() def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) { if (price && price.isNumber()) {
detail.total_price = price.toDouble() * 10000 detail.total_price = (int)(price.toDouble() * 10000)
} }
if (detail.total_price && detail.area) { if (detail.total_price && detail.area) {
@@ -3,12 +3,15 @@ package com.fangjia.script.gj.test
import ch.qos.logback.core.db.dialect.DBUtil import ch.qos.logback.core.db.dialect.DBUtil
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.Mongo import com.mongodb.Mongo
@@ -24,6 +27,75 @@ DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "1
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL) DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com")
DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
//ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因.xls", { d ->
//})
ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因(标准化省市区).xls", { d ->
def info = d['A']
if(info && !d['E']){
def arrs = info.split("!~")
if(arrs && arrs[2] && arrs[3]){
// def rMap = SeaweedDataMatchProcess.doMatch(d['B'], d['C'], arrs[4], arrs[4]);
// if(rMap) {
// d['E'] = rMap.name
// d['F'] = rMap.id
// }
def address = null
if(d['G']){
address = d['G']
}else{
address = arrs[4]
}
def jsonObject = OpenApiClient.addressMatch(d['C'], address, "property", false, d['D'], null)
if(jsonObject != null && jsonObject.precise!=-1) {
d['H'] = jsonObject.get("city")
d['I'] = jsonObject.get("district")
d['J'] = jsonObject.get("name")
d['K'] = jsonObject.get("id")
}
// def jsonObj = FangjiaClientUtils.addressStructure(d['B'], arrs[4])
// if(jsonObj && jsonObj.result) {
// def street = null
// def street_number = null
// if (jsonObj.result.street) {
// d['E'] = jsonObj.result.street
// }
// if (jsonObj.result.streetNumber) {
// d['F'] = jsonObj.result.streetNumber
// }
// d['G'] = jsonObj.result.name
// }
// def rets = findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, arrs[1], arrs[3])
// if(rets && rets.size()==1){
// def obj = rets[0]
// d['B'] = obj.province
// d['C'] = obj.fjw_city
// d['D'] = obj.fjw_district
// }
}
}
});
//
//def findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, cityCode, regionCode){
// def q = new BasicDBObject()
// q.put("city_code", cityCode.toString().toInteger())
// q.put("district_code", regionCode.toString().toInteger())
// def dbcursor = FANGJIA_AREA_CODE.find(q)
// def rets = []
// for(def d : dbcursor){
// rets << d
// }
// return rets;
//}
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d-> //ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
// def city = RegionData.formatCity(d['E']) // def city = RegionData.formatCity(d['E'])
// if(city && !d['J']){ // if(city && !d['J']){
@@ -103,33 +175,33 @@ DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "townshi
// } // }
//}) //})
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> //ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
if(!d['D']){ // if(!d['D']){
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) // def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
if(infos && infos.size()==1){ // if(infos && infos.size()==1){
def info = infos[0] // def info = infos[0]
d['D'] = info.province_code // d['D'] = info.province_code
d['E'] = info.area_code // d['E'] = info.area_code
d['F'] = info.street_code // d['F'] = info.street_code
}
}
})
def findTownshipInfo(city, region, coll){
def q = new BasicDBObject()
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
def dbcursor = coll.find(q)
def rows = []
for(def d : dbcursor){
// if(d.street_name.contains(region)){
rows << d
// } // }
} // }
return rows //})
} //
//
//def findTownshipInfo(city, region, coll){
// def q = new BasicDBObject()
// q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
// q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
//
// def dbcursor = coll.find(q)
// def rows = []
// for(def d : dbcursor){
//// if(d.street_name.contains(region)){
// rows << d
//// }
// }
// return rows
//}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.* import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220601", "yyyyMMdd") def startDate = DateUtil.parseDate("20220901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220701", "yyyyMMdd") def endDate = DateUtil.parseDate("20221001", "yyyyMMdd")
Calendar calendar = Calendar.getInstance() Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate) calendar.setTime(startDate)
def count = 0 def count = 0
@@ -25,7 +25,7 @@ while (startDate.before(endDate)) {
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result) def result = JSONObject.fromObject(doc.res_map.result)
if (!hashSet.contains(result)) { if (!hashSet.contains(result)) {
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] def item = ["60a3202c0b42a00001aad385", "qlkj", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item data << item
hashSet << result hashSet << result
} else { } else {
@@ -38,4 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate startDate = nextDate
} }
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志") ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")