Merge branch 'script_gj' into 'master'

融合均价数据连接数据库修改

See merge request !477
This commit is contained in:
2022-12-06 14:57:28 +08:00
17 changed files with 730 additions and 161 deletions
@@ -17,8 +17,8 @@ class AddThirdPartyPrice {
}
def run(){
//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"
def citys = ["林芝"]
//
def citys = ["林芝","海西州","天门","大兴安岭","儋州","鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"]
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def c : citys){
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
@@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
}
@@ -52,7 +52,7 @@ class CcbCommonUtils {
def countStr = args[0]
if(countStr=="344"){
citys = CcbCommonUtils.citys_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="92"){
citys = CcbCommonUtils.citys_92
@@ -13,7 +13,7 @@ class CcbFtpTestUtils {
static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb_evaluation/output/"
static def ftpPath_rh = "/home/ccb_rh/output/"
static def ftpPath_rh = "/home/ccb_ronghe/output/"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
@@ -70,16 +70,26 @@ class CcbFtpTestUtils {
}
static def listFileNames(month){
static def listFileNames(type, month){
//上传到ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
Vector files = null;
if(type && type=="rh"){
files = sftp.listFiles(ftpPath_rh+ month+"/", sftpChannel);
}else{
files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
}
return files
}
static def getFileLength(String path, String month){
static def getFileLength(String type, String path, String month){
sftpChannel = reConnect();
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
def size = 0;
if(type && type=="rh"){
size = sftp.getFileSize(ftpPath_rh+ month+"/"+path, sftpChannel)
}else{
size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
}
return size
}
@@ -371,11 +371,11 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_92
def month = 202209
def month = 202210
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202210)
def rets = findPeGteThirdParty(city, 202211)
def total = 0
println city +" "+rets.size()
for(def m : rets){
@@ -54,7 +54,7 @@ class ExportOracleAllDataTest {
// exportCommunityStaging()//没cityid
// exportCommunityBuileType()//
exportThirdPartyPrice2(new_dists)//
// exportThirdPartyPrice2(new_dists)//
// exportBigdataForsale(new_dists) //2022-07-19
// exportBigdataDeal( new_dists) //2022-07-01
//
@@ -325,7 +325,7 @@ class ExportOracleAllDataTest {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')";
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' ";//and createtime>=to_date('${date}','yyyyMMdd')
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
@@ -1172,14 +1172,14 @@ class ExportOracleAllDataTest {
def exportInOk(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
Vector files = CcbFtpTestUtils.listFileNames(curDate)
Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate)
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
}
}
def fileName = "/ccb_evaluation/${curDate}.ok"
@@ -38,16 +38,17 @@ class GuiDingPriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
def rows = []
CcbCommonUtils.deleteGuidedPrice(curDate)
// CcbCommonUtils.deleteGuidedPrice(curDate)
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -45,7 +45,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
}
@@ -40,7 +40,7 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
task.run()//统计最近3个月抓取的成交数据量
// task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
@@ -50,8 +50,25 @@ class TradeStatLog {
// task.deleteDealData2()
// task.statTableCount()
// task.filterTrade()
task.updateDealPriceType()
}
def updateDealPriceType(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def c : citys){
DEAL_PRICE_COLL.find(new BasicDBObject("city", c)).sort(new BasicDBObject("trade_date",-1)).batchSize(5000).each {it->
if(it.total_price instanceof Double){
index ++
println index+" "+it.city+" "+it.total_price+" "+it._id.toString()
def udata = new BasicDBObject()
udata.put("total_price", (int)it.total_price)
DEAL_PRICE_COLL.update(new BasicDBObject("_id", it._id), new BasicDBObject('$set', udata))
}
}
}
}
def statTableCount(){
@@ -8,6 +8,7 @@ import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
@BatchOwner(Watcher.GaoJuan)
@@ -25,7 +26,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
@Override
void process(Object args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def curDate = "20221110"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
@@ -48,17 +49,17 @@ class ExportDistrictRongHeTest extends BaseBatch{
// exportBigdataDistrictFJW()//
// exportBigdataAliasFJW()//
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
exportBigdataCityPrice(queryDate)
exportBigdataDistPrice(queryDate)
exportBigdataCommPrice(queryDate)
// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
// exportBigdataCityPrice(queryDate)
// exportBigdataDistPrice(queryDate)
// exportBigdataCommPrice(queryDate)
exportInOk()
// }
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
exportOk(curDate)
System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
}
@@ -66,9 +67,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
def index = 0
for(def city : citys){
@@ -133,9 +134,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -162,9 +163,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -190,9 +191,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
@@ -228,9 +229,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -258,9 +259,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -291,9 +292,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -323,9 +324,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
@@ -358,9 +359,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -382,9 +383,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
def allRows = []
@@ -482,9 +483,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
@@ -675,9 +676,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......"
@@ -686,9 +687,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
def index = 0
for(def city : citys) {
@@ -743,9 +744,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
@@ -789,9 +790,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
@@ -836,9 +837,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
@@ -899,9 +900,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -910,9 +911,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
@@ -921,23 +922,35 @@ class ExportDistrictRongHeTest extends BaseBatch{
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
def exportInOk(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
Vector files = CcbFtpTestUtils.listFileNames("rh", curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate)
}
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
def fileName = "/ccb_ronghe/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\${curDate}.ok"
}
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "融合统计文件长度ok文件 上传Ftp完成......"
}
}
@@ -755,7 +755,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","湖畔佳苑")
// q.put("name","万达华府一期")
DBObject keys = new BasicDBObject();
if(fields){
@@ -903,6 +903,36 @@ class CommonDatautils {
}
return null;
}
public static def findSeaweedById(String id, def filelds){
if(!id){
return null;
}
try{
DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
DBObject keys = new BasicDBObject();
filelds.each{
keys.put(it, 1);
}
try{
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
}
}
}catch(Exception e){
println "error_id:"+id
e.printStackTrace()
}
}catch(Exception e){
}
return null;
}
public static def findDistrictBySid(String sid){
File diff suppressed because one or more lines are too long
@@ -2,51 +2,172 @@ package com.fangjia.script.gj.crawl
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class AnjukeDistrictMatch {
static main(args) {
def task = new AnjukeDistrictMatch();
task.run();
// task.run();
// task.run1()
// task.updateField()
task.fenQiRun()
}
def run1(){
def regionPointsMap = findRegionRange("重庆")
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
def fenQiRun(){
def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
for(def c : citys){
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region","name","virescence_percent","build_area","all_area","cubage_percent","master_district_name"])
def distMap = findFiqiDatas(dists)
println c +" "+dists.size() +" "+ distMap.size()
addFieldValue(distMap);
}
}
def addFieldValue(distMap){
def index = 0
distMap.each {k,v->
if(v.master_district_name!="master"){
def masterKey = v.city +"##"+v.region+"##"+v.master_district_name;
// def masterObj = distMap.get(masterKey)
def masterObj = CommonDatautils.findDistrict(v.city, v.region, v.master_district_name, "district");
if(masterObj){
def udata = new BasicDBObject()
if(!v.virescence_percent && masterObj.virescence_percent){
udata.put("virescence_percent", masterObj.virescence_percent)
}
if(!v.build_area && masterObj.build_area){
udata.put("build_area", masterObj.build_area)
}
if(!v.all_area && masterObj.all_area){
udata.put("all_area", masterObj.all_area)
}
if(!v.cubage_percent && masterObj.cubage_percent){
udata.put("cubage_percent", masterObj.cubage_percent)
}
if(udata){
index ++
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(v._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
println index;
}
def findFiqiDatas(dists){
def distMap = [:]
for(def d : dists){
def master_district_name = d.master_district_name
if(master_district_name && master_district_name!="null"){
distMap.put(d.city+"##"+d.region+"##"+d.name, d);
}
}
return distMap;
}
def updateField(){
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['Q'] && !row['Q'].contains("接口")){
def dist = CommonDatautils.findSeaweedById(row['R'], ["virescence_percent"])
if(dist && !dist.virescence_percent){
def val = row['K']
if(!val){
val = row['J']
}
if(!val){
val = row['M']
}
if(!val){
val = row['O']
}
if(!val){
val = row['G']
}
if(!val){
val = row['E']
}
if(!val){
val = row['F']
}
if(!val){
val = row['H']
}
if(!val){
val = row['I']
}
if(!val){
val = row['L']
}
if(!val){
val = row['N']
}
if(!val){
val = row['O']
}
if(val){
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
val = val.toString().toDouble().toInteger()
}
def udata = new BasicDBObject()
udata.put("virescence_percent", val.toString())
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(dist._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
row['U'] = 1
}
}
}
})
}
// def run1(){
// def regionPointsMap = findRegionRange("重庆")
// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
// }
def run(){
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
ExcelUtils.multiThreadingModify(filePath,{row->
if(row['A'] && !row['J']){
if(row['A'] && !row['Q']){
def city = row['B']
def region = row['C']
def name = row['D']
def address = row['E']
// def flag = "别名匹配"
// def dist = matchDist(city, null, name)
// if(!row['J'] && name.contains("(")){
// name = name.replace("(${region})","").replace("(","").replace(")","")
// flag = "去括号匹配"
// dist = matchDist(city, region, name)
// }
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
// flag = "映射别名匹配"
// }
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
// if(rets && rets.size()==1){
// dist = rets[0]
// }
// }
def flag = "名称匹配"
def dist = matchDist(city, region, name)
if(!row['D'] && name.contains("(")){
name = name.replace("(${region})","").replace("(","").replace(")","")
flag = "去括号匹配"
dist = matchDist(city, region, name)
}
if(!dist){
dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
flag = "映射别名匹配"
}
if(!dist) {
def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if(rets && rets.size()==1){
dist = rets[0]
}
}
//
// if(!dist){
@@ -61,23 +182,21 @@ class AnjukeDistrictMatch {
// }
// }
// if(dist){
// row['J'] = flag
// row['K'] = dist._id.toString()
// row['L'] = dist.region
// row['M'] = dist.name
// row['N'] = dist.address
// }
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
if(jsonObject != null && jsonObject.precise!=-1) {
// row['J'] = "匹配接口"
// row['K'] = jsonObject.get("id")
// row['L'] = jsonObject.get("district")
// row['M'] = jsonObject.get("name")
// row['N'] = jsonObject.get("address")
row['M'] = jsonObject.get("name")
if(dist){
row['Q'] = flag
row['R'] = dist._id.toString()
row['S'] = dist.region
row['T'] = dist.name
}else{
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Q'] = "匹配接口"
row['R'] = jsonObject.get("id")
row['S'] = jsonObject.get("district")
row['T'] = jsonObject.get("name")
}
}
}
})
}
@@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toDouble() * 10000
detail.total_price = (int)(price.toDouble() * 10000)
}
if (detail.total_price && detail.area) {
@@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome {
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toDouble() * 10000
detail.total_price = (int)(price.toDouble() * 10000)
}
if (detail.total_price && detail.area) {
@@ -3,12 +3,15 @@ package com.fangjia.script.gj.test
import ch.qos.logback.core.db.dialect.DBUtil
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
@@ -24,6 +27,75 @@ DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "1
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com")
DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
//ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因.xls", { d ->
//})
ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因(标准化省市区).xls", { d ->
def info = d['A']
if(info && !d['E']){
def arrs = info.split("!~")
if(arrs && arrs[2] && arrs[3]){
// def rMap = SeaweedDataMatchProcess.doMatch(d['B'], d['C'], arrs[4], arrs[4]);
// if(rMap) {
// d['E'] = rMap.name
// d['F'] = rMap.id
// }
def address = null
if(d['G']){
address = d['G']
}else{
address = arrs[4]
}
def jsonObject = OpenApiClient.addressMatch(d['C'], address, "property", false, d['D'], null)
if(jsonObject != null && jsonObject.precise!=-1) {
d['H'] = jsonObject.get("city")
d['I'] = jsonObject.get("district")
d['J'] = jsonObject.get("name")
d['K'] = jsonObject.get("id")
}
// def jsonObj = FangjiaClientUtils.addressStructure(d['B'], arrs[4])
// if(jsonObj && jsonObj.result) {
// def street = null
// def street_number = null
// if (jsonObj.result.street) {
// d['E'] = jsonObj.result.street
// }
// if (jsonObj.result.streetNumber) {
// d['F'] = jsonObj.result.streetNumber
// }
// d['G'] = jsonObj.result.name
// }
// def rets = findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, arrs[1], arrs[3])
// if(rets && rets.size()==1){
// def obj = rets[0]
// d['B'] = obj.province
// d['C'] = obj.fjw_city
// d['D'] = obj.fjw_district
// }
}
}
});
//
//def findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, cityCode, regionCode){
// def q = new BasicDBObject()
// q.put("city_code", cityCode.toString().toInteger())
// q.put("district_code", regionCode.toString().toInteger())
// def dbcursor = FANGJIA_AREA_CODE.find(q)
// def rets = []
// for(def d : dbcursor){
// rets << d
// }
// return rets;
//}
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
// def city = RegionData.formatCity(d['E'])
// if(city && !d['J']){
@@ -103,33 +175,33 @@ DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "townshi
// }
//})
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
if(!d['D']){
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
if(infos && infos.size()==1){
def info = infos[0]
d['D'] = info.province_code
d['E'] = info.area_code
d['F'] = info.street_code
}
}
})
def findTownshipInfo(city, region, coll){
def q = new BasicDBObject()
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
def dbcursor = coll.find(q)
def rows = []
for(def d : dbcursor){
// if(d.street_name.contains(region)){
rows << d
//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
// if(!d['D']){
// def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
// if(infos && infos.size()==1){
// def info = infos[0]
// d['D'] = info.province_code
// d['E'] = info.area_code
// d['F'] = info.street_code
// }
}
return rows
}
// }
//})
//
//
//def findTownshipInfo(city, region, coll){
// def q = new BasicDBObject()
// q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
// q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
//
// def dbcursor = coll.find(q)
// def rows = []
// for(def d : dbcursor){
//// if(d.street_name.contains(region)){
// rows << d
//// }
// }
// return rows
//}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220601", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220701", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20221001", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -25,7 +25,7 @@ while (startDate.before(endDate)) {
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if (!hashSet.contains(result)) {
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
def item = ["60a3202c0b42a00001aad385", "qlkj", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << result
} else {
@@ -38,4 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")