diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy index 7a641d018..ca14ea514 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddThirdPartyPrice.groovy @@ -17,8 +17,8 @@ class AddThirdPartyPrice { } def run(){ -//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关" - def citys = ["林芝"] +// + def citys = ["林芝","海西州","天门","大兴安岭","儋州","鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"] def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def c : citys){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 28c543e60..d73c64d44 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } @@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{ } - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 517f8c44c..a551c50b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -52,7 +52,7 @@ class CcbCommonUtils { def countStr = args[0] if(countStr=="344"){ citys = CcbCommonUtils.citys_all - sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") }else if(countStr=="92"){ citys = CcbCommonUtils.citys_92 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 2a9ad549f..4bac05c3f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -13,7 +13,7 @@ class CcbFtpTestUtils { static def ftpPassword = "ccb!@#" static def ftpPath = "/home/ccb_evaluation/output/" - static def ftpPath_rh = "/home/ccb_rh/output/" + static def ftpPath_rh = "/home/ccb_ronghe/output/" static SFTP sftp = new SFTP(); static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); @@ -70,16 +70,26 @@ class CcbFtpTestUtils { } - static def listFileNames(month){ + static def listFileNames(type, month){ //上传到ftp sftpChannel = reConnect(); - Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + Vector files = null; + if(type && type=="rh"){ + files = sftp.listFiles(ftpPath_rh+ month+"/", sftpChannel); + }else{ + files = sftp.listFiles(ftpPath+ month+"/", sftpChannel); + } return files } - static def getFileLength(String path, String month){ + static def getFileLength(String type, String path, String month){ sftpChannel = reConnect(); - def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + def size = 0; + if(type && type=="rh"){ + size = sftp.getFileSize(ftpPath_rh+ month+"/"+path, sftpChannel) + }else{ + size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel) + } return size } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index a96478424..e67f2a8cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -371,11 +371,11 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_92 - def month = 202209 + def month = 202210 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys) { - def rets = findPeGteThirdParty(city, 202210) + def rets = findPeGteThirdParty(city, 202211) def total = 0 println city +" "+rets.size() for(def m : rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy index 71c378532..d35310b33 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleAllDataTest.groovy @@ -54,7 +54,7 @@ class ExportOracleAllDataTest { // exportCommunityStaging()//没cityid // exportCommunityBuileType()// - exportThirdPartyPrice2(new_dists)// +// exportThirdPartyPrice2(new_dists)// // exportBigdataForsale(new_dists) //2022-07-19 // exportBigdataDeal( new_dists) //2022-07-01 // @@ -325,7 +325,7 @@ class ExportOracleAllDataTest { def allRows = [] for(def city : citys) { def cityCode = cityCodeMap.get(city) - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')"; + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' ";//and createtime>=to_date('${date}','yyyyMMdd') sqlInstance.eachRow(sql, { r -> def data = [] data << r.ID @@ -1172,14 +1172,14 @@ class ExportOracleAllDataTest { def exportInOk(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - Vector files = CcbFtpTestUtils.listFileNames(curDate) + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) Iterator fileVec = files.iterator() def fileRows = [] while(fileVec.hasNext()){ ChannelSftp.LsEntry file = fileVec.next() if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ - fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate) + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) } } def fileName = "/ccb_evaluation/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index bd64cc3f8..0d15402de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -38,16 +38,17 @@ class GuiDingPriceBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.getRunningCitys(args) + def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args) def rows = [] - CcbCommonUtils.deleteGuidedPrice(curDate) +// CcbCommonUtils.deleteGuidedPrice(curDate) for (def city : citys) { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25") +// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 03d4cfcb7..027e71834 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -45,7 +45,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] def index = 0 - def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd")) def cityCodeDatas = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ @@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index fd96c4d80..ae3aa4800 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,7 +40,7 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 +// task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) @@ -50,8 +50,25 @@ class TradeStatLog { // task.deleteDealData2() // task.statTableCount() // task.filterTrade() + task.updateDealPriceType() + } + def updateDealPriceType(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def c : citys){ + DEAL_PRICE_COLL.find(new BasicDBObject("city", c)).sort(new BasicDBObject("trade_date",-1)).batchSize(5000).each {it-> + if(it.total_price instanceof Double){ + index ++ + println index+" "+it.city+" "+it.total_price+" "+it._id.toString() + + def udata = new BasicDBObject() + udata.put("total_price", (int)it.total_price) + DEAL_PRICE_COLL.update(new BasicDBObject("_id", it._id), new BasicDBObject('$set', udata)) + } + } + } } def statTableCount(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 161f82f00..312032ca4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.ccbevaluation.CcbFtpTestUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp import groovy.sql.Sql @BatchOwner(Watcher.GaoJuan) @@ -25,7 +26,7 @@ class ExportDistrictRongHeTest extends BaseBatch{ @Override void process(Object args) { citys = CcbCommonUtils.getRunningCitys(args) - def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def curDate = "20221110"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] //按天导(每天一个文件夹) // exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists) @@ -48,17 +49,17 @@ class ExportDistrictRongHeTest extends BaseBatch{ // exportBigdataDistrictFJW()// // exportBigdataAliasFJW()// - def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() - exportBigdataCityPrice(queryDate) - exportBigdataDistPrice(queryDate) - exportBigdataCommPrice(queryDate) +// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger() +// exportBigdataCityPrice(queryDate) +// exportBigdataDistPrice(queryDate) +// exportBigdataCommPrice(queryDate) + + exportInOk() // } + SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); - - exportOk(curDate) - + System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}")); } @@ -66,9 +67,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommunityGJRH(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } def index = 0 for(def city : citys){ @@ -133,9 +134,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -162,9 +163,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -190,9 +191,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) @@ -228,9 +229,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ }); } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -258,9 +259,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -291,9 +292,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -323,9 +324,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) @@ -358,9 +359,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -382,9 +383,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ index++ def curStartStr = curStartDateStr - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } def allRows = [] @@ -482,9 +483,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") // def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } def allRows = [] @@ -675,9 +676,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); println "BIGDATA_DEAL_FJW 导出完成......" @@ -686,9 +687,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataBuildingFJW(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } def index = 0 for(def city : citys) { @@ -743,9 +744,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCityPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { @@ -789,9 +790,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataDistPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { @@ -836,9 +837,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommPrice(queryDate){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat" } def allRows = [] for(def city : citys) { @@ -899,9 +900,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -910,9 +911,9 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) @@ -921,23 +922,35 @@ class ExportDistrictRongHeTest extends BaseBatch{ def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } - def exportOk(curDate){ -// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") - def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" - if(CommonDatautils.isWindow()){ - dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + + def exportInOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + Vector files = CcbFtpTestUtils.listFileNames("rh", curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate) + } } - FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); - CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate) + def fileName = "/ccb_ronghe/${curDate}/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\${curDate}.ok" + } + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "融合统计文件长度ok文件 上传Ftp完成......" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 49bacf4ee..c6d2373a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -755,7 +755,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","湖畔佳苑") +// q.put("name","万达华府一期") DBObject keys = new BasicDBObject(); if(fields){ @@ -903,6 +903,36 @@ class CommonDatautils { } return null; } + + + public static def findSeaweedById(String id, def filelds){ + if(!id){ + return null; + } + try{ + DBObject q = new BasicDBObject(); + q.put("_id", new ObjectId(id.toString().replace("[","").replace("]",""))); + + DBObject keys = new BasicDBObject(); + filelds.each{ + keys.put(it, 1); + } + try{ + DSCursor cursor = getDataStoreConnection().find(q, keys); + if(cursor){ + for(def obj : cursor){ + return obj; + } + } + }catch(Exception e){ + println "error_id:"+id + e.printStackTrace() + } + }catch(Exception e){ + + } + return null; + } public static def findDistrictBySid(String sid){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy new file mode 100644 index 000000000..d1aca25f7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YinLianMeiShi.groovy @@ -0,0 +1,305 @@ +package com.fangjia.script.gj + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import net.sf.json.JSONObject +import org.apache.poi.ss.util.RegionUtil + +class YinLianMeiShi { + + def baiduCityCodeMap = ["北京": "131", "天津": "332", "石家庄": "150", "唐山": "265", "秦皇岛": "148", "邯郸": "151", "邢台": "266", "保定": "307", "张家口": "264", "承德": "207", "沧州": "149", "廊坊": "191", "衡水": "208", "太原": "176", "大同": "355", "阳泉": "357", "长治": "356", "晋城": "290", "朔州": "237", "晋中": "238", "运城": "328", "忻州": "367", "临汾": "368", "吕梁": "327", "呼和浩特": "321", "包头": "229", "乌海": "123", "赤峰": "297", "通辽": "64", "鄂尔多斯": "283", "呼伦贝尔": "61", "巴彦淖尔": "169", "乌兰察布": "168", "兴安盟": "62", "锡林郭勒盟": "63", "阿拉善盟": "230", "沈阳": "58", "大连": "167", "鞍山": "320", "抚顺": "184", "本溪": "227", "丹东": "282", "锦州": "166", "营口": "281", "阜新": "59", "辽阳": "351", "盘锦": "228", "铁岭": "60", "朝阳": "280", "葫芦岛": "319", "长春": "53", "吉林市": "55", "四平": "56", "辽源": "183", "通化": "165", "白山": "57", "松原": "52", "白城": "51", "延边朝鲜族自治州": "54", "哈尔滨": "48", "齐齐哈尔": "41", "鸡西": "46", "鹤岗": "43", "双鸭山": "45", "大庆": "50", "伊春": "40", "佳木斯": "42", "七台河": "47", "牡丹江": "49", "黑河": "39", "绥化": "44", "大兴安岭地区": "38", "上海": "289", "南京": "315", "无锡": "317", "徐州": "316", "常州": "348", "苏州": "224", "南通": "161", "连云港": "347", "淮安": "162", "盐城": "223", "扬州": "346", "镇江": "160", "泰州": "276", "宿迁": "277", "杭州": "179", "宁波": "180", "温州": "178", "嘉兴": "334", "湖州": "294", "绍兴": "293", "金华": "333", "衢州": "243", "舟山": "245", "台州": "244", "丽水": "292", "合肥": "127", "芜湖": "129", "蚌埠": "126", "淮南": "250", "马鞍山": "358", "淮北": "253", "铜陵": "337", "安庆": "130", "黄山": "252", "滁州": "189", "阜阳": "128", "宿州": "370", "巢湖": "251", "六安": "298", "亳州": "188", "池州": "299", "宣城": "190", "福州": "300", "厦门": "194", "莆田": "195", "三明": "254", "泉州": "134", "漳州": "255", "南平": "133", "龙岩": "193", "宁德": "192", "南昌": "163", "景德镇": "225", "萍乡": "350", "九江": "349", "新余": "164", "鹰潭": "279", "赣州": "365", "吉安": "318", "宜春": "278", "抚州": "226", "上饶": "364", "济南": "288", "青岛": "236", "淄博": "354", "枣庄": "172", "东营": "174", "烟台": "326", "潍坊": "287", "济宁": "286", "泰安": "325", "威海": "175", "日照": "173", "莱芜": "124", "临沂": "234", "德州": "372", "聊城": "366", "滨州": "235", "菏泽": "353", "郑州": "268", "开封": "210", "洛阳": "153", "平顶山": "213", "安阳": "267", "鹤壁": "215", "新乡": "152", "焦作": "211", "濮阳": "209", "许昌": "155", "漯河": "344", "三门峡": "212", "南阳": "309", "商丘": "154", "信阳": "214", "周口": "308", "驻马店": "269", "武汉": "218", "黄石": "311", "十堰": "216", "宜昌": "270", "襄阳": "156", "鄂州": "122", "荆门": "217", "孝感": "310", "荆州": "157", "黄冈": "271", "咸宁": "362", "随州": "371", "恩施土家族苗族自治州": "373", "仙桃": "1713", "潜江": "1293", "天门": "2654", "神农架林区": "2734", "长沙": "158", "株洲": "222", "湘潭": "313", "衡阳": "159", "邵阳": "273", "岳阳": "220", "常德": "219", "张家界": "312", "益阳": "272", "郴州": "275", "永州": "314", "怀化": "363", "娄底": "221", "湘西土家族苗族自治州": "274", "广州": "257", "韶关": "137", "深圳": "340", "珠海": "140", "汕头": "303", "佛山": "138", "江门": "302", "湛江": "198", "茂名": "139", "肇庆": "338", "惠州": "301", "梅州": "141", "汕尾": "339", "河源": "200", "阳江": "199", "清远": "197", "东莞": "119", "中山": "187", "潮州": "201", "揭阳": "259", "云浮": "258", "南宁": "261", "柳州": "305", "桂林": "142", "梧州": "304", "北海": "295", "防城港": "204", "钦州": "145", "贵港": "341", "玉林": "361", "百色": "203", "贺州": "260", "河池": "143", "来宾": "202", "崇左": "144", "海口": "125", "三亚": "121", "五指山": "1644", "琼海": "2358", "儋州": "1215", "文昌": "2758", "万宁": "1216", "东方": "2634", "定安": "1214", "屯昌": "1641", "澄迈": "2757", "临高": "2033", "白沙黎族自治": "2359", "昌江黎族自治": "1642", "乐东黎族自治": "2032", "陵水黎族自治": "1643", "保亭黎族苗族自治": "1217", "琼中黎族苗族自治": "2031", "重庆": "132", "成都": "75", "自贡": "78", "攀枝花": "81", "泸州": "331", "德阳": "74", "绵阳": "240", "广元": "329", "遂宁": "330", "内江": "248", "乐山": "79", "南充": "291", "眉山": "77", "宜宾": "186", "广安": "241", "达州": "369", "雅安": "76", "巴中": "239", "资阳": "242", "阿坝藏族羌族自治州": "185", "甘孜藏族自治州": "73", "凉山彝族自治州": "80", "贵阳": "146", "六盘水": "147", "遵义": "262", "安顺": "263", "铜仁地区": "205", "黔西南布依族苗族自治州": "343", "毕节地区": "206", "黔东南苗族侗族自治州": "342", "黔南布依族苗族自治州": "306", "昆明": "104", "曲靖": "249", "玉溪": "106", "保山": "112", "昭通": "336", "丽江": "114", "临沧": "110", "楚雄彝族自治州": "105", "红河哈尼族彝族自治州": "107", "文山壮族苗族自治州": "177", "普洱": "108", "西双版纳傣族自治州": "109", "大理白族自治州": "111", "德宏傣族景颇族自治州": "116", "怒江傈僳族自治州": "113", "迪庆藏族自治州": "115", "拉萨": "100", "昌都地区": "99", "山南地区": "97", "日喀则地区": "102", "那曲地区": "101", "阿里地区": "103", "林芝地区": "98", "西安": "233", "铜川": "232", "宝鸡": "171", "咸阳": "323", "渭南": "170", "延安": "284", "汉中": "352", "榆林": "231", "安康": "324", "商洛": "285", "兰州": "36", "嘉峪关": "33", "金昌": "34", "白银": "35", "天水": "196", "武威": "118", "张掖": "117", "平凉": "359", "酒泉": "37", "庆阳": "135", "定西": "136", "陇南": "256", "临夏回族自治州": "182", "甘南藏族自治州": "247", "西宁": "66", "海东地区": "69", "海北藏族自治州": "67", "黄南藏族自治州": "70", "海南藏族自治州": "68", "果洛藏族自治州": "72", "玉树藏族自治州": "71", "海西蒙古族藏族自治州": "65", "银川": "360", "石嘴山": "335", "吴忠": "322", "固原": "246", "中卫": "181", "乌鲁木齐": "92", "克拉玛依": "95", "吐鲁番地区": "89", "哈密地区": "91", "昌吉回族自治州": "93", "博尔塔拉蒙古自治州": "88", "巴音郭楞蒙古自治州": "86", "阿克苏地区": "85", "克孜勒苏柯尔克孜自治州": "84", "喀什地区": "83", "和田地区": "82", "伊犁": "90", "塔城地区": "94", "阿勒泰地区": "96", "石河子": "770", "阿拉尔": "731", "图木舒克": "792", "五家渠": "789", "香港特别行政区": "2912", "澳门特别行政区": "2911"] + + def filters = [] + + static main(args) { + def task = new YinLianMeiShi(); + task.run() + } + + def run(){ + ExcelUtils.modify("D:\\数据\\银联\\数据500条-关键字.xlsx",{row-> + def score = row['Q'] + if(row['E'] && (!score || (score && score.toString().toDouble()<0.8))){ + def city = RegionData.formatCity(row['E']); + + def baiduCode = getBaiduMapCode(city); + def newKeys = filterName(row['D'], row['E'], row['F'], row['B']) + row['I'] = newKeys + + if(baiduCode){ + def rets = searchBaiDuMap(row['B'], baiduCode.toInteger()); + def simpleAddr = getSimpleAddress(city, row['C']) + def bdRes = searchMapBySimilar(city, row['F'], row['B'], simpleAddr, rets, newKeys); + if (bdRes && (!score || (score && bdRes.score>score.toString().toDouble()))) { + row['J'] = bdRes.uid + row['K'] = bdRes.region + row['L'] = bdRes.name + row['M'] = bdRes.old_addr + row['N'] = bdRes.type + row['O'] = bdRes.baidu_lng2 + row['P'] = bdRes.baidu_lat2 + row['Q'] = bdRes.score + if(bdRes.matchType){ + row['R'] = bdRes.matchType + } + } + println row['B'] +" "+ bdRes + } + + } + }) + } + + + def getSimpleAddress(city, address){ + def jsonObj = FangjiaClientUtils.addressStructure(city, address) + def old_addr = null + if(jsonObj && jsonObj.result){ + if(jsonObj.result.street){ + def street = jsonObj.result.street + if(street){ + old_addr = street + } + } +// if(jsonObj.result.streetNumber){ +// def street_number = jsonObj.result.streetNumber +// if(street_number){ +// old_addr = old_addr + street_number +// } +// } + } + return old_addr; + } + + def filterName(pro,city, region, name){ + def newKey = name.replace("有限责任公司", "").replace("有限公司","") + if(pro && name.contains(pro)){ + newKey = newKey.replace(pro, "") + } + + def newCity = RegionData.formatCity(city) + if(city && (name.contains(city) || name.contains(newCity))){ + newKey = newKey.replace(city, "").replace(newCity, "") + } + + + if(region && newKey.contains(region)){ + newKey = newKey.replace(region, "") + } + if(newKey.contains("(")){ + def subName = newKey.substring(newKey.indexOf("(")+1, newKey.indexOf(")")); + if(subName.endsWith("街") || subName.endsWith("路") || subName.contains("*")){ + newKey = newKey.replace("("+subName+")", ""); + } + newKey = newKey.replace("(","").replace(")","") + } + return newKey; + } + + def getBaiduMapCode(city){ + if(city){ + return baiduCityCodeMap[city] + } + return city + } + + static def searchBaiDuMap(String address, int cityCode) { + def rets = [] + try { + Thread.sleep(500) + String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${System.currentTimeMillis()}&auth=AUeHbGFG3ZMg2Bv8Y2b8GfVB3OGc35VOuxLNVzzzLHHtSAFSAS7I7vICu2I%400GdFvkGcuVtvvhguVtvyheuVtvCMGuVtvCQMuVtvIPcuxtw8wkvyuw904v7uv%40vcuVtvc3CuVtvcPPuVtveGvuNzHtAimNzCywiKDv7uvhgMuzVVtcvY1SGpuxRtKHZz3%40SdJMjjejCEBf0wd0vyMO7Ay7MOI&seckey=Im7p%2FanwNmQx078ijew%2Fsx5QUzMQU8M7SbrqWr3JGEc%3D%2C0-Ulo82T3hd3wuKMf_tTM6DbOSuS4Zlg9ZEtPAqKkNKXysGCBLK7zryshf7oP6p8_Hynb5vDXmd6C0TtcF9impZZK2kzkn1_SWeBVEJH4Zdh0n81wXv-1j6enHaAwiYj7T7eCaCw3g5vDcHvZxUxtBsRphpZI-Rco19dgpXwT9xb-XVcCb8GyvbMt5s2wIpb"; + def content = new HttpRequest().get(url, null, "utf-8") + JSONObject json = JSONObject.fromObject(content) + if (json && json.content && json.content.size() > 0) { + for (def loc : json.content) { +// String name = formatName(address) + def x = loc.x + def y = loc.y + if(x){ + x = x/100 + } + if(y){ + y = y/100 + } + if(x && y){ + def locJson = BaiDuApiUtils.geoconv(x, y, 6) + if(locJson){ + def baiduCity = RegionData.parseCityInAddress(loc.area_name) + def bean = [:] + def lng2 = locJson.get("x") + def lat2 = locJson.get("y") + def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2) + def lng = gmap.get("lng") + def lat = gmap.get("lat") + bean.uid = loc.uid + bean.baidu_lng2 = lng2 + bean.baidu_lat2 = lat2 + bean.longitude = lng + bean.latitude = lat + bean.name = loc.name + bean.address = getAddressInfo(loc.address_norm, loc.addr); + bean.old_addr = loc.addr + +// def baiduCity = RegionData.formatCity(loc.city_name); + bean.cityname = baiduCity + def regionStr = loc.area_name.replace(loc.city_name, "") + def baiduRegion = RegionData.formatRegion(baiduCity, regionStr); + bean.region = baiduRegion + if(loc.std_tag){ + bean.type = loc.std_tag + } + bean.flag = loc.acc_flag + rets << bean; + } + } + } + } + } catch(Exception e) { + } + return rets; + } + + def static formatName(String name) { + if(name.indexOf("·") != -1) { + name = name.replaceAll("·", "") + } + + if(name.indexOf(".") != -1) { + name = name.replace(".", "") + } + if(name.indexOf("(") != -1) { + name = name.replace("(", "") + } + if(name.indexOf(")") != -1) { + name = name.replace(")", "") + } + if(name.indexOf("-")!=-1){ + name = name.replace("-","") + } + return name.replaceAll("\\s+", "").toLowerCase() + } + + + static def getAddressInfo(def s, def addr){ + def arrs = s.split("]") + def map = [:] + if(arrs && arrs.size()>=3){ + if(arrs[0] && arrs[0].contains("PROV")){ + map.prov = arrs[0].substring(1, arrs[0].indexOf("(")) + } + if(arrs[1] && arrs[1].contains("CITY")){ + map.city = arrs[1].substring(1, arrs[1].indexOf("(")) + } + if(arrs[2] && arrs[2].contains("AREA")){ + map.region = arrs[2].substring(1, arrs[2].indexOf("(")) + } + } + if(addr){ + addr = addr.replace("[]","") + } + if(addr && map){ + if(map.region && !addr.contains(map.region)){ + addr = map.region + addr + } + if(map.city && !addr.contains(map.city)){ + addr = map.city + addr + } + + if(map.prov && !addr.contains(map.prov)){ + addr = map.prov + addr + } + } + return addr; + } + + + + def searchMapBySimilar(def city, def region, def name, def inpAddr, def rets, newKey){ + def bo = null; + if(rets){ + def maxScore = 0; + if(!bo || !maxScore){ + for(def o :rets){ + + if(!region.contains(o.region)){ + continue; + } + def s = o.name +// if(o.name && o.name.contains("(") && !o.name.toString().startsWith("(")){ +// s = o.name.substring(0, o.name.indexOf("(")) +// } +// s = formatName(s) + def addr = o.address + def mapKey = filterName(null, o.cityname, o.region, o.name) + + if(mapKey){ + def score = CosineSimilarAlgorithm.getSimilarityRatio(mapKey, newKey); + if(city==o.cityname){ + if(score > maxScore){ + maxScore = score; + o.matchType = "城市相同,名称相似" + o.city = o.cityname + bo = o + bo.score = maxScore + } + if(bo && score!=1){ + if(inpAddr){ + def baidu_addr = getSimpleAddress(o.cityname, o.old_addr) + if(baidu_addr){ + def addr_score = CosineSimilarAlgorithm.getSimilarityRatio(inpAddr, baidu_addr); + if(addr_score==1){ + o.matchType = "城市相同,名称相似,地址有关" + bo.score = bo.score + 0.1 + } + } + } + +// for(def word : filters){ +// if(mapKeyName.toString().endsWith(word)){ +// mapKeyName = mapKeyName.replace(word, "") +// } +// if(newName.toString().endsWith(word)){ +// newName = newName.replace(word, ""); +// } +// } +// +// +// if(mapKeyName && newName && mapKeyName.length()>=2){ +// score = CosineSimilarAlgorithm.getSimilarityRatio(mapKeyName, newName); +// if(score > maxScore){ +// maxScore = score +// o.matchType = "名称相似" +// o.city = obj.city +// bo = o +// bo.score = maxScore +// }else{ +// if(mapKeyName.length() == newName.length()){ +// def mapKeyNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(mapKeyName, " ") +// def newNameBY = com.fangjia.core.util.DataUtil.hanyuQuanpin(newName, " ") +// if(mapKeyNameBY == newNameBY){ +// maxScore = 0.977; +// o.matchType = "拼音相似" +// o.city = obj.city +// bo = o +// bo.score = maxScore +// } +// } +// } +// } + } + } + } + } + } + } + return bo; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy index f990711be..7a1d976ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy @@ -2,51 +2,172 @@ package com.fangjia.script.gj.crawl import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import org.bson.types.ObjectId class AnjukeDistrictMatch { static main(args) { def task = new AnjukeDistrictMatch(); - task.run(); +// task.run(); // task.run1() +// task.updateField() + task.fenQiRun() } - def run1(){ - def regionPointsMap = findRegionRange("重庆") - def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) + def fenQiRun(){ + def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水'] + for(def c : citys){ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region","name","virescence_percent","build_area","all_area","cubage_percent","master_district_name"]) + def distMap = findFiqiDatas(dists) + println c +" "+dists.size() +" "+ distMap.size() + addFieldValue(distMap); + } } + + def addFieldValue(distMap){ + def index = 0 + distMap.each {k,v-> + if(v.master_district_name!="master"){ + def masterKey = v.city +"##"+v.region+"##"+v.master_district_name; +// def masterObj = distMap.get(masterKey) + def masterObj = CommonDatautils.findDistrict(v.city, v.region, v.master_district_name, "district"); + if(masterObj){ + def udata = new BasicDBObject() + if(!v.virescence_percent && masterObj.virescence_percent){ + udata.put("virescence_percent", masterObj.virescence_percent) + } + + if(!v.build_area && masterObj.build_area){ + udata.put("build_area", masterObj.build_area) + } + + if(!v.all_area && masterObj.all_area){ + udata.put("all_area", masterObj.all_area) + } + + if(!v.cubage_percent && masterObj.cubage_percent){ + udata.put("cubage_percent", masterObj.cubage_percent) + } + + if(udata){ + index ++ + udata.put("m_date",new Date()) + def q = new BasicDBObject() + q.put("_id", new ObjectId(v._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + + } + } + } + println index; + } + + def findFiqiDatas(dists){ + def distMap = [:] + for(def d : dists){ + def master_district_name = d.master_district_name + if(master_district_name && master_district_name!="null"){ + distMap.put(d.city+"##"+d.region+"##"+d.name, d); + } + } + return distMap; + } + + + def updateField(){ + def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx" + ExcelUtils.multiThreadingModify(filePath, { row -> + if(row['Q'] && !row['Q'].contains("接口")){ + def dist = CommonDatautils.findSeaweedById(row['R'], ["virescence_percent"]) + if(dist && !dist.virescence_percent){ + def val = row['K'] + if(!val){ + val = row['J'] + } + if(!val){ + val = row['M'] + } + if(!val){ + val = row['O'] + } + if(!val){ + val = row['G'] + } + if(!val){ + val = row['E'] + } + if(!val){ + val = row['F'] + } + if(!val){ + val = row['H'] + } + if(!val){ + val = row['I'] + } + if(!val){ + val = row['L'] + } + if(!val){ + val = row['N'] + } + if(!val){ + val = row['O'] + } + + if(val){ + if(val.toString().toDouble().toInteger()==val.toString().toDouble()){ + val = val.toString().toDouble().toInteger() + } + def udata = new BasicDBObject() + udata.put("virescence_percent", val.toString()) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(dist._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + row['U'] = 1 + } + } + } + }) + } + +// def run1(){ +// def regionPointsMap = findRegionRange("重庆") +// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region)) +// } def run(){ - - - def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx" + def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx" ExcelUtils.multiThreadingModify(filePath,{row-> - if(row['A'] && !row['J']){ + if(row['A'] && !row['Q']){ def city = row['B'] def region = row['C'] def name = row['D'] - def address = row['E'] -// def flag = "别名匹配" - -// def dist = matchDist(city, null, name) -// if(!row['J'] && name.contains("(")){ -// name = name.replace("(${region})","").replace("(","").replace(")","") -// flag = "去括号匹配" -// dist = matchDist(city, region, name) -// } -// if(!dist){ -// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") -// flag = "映射别名匹配" -// } -// if(!dist) { -// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") -// if(rets && rets.size()==1){ -// dist = rets[0] -// } -// } + def flag = "名称匹配" + def dist = matchDist(city, region, name) + if(!row['D'] && name.contains("(")){ + name = name.replace("(${region})","").replace("(","").replace(")","") + flag = "去括号匹配" + dist = matchDist(city, region, name) + } + if(!dist){ + dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district") + flag = "映射别名匹配" + } + if(!dist) { + def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district") + if(rets && rets.size()==1){ + dist = rets[0] + } + } // // if(!dist){ @@ -61,23 +182,21 @@ class AnjukeDistrictMatch { // } // } -// if(dist){ -// row['J'] = flag -// row['K'] = dist._id.toString() -// row['L'] = dist.region -// row['M'] = dist.name -// row['N'] = dist.address -// } - - def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name) - if(jsonObject != null && jsonObject.precise!=-1) { -// row['J'] = "匹配接口" -// row['K'] = jsonObject.get("id") -// row['L'] = jsonObject.get("district") -// row['M'] = jsonObject.get("name") -// row['N'] = jsonObject.get("address") - row['M'] = jsonObject.get("name") + if(dist){ + row['Q'] = flag + row['R'] = dist._id.toString() + row['S'] = dist.region + row['T'] = dist.name + }else{ + def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + row['Q'] = "匹配接口" + row['R'] = jsonObject.get("id") + row['S'] = jsonObject.get("district") + row['T'] = jsonObject.get("name") + } } + } }) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index ce6242a31..a01d0b3eb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome { def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { - detail.total_price = price.toDouble() * 10000 + detail.total_price = (int)(price.toDouble() * 10000) } if (detail.total_price && detail.area) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 8e74425cb..bb9afd6f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome { def price = e.select(".totalPrice .number").text() if (price && price.isNumber()) { - detail.total_price = price.toDouble() * 10000 + detail.total_price = (int)(price.toDouble() * 10000) } if (detail.total_price && detail.area) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index e1976f1cb..ca7c9e388 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -3,12 +3,15 @@ package com.fangjia.script.gj.test import ch.qos.logback.core.db.dialect.DBUtil import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.Mongo @@ -24,6 +27,75 @@ DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "1 def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL) + +DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com") +DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com") +//ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因.xls", { d -> +//}) + + +ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因(标准化省市区).xls", { d -> + def info = d['A'] + if(info && !d['E']){ + def arrs = info.split("!~") + if(arrs && arrs[2] && arrs[3]){ +// def rMap = SeaweedDataMatchProcess.doMatch(d['B'], d['C'], arrs[4], arrs[4]); +// if(rMap) { +// d['E'] = rMap.name +// d['F'] = rMap.id +// } + + def address = null + if(d['G']){ + address = d['G'] + }else{ + address = arrs[4] + } + + def jsonObject = OpenApiClient.addressMatch(d['C'], address, "property", false, d['D'], null) + if(jsonObject != null && jsonObject.precise!=-1) { + d['H'] = jsonObject.get("city") + d['I'] = jsonObject.get("district") + d['J'] = jsonObject.get("name") + d['K'] = jsonObject.get("id") + } + +// def jsonObj = FangjiaClientUtils.addressStructure(d['B'], arrs[4]) +// if(jsonObj && jsonObj.result) { +// def street = null +// def street_number = null +// if (jsonObj.result.street) { +// d['E'] = jsonObj.result.street +// } +// if (jsonObj.result.streetNumber) { +// d['F'] = jsonObj.result.streetNumber +// } +// d['G'] = jsonObj.result.name +// } + +// def rets = findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, arrs[1], arrs[3]) +// if(rets && rets.size()==1){ +// def obj = rets[0] +// d['B'] = obj.province +// d['C'] = obj.fjw_city +// d['D'] = obj.fjw_district +// } + } + } +}); +// +//def findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, cityCode, regionCode){ +// def q = new BasicDBObject() +// q.put("city_code", cityCode.toString().toInteger()) +// q.put("district_code", regionCode.toString().toInteger()) +// def dbcursor = FANGJIA_AREA_CODE.find(q) +// def rets = [] +// for(def d : dbcursor){ +// rets << d +// } +// return rets; +//} + //ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d-> // def city = RegionData.formatCity(d['E']) // if(city && !d['J']){ @@ -103,33 +175,33 @@ DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "townshi // } //}) -ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> - if(!d['D']){ - def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) - if(infos && infos.size()==1){ - def info = infos[0] - d['D'] = info.province_code - d['E'] = info.area_code - d['F'] = info.street_code - } - } -}) - - -def findTownshipInfo(city, region, coll){ - def q = new BasicDBObject() - q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) - q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE)) - - def dbcursor = coll.find(q) - def rows = [] - for(def d : dbcursor){ -// if(d.street_name.contains(region)){ - rows << d +//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d-> +// if(!d['D']){ +// def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN) +// if(infos && infos.size()==1){ +// def info = infos[0] +// d['D'] = info.province_code +// d['E'] = info.area_code +// d['F'] = info.street_code // } - } - return rows -} +// } +//}) +// +// +//def findTownshipInfo(city, region, coll){ +// def q = new BasicDBObject() +// q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE)) +// q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE)) +// +// def dbcursor = coll.find(q) +// def rows = [] +// for(def d : dbcursor){ +//// if(d.street_name.contains(region)){ +// rows << d +//// } +// } +// return rows +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 8bca701d7..04e45fb88 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220601", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220701", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220901", "yyyyMMdd") +def endDate = DateUtil.parseDate("20221001", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -25,7 +25,7 @@ while (startDate.before(endDate)) { OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) if (!hashSet.contains(result)) { - def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + def item = ["60a3202c0b42a00001aad385", "qlkj", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] data << item hashSet << result } else { @@ -38,4 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file +ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志") + +