Merge branch 'script_gj' into 'master'
融合均价数据连接数据库修改 See merge request !477
This commit is contained in:
+2
-2
@@ -17,8 +17,8 @@ class AddThirdPartyPrice {
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"海西州","天门","大兴安岭","儋州",,"鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"
|
||||
def citys = ["林芝"]
|
||||
//
|
||||
def citys = ["林芝","海西州","天门","大兴安岭","儋州","鹤岗","日喀则","吐鲁番", "阿勒泰","迪庆州","潜江","塔城","克孜勒苏","和田","嘉峪关"]
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def c : citys){
|
||||
|
||||
+11
-11
@@ -58,7 +58,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
@@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class CcbCommonUtils {
|
||||
def countStr = args[0]
|
||||
if(countStr=="344"){
|
||||
citys = CcbCommonUtils.citys_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="92"){
|
||||
citys = CcbCommonUtils.citys_92
|
||||
|
||||
+15
-5
@@ -13,7 +13,7 @@ class CcbFtpTestUtils {
|
||||
static def ftpPassword = "ccb!@#"
|
||||
static def ftpPath = "/home/ccb_evaluation/output/"
|
||||
|
||||
static def ftpPath_rh = "/home/ccb_rh/output/"
|
||||
static def ftpPath_rh = "/home/ccb_ronghe/output/"
|
||||
|
||||
static SFTP sftp = new SFTP();
|
||||
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
@@ -70,16 +70,26 @@ class CcbFtpTestUtils {
|
||||
}
|
||||
|
||||
|
||||
static def listFileNames(month){
|
||||
static def listFileNames(type, month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
|
||||
Vector files = null;
|
||||
if(type && type=="rh"){
|
||||
files = sftp.listFiles(ftpPath_rh+ month+"/", sftpChannel);
|
||||
}else{
|
||||
files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
static def getFileLength(String path, String month){
|
||||
static def getFileLength(String type, String path, String month){
|
||||
sftpChannel = reConnect();
|
||||
def size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
|
||||
def size = 0;
|
||||
if(type && type=="rh"){
|
||||
size = sftp.getFileSize(ftpPath_rh+ month+"/"+path, sftpChannel)
|
||||
}else{
|
||||
size = sftp.getFileSize(ftpPath+ month+"/"+path, sftpChannel)
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -371,11 +371,11 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_92
|
||||
def month = 202209
|
||||
def month = 202210
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202210)
|
||||
def rets = findPeGteThirdParty(city, 202211)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
for(def m : rets){
|
||||
|
||||
+4
-4
@@ -54,7 +54,7 @@ class ExportOracleAllDataTest {
|
||||
// exportCommunityStaging()//没cityid
|
||||
// exportCommunityBuileType()//
|
||||
|
||||
exportThirdPartyPrice2(new_dists)//
|
||||
// exportThirdPartyPrice2(new_dists)//
|
||||
// exportBigdataForsale(new_dists) //2022-07-19
|
||||
// exportBigdataDeal( new_dists) //2022-07-01
|
||||
//
|
||||
@@ -325,7 +325,7 @@ class ExportOracleAllDataTest {
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' and createtime>=to_date('${date}','yyyyMMdd')";
|
||||
def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where CITYID='${cityCode}' ";//and createtime>=to_date('${date}','yyyyMMdd')
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.ID
|
||||
@@ -1172,14 +1172,14 @@ class ExportOracleAllDataTest {
|
||||
|
||||
def exportInOk(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
Vector files = CcbFtpTestUtils.listFileNames(curDate)
|
||||
Vector files = CcbFtpTestUtils.listFileNames(null, curDate)
|
||||
Iterator fileVec = files.iterator()
|
||||
def fileRows = []
|
||||
while(fileVec.hasNext()){
|
||||
ChannelSftp.LsEntry file = fileVec.next()
|
||||
|
||||
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
|
||||
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(file.getFilename(), curDate)
|
||||
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate)
|
||||
}
|
||||
}
|
||||
def fileName = "/ccb_evaluation/${curDate}.ok"
|
||||
|
||||
+4
-3
@@ -38,16 +38,17 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
CcbCommonUtils.deleteGuidedPrice(curDate)
|
||||
// CcbCommonUtils.deleteGuidedPrice(curDate)
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+2
-2
@@ -45,7 +45,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date())
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
@@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
|
||||
|
||||
}
|
||||
|
||||
|
||||
+18
-1
@@ -40,7 +40,7 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
@@ -50,8 +50,25 @@ class TradeStatLog {
|
||||
// task.deleteDealData2()
|
||||
// task.statTableCount()
|
||||
// task.filterTrade()
|
||||
task.updateDealPriceType()
|
||||
|
||||
}
|
||||
|
||||
def updateDealPriceType(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def c : citys){
|
||||
DEAL_PRICE_COLL.find(new BasicDBObject("city", c)).sort(new BasicDBObject("trade_date",-1)).batchSize(5000).each {it->
|
||||
if(it.total_price instanceof Double){
|
||||
index ++
|
||||
println index+" "+it.city+" "+it.total_price+" "+it._id.toString()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("total_price", (int)it.total_price)
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id", it._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def statTableCount(){
|
||||
|
||||
+67
-54
@@ -8,6 +8,7 @@ import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import groovy.sql.Sql
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
@@ -25,7 +26,7 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
@Override
|
||||
void process(Object args) {
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def curDate = "20221110"//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def new_dists = []//["350604","350605","410307","410308","450181"]
|
||||
//按天导(每天一个文件夹)
|
||||
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
|
||||
@@ -48,17 +49,17 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
// exportBigdataDistrictFJW()//
|
||||
// exportBigdataAliasFJW()//
|
||||
|
||||
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
exportBigdataCityPrice(queryDate)
|
||||
exportBigdataDistPrice(queryDate)
|
||||
exportBigdataCommPrice(queryDate)
|
||||
// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
// exportBigdataCityPrice(queryDate)
|
||||
// exportBigdataDistPrice(queryDate)
|
||||
// exportBigdataCommPrice(queryDate)
|
||||
|
||||
exportInOk()
|
||||
|
||||
// }
|
||||
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
|
||||
exportOk(curDate)
|
||||
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
|
||||
}
|
||||
|
||||
|
||||
@@ -66,9 +67,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
def exportBigdataCommunityGJRH(){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
@@ -133,9 +134,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
@@ -162,9 +163,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
@@ -190,9 +191,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
@@ -228,9 +229,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
});
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
@@ -258,9 +259,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
@@ -291,9 +292,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
@@ -323,9 +324,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
@@ -358,9 +359,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
@@ -382,9 +383,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
index++
|
||||
def curStartStr = curStartDateStr
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
@@ -482,9 +483,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
@@ -675,9 +676,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
println "BIGDATA_DEAL_FJW 导出完成......"
|
||||
@@ -686,9 +687,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def exportBigdataBuildingFJW(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def index = 0
|
||||
for(def city : citys) {
|
||||
@@ -743,9 +744,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def exportBigdataCityPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
@@ -789,9 +790,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def exportBigdataDistPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
@@ -836,9 +837,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def exportBigdataCommPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
@@ -899,9 +900,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
@@ -910,9 +911,9 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
@@ -921,23 +922,35 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_ronghe/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOk(curDate){
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
|
||||
def exportInOk(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
Vector files = CcbFtpTestUtils.listFileNames("rh", curDate)
|
||||
Iterator fileVec = files.iterator()
|
||||
def fileRows = []
|
||||
while(fileVec.hasNext()){
|
||||
ChannelSftp.LsEntry file = fileVec.next()
|
||||
|
||||
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
|
||||
fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength("rh", file.getFilename(), curDate)
|
||||
}
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
|
||||
def fileName = "/ccb_ronghe/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "融合统计文件长度ok文件 上传Ftp完成......"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+31
-1
@@ -755,7 +755,7 @@ class CommonDatautils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","湖畔佳苑")
|
||||
// q.put("name","万达华府一期")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -903,6 +903,36 @@ class CommonDatautils {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
public static def findSeaweedById(String id, def filelds){
|
||||
if(!id){
|
||||
return null;
|
||||
}
|
||||
try{
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", new ObjectId(id.toString().replace("[","").replace("]","")));
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
filelds.each{
|
||||
keys.put(it, 1);
|
||||
}
|
||||
try{
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
println "error_id:"+id
|
||||
e.printStackTrace()
|
||||
}
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrictBySid(String sid){
|
||||
|
||||
File diff suppressed because one or more lines are too long
+162
-43
@@ -2,51 +2,172 @@ package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class AnjukeDistrictMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeDistrictMatch();
|
||||
task.run();
|
||||
// task.run();
|
||||
// task.run1()
|
||||
// task.updateField()
|
||||
task.fenQiRun()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def regionPointsMap = findRegionRange("重庆")
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
def fenQiRun(){
|
||||
def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
|
||||
for(def c : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region","name","virescence_percent","build_area","all_area","cubage_percent","master_district_name"])
|
||||
def distMap = findFiqiDatas(dists)
|
||||
println c +" "+dists.size() +" "+ distMap.size()
|
||||
addFieldValue(distMap);
|
||||
}
|
||||
}
|
||||
|
||||
def addFieldValue(distMap){
|
||||
def index = 0
|
||||
distMap.each {k,v->
|
||||
if(v.master_district_name!="master"){
|
||||
def masterKey = v.city +"##"+v.region+"##"+v.master_district_name;
|
||||
// def masterObj = distMap.get(masterKey)
|
||||
def masterObj = CommonDatautils.findDistrict(v.city, v.region, v.master_district_name, "district");
|
||||
if(masterObj){
|
||||
def udata = new BasicDBObject()
|
||||
if(!v.virescence_percent && masterObj.virescence_percent){
|
||||
udata.put("virescence_percent", masterObj.virescence_percent)
|
||||
}
|
||||
|
||||
if(!v.build_area && masterObj.build_area){
|
||||
udata.put("build_area", masterObj.build_area)
|
||||
}
|
||||
|
||||
if(!v.all_area && masterObj.all_area){
|
||||
udata.put("all_area", masterObj.all_area)
|
||||
}
|
||||
|
||||
if(!v.cubage_percent && masterObj.cubage_percent){
|
||||
udata.put("cubage_percent", masterObj.cubage_percent)
|
||||
}
|
||||
|
||||
if(udata){
|
||||
index ++
|
||||
udata.put("m_date",new Date())
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(v._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
println index;
|
||||
}
|
||||
|
||||
def findFiqiDatas(dists){
|
||||
def distMap = [:]
|
||||
for(def d : dists){
|
||||
def master_district_name = d.master_district_name
|
||||
if(master_district_name && master_district_name!="null"){
|
||||
distMap.put(d.city+"##"+d.region+"##"+d.name, d);
|
||||
}
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
|
||||
def updateField(){
|
||||
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
if(row['Q'] && !row['Q'].contains("接口")){
|
||||
def dist = CommonDatautils.findSeaweedById(row['R'], ["virescence_percent"])
|
||||
if(dist && !dist.virescence_percent){
|
||||
def val = row['K']
|
||||
if(!val){
|
||||
val = row['J']
|
||||
}
|
||||
if(!val){
|
||||
val = row['M']
|
||||
}
|
||||
if(!val){
|
||||
val = row['O']
|
||||
}
|
||||
if(!val){
|
||||
val = row['G']
|
||||
}
|
||||
if(!val){
|
||||
val = row['E']
|
||||
}
|
||||
if(!val){
|
||||
val = row['F']
|
||||
}
|
||||
if(!val){
|
||||
val = row['H']
|
||||
}
|
||||
if(!val){
|
||||
val = row['I']
|
||||
}
|
||||
if(!val){
|
||||
val = row['L']
|
||||
}
|
||||
if(!val){
|
||||
val = row['N']
|
||||
}
|
||||
if(!val){
|
||||
val = row['O']
|
||||
}
|
||||
|
||||
if(val){
|
||||
if(val.toString().toDouble().toInteger()==val.toString().toDouble()){
|
||||
val = val.toString().toDouble().toInteger()
|
||||
}
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("virescence_percent", val.toString())
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(dist._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
row['U'] = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// def run1(){
|
||||
// def regionPointsMap = findRegionRange("重庆")
|
||||
// def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
// }
|
||||
def run(){
|
||||
|
||||
|
||||
|
||||
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\二期\\第三方小区字段补充\\绿化率.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath,{row->
|
||||
if(row['A'] && !row['J']){
|
||||
if(row['A'] && !row['Q']){
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = row['D']
|
||||
def address = row['E']
|
||||
// def flag = "别名匹配"
|
||||
|
||||
|
||||
// def dist = matchDist(city, null, name)
|
||||
// if(!row['J'] && name.contains("(")){
|
||||
// name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||
// flag = "去括号匹配"
|
||||
// dist = matchDist(city, region, name)
|
||||
// }
|
||||
// if(!dist){
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
|
||||
// flag = "映射别名匹配"
|
||||
// }
|
||||
// if(!dist) {
|
||||
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(city, region, name)
|
||||
if(!row['D'] && name.contains("(")){
|
||||
name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||
flag = "去括号匹配"
|
||||
dist = matchDist(city, region, name)
|
||||
}
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
|
||||
flag = "映射别名匹配"
|
||||
}
|
||||
if(!dist) {
|
||||
def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||
if(rets && rets.size()==1){
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// if(!dist){
|
||||
@@ -61,23 +182,21 @@ class AnjukeDistrictMatch {
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(dist){
|
||||
// row['J'] = flag
|
||||
// row['K'] = dist._id.toString()
|
||||
// row['L'] = dist.region
|
||||
// row['M'] = dist.name
|
||||
// row['N'] = dist.address
|
||||
// }
|
||||
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// row['J'] = "匹配接口"
|
||||
// row['K'] = jsonObject.get("id")
|
||||
// row['L'] = jsonObject.get("district")
|
||||
// row['M'] = jsonObject.get("name")
|
||||
// row['N'] = jsonObject.get("address")
|
||||
row['M'] = jsonObject.get("name")
|
||||
if(dist){
|
||||
row['Q'] = flag
|
||||
row['R'] = dist._id.toString()
|
||||
row['S'] = dist.region
|
||||
row['T'] = dist.name
|
||||
}else{
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false,region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['Q'] = "匹配接口"
|
||||
row['R'] = jsonObject.get("id")
|
||||
row['S'] = jsonObject.get("district")
|
||||
row['T'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+1
-1
@@ -240,7 +240,7 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toDouble() * 10000
|
||||
detail.total_price = (int)(price.toDouble() * 10000)
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
|
||||
+1
-1
@@ -316,7 +316,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toDouble() * 10000
|
||||
detail.total_price = (int)(price.toDouble() * 10000)
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
|
||||
@@ -3,12 +3,15 @@ package com.fangjia.script.gj.test
|
||||
import ch.qos.logback.core.db.dialect.DBUtil
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
@@ -24,6 +27,75 @@ DBCollection coll = MongoUtils.getCollectionByName("seeds_db", "city_seeds", "1
|
||||
def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", ZS_MONGODB_URL)
|
||||
|
||||
|
||||
DBCollection FANGJIA_AREA_CODE = MongoUtils.getCollectionByName("business", "fangjia_area_code", "mongo.db.fangjia.com")
|
||||
DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
|
||||
//ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因.xls", { d ->
|
||||
//})
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\证券\\国君见证失败原因(标准化省市区).xls", { d ->
|
||||
def info = d['A']
|
||||
if(info && !d['E']){
|
||||
def arrs = info.split("!~")
|
||||
if(arrs && arrs[2] && arrs[3]){
|
||||
// def rMap = SeaweedDataMatchProcess.doMatch(d['B'], d['C'], arrs[4], arrs[4]);
|
||||
// if(rMap) {
|
||||
// d['E'] = rMap.name
|
||||
// d['F'] = rMap.id
|
||||
// }
|
||||
|
||||
def address = null
|
||||
if(d['G']){
|
||||
address = d['G']
|
||||
}else{
|
||||
address = arrs[4]
|
||||
}
|
||||
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], address, "property", false, d['D'], null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
d['H'] = jsonObject.get("city")
|
||||
d['I'] = jsonObject.get("district")
|
||||
d['J'] = jsonObject.get("name")
|
||||
d['K'] = jsonObject.get("id")
|
||||
}
|
||||
|
||||
// def jsonObj = FangjiaClientUtils.addressStructure(d['B'], arrs[4])
|
||||
// if(jsonObj && jsonObj.result) {
|
||||
// def street = null
|
||||
// def street_number = null
|
||||
// if (jsonObj.result.street) {
|
||||
// d['E'] = jsonObj.result.street
|
||||
// }
|
||||
// if (jsonObj.result.streetNumber) {
|
||||
// d['F'] = jsonObj.result.streetNumber
|
||||
// }
|
||||
// d['G'] = jsonObj.result.name
|
||||
// }
|
||||
|
||||
// def rets = findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, arrs[1], arrs[3])
|
||||
// if(rets && rets.size()==1){
|
||||
// def obj = rets[0]
|
||||
// d['B'] = obj.province
|
||||
// d['C'] = obj.fjw_city
|
||||
// d['D'] = obj.fjw_district
|
||||
// }
|
||||
}
|
||||
}
|
||||
});
|
||||
//
|
||||
//def findProinvceAndCityAndRegion(FANGJIA_AREA_CODE, cityCode, regionCode){
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city_code", cityCode.toString().toInteger())
|
||||
// q.put("district_code", regionCode.toString().toInteger())
|
||||
// def dbcursor = FANGJIA_AREA_CODE.find(q)
|
||||
// def rets = []
|
||||
// for(def d : dbcursor){
|
||||
// rets << d
|
||||
// }
|
||||
// return rets;
|
||||
//}
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\二期\\国家行政区划特殊城市导出.xlsx",{d->
|
||||
// def city = RegionData.formatCity(d['E'])
|
||||
// if(city && !d['J']){
|
||||
@@ -103,33 +175,33 @@ DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "townshi
|
||||
// }
|
||||
//})
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
|
||||
if(!d['D']){
|
||||
def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
|
||||
if(infos && infos.size()==1){
|
||||
def info = infos[0]
|
||||
d['D'] = info.province_code
|
||||
d['E'] = info.area_code
|
||||
d['F'] = info.street_code
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
def findTownshipInfo(city, region, coll){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
|
||||
q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
|
||||
|
||||
def dbcursor = coll.find(q)
|
||||
def rows = []
|
||||
for(def d : dbcursor){
|
||||
// if(d.street_name.contains(region)){
|
||||
rows << d
|
||||
//ExcelUtils.modify("D:\\数据\\亲邻\\房价网省市区.xlsx",{d->
|
||||
// if(!d['D']){
|
||||
// def infos = findTownshipInfo(d['B'], d['C'], TOWNSHIP_CONN)
|
||||
// if(infos && infos.size()==1){
|
||||
// def info = infos[0]
|
||||
// d['D'] = info.province_code
|
||||
// d['E'] = info.area_code
|
||||
// d['F'] = info.street_code
|
||||
// }
|
||||
}
|
||||
return rows
|
||||
}
|
||||
// }
|
||||
//})
|
||||
//
|
||||
//
|
||||
//def findTownshipInfo(city, region, coll){
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city_name", Pattern.compile(city, Pattern.CASE_INSENSITIVE))
|
||||
// q.put("street_name", Pattern.compile('^.*'+region+'街道.*$', Pattern.CASE_INSENSITIVE))
|
||||
//
|
||||
// def dbcursor = coll.find(q)
|
||||
// def rows = []
|
||||
// for(def d : dbcursor){
|
||||
//// if(d.street_name.contains(region)){
|
||||
// rows << d
|
||||
//// }
|
||||
// }
|
||||
// return rows
|
||||
//}
|
||||
|
||||
|
||||
|
||||
|
||||
+6
-4
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20220601", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220701", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20220901", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20221001", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -25,7 +25,7 @@ while (startDate.before(endDate)) {
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007705"), eq("request_uri", "/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
if (!hashSet.contains(result)) {
|
||||
def item = ["60a3202c0b42a00001aad385", "qlkjgx", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
def item = ["60a3202c0b42a00001aad385", "qlkj", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
hashSet << result
|
||||
} else {
|
||||
@@ -38,4 +38,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
ExcelUtils.write("D:\\亲邻数据${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user