diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index af28b3fd4..42766952e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -633,7 +633,8 @@ class BigdataDealBatch extends BaseBatch{ def addDealPriceMatch(city){ - def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) +// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) + def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd") def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd")) // def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 22861ba1d..b25c4efbd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -43,7 +43,8 @@ class ExportOracleData { exportCommunityBuileType()// exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 之前按publish_date导出 + exportBigdataForsale(lastMonthStr +"-18", new_dists) //现在按updatetime导出 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //第三方网站权重表 @@ -659,13 +660,13 @@ class ExportOracleData { def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + - "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; allRows = getHouseDatas(sql) } if(allRows.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy index f1f2e17bc..88902dc7b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleDataTest.groovy @@ -12,6 +12,7 @@ import org.apache.commons.codec.digest.DigestUtils import java.sql.ResultSet +//导出估值 增量数据 新版 class ExportOracleDataTest { def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", @@ -45,7 +46,7 @@ class ExportOracleDataTest { exportCommunityBuileType()// exportThirdPartyPrice2(lastMonth, new_dists)// - exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 + exportBigdataForsale(lastMonthStr +"-18", new_dists) //2022-07-19 按更新时间导出数据 exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 //第三方网站权重表 @@ -658,13 +659,13 @@ class ExportOracleDataTest { def sql1 = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj " + - "where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; + "where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd') and distid not in ${newstr}"; def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ def sql = "select CITYID,FORSALEID,PROPCODE,CITYNAME,DISTID,DISTNAME,COMMID,COMMNAME, BUILDID,BUILDNO, BUILDNAME,UNITID,UNITNO,UNITNAME,HOUSEID,ADDRESS,BUILDYEAR,PROPTYPE,PROPYEARS,UNITPRICE,TOTALPRICE,PAYTYPE,FLOORNUM,FLOORCNT,\n" + " BUILDTYPE, HOUSESTRU,BUILDSTRU,BUILDAREA,LIVINGAREA,HOUSESTYLE,ROOMCNT,LIVINGROOMCNT,KITCHENCNT, BATHROOMCNT,HOUSEASPECT,FITMENT,IFELEVATOR,ELEVATORHOUSE,to_char(PUBLISHTIME,'yyyy-mm-dd hh24:mi:ss') as PUBLISHTIME,to_char(EXPIRETIME,'yyyy-mm-dd hh24:mi:ss') as EXPIRETIME,\n" + - " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and publishtime>=to_date('${publishDate}','yyyy-MM-dd')"; + " IFFIRST,DATASOURCE,AGENCYID,AGENCYNAME,PUBLISHER,PHONE,DESCRIPTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE,ATTACH from bigdata_forsale_gj where cityId='${cityCode}' and updatetime>=to_date('${publishDate}','yyyy-MM-dd')"; allRows = getHouseDatas(sql) } if(allRows.size()>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 295ab8915..8a2b1c55e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202211//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202209 - def endDate = 202211 + def sDate= 202212//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202210 + def endDate = 202212 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 4f5efcea8..bbd0ec13b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-10-01" + static def statDate = "2022-12-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2022年12月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年1月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 4b363b071..70249a1f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -73,7 +73,10 @@ class FangjiaClientUtils { def content = null try { address = address.replace("{","") - def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8") + def url = "http://192.168.0.19:31080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + if(city){ + url = url +"&city="+URLEncoder.encode(city,"UTF-8") + } // def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") content = new HttpRequest().newGet(url, null, "utf-8")