Merge remote-tracking branch 'origin/script_gj' into script_gj

This commit is contained in:
Jian Shen
2022-03-02 17:12:46 +08:00
47 changed files with 3101 additions and 416 deletions
@@ -13,7 +13,7 @@ import java.util.*;
*/
public class BaiDuApiUtils {
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
private static String AK = "ajgWIBd9rYsXed1hcDQqYvkX";
public static String[] AKS = new String[]{"hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
@@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils {
public static void main(String[] args) {
for (int i = 0; i < 1000; i++) {
System.out.println(random(0, 10));
System.out.println(random(-9, 9));
}
}
}
@@ -54,10 +54,10 @@ class BigdataDealBatch extends BaseBatch{
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
for(def city : citys){
addDealPriceMatch(city);
}
println citys.unique().size()
// for(def city : citys){
// addDealPriceMatch(city);
// }
// println citys.unique().size()
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220118, 20220219);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -149,8 +149,8 @@ class CcbCommonUtils {
public static def findTradesDetailById(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
// q.put("city", dist.city)
// q.put("region", dist.region)
q.put("area", new BasicDBObject('$gte', 20))
q.put("avg_price", new BasicDBObject('$gt', 0))
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
@@ -161,7 +161,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -367,7 +367,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","长隆小区")
// q.put("name","特房锦绣一方")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -488,4 +488,27 @@ class CcbCommonUtils {
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id), new BasicDBObject('$set',udata))
}
public static def findDistrictSeqno(){
def sql = "select cityid,commid,seqno from bigdata_position_seqno";
def map = new HashMap()
sqlInstance.eachRow(sql,{r->
def detailMap = map.get(r.cityid)
if(!detailMap){
detailMap = [:]
map.put(r.cityid, detailMap)
}
detailMap.put(r.commid, r.seqno)
});
return map
}
public static def findMaxSeqno(def cityid){
def sql = "select max(seqno) as max_seqno from bigdata_position_seqno where cityid='${cityid}'"
def maxSeqno = null;
sqlInstance.eachRow(sql,{r->
maxSeqno = r.max_seqno
});
return maxSeqno
}
}
@@ -0,0 +1,65 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jiangnanblank.SFTP
import com.jcraft.jsch.ChannelSftp
import org.apache.commons.codec.digest.DigestUtils
class CcbFtpTestUtils {
static def ftpServer = "192.168.0.86"
static def ftpPort = 17495
static def ftpUser = "ccb"
static def ftpPassword = "ccb!@#"
static def ftpPath = "/home/ccb/output/"
static def ftpPath_rh = "/home/ccb_rh/output/"
static SFTP sftp = new SFTP();
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
static def ftpUpload(type,outputPath_dist, month){
//上传到ftp
sftpChannel = reConnect();
if(type && type=="rh"){
sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel);
}else{
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
}
}
static def reConnect(){
if(sftpChannel == null){
println "sftpChannel 为空,需重连....."
if(sftp == null){
sftp = new SFTP();
}
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
}else{
println "sftpChannel 正常....."
}
return sftpChannel;
}
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
def data = new FileInputStream(new File(dataFileName))
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static main(args) {
}
}
@@ -1,15 +1,20 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -37,15 +42,74 @@ class DistrictDelete {
dataSource.setProperties(connectionProperties)
}
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new DistrictDelete()
// task.run()
// task.gzRun();
// task.deleteDistrict()
task.deleteCommPrice()
// task.deleteCommPrice()
// task.distHouseCountStat()
task.addThirdPartyStat()
// task.processRegion()
}
def processRegion(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d->
if(d['B']){
def region = RegionData.formatRegion("重庆",d['B'])
if(region){
d['B'] = region
}
}
},0)
}
def addThirdPartyStat(){
def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1)
def idMaps = [:]
for(def d : datas){
def id = d['A']
idMaps.put(id, d['D'])
}
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
def totalCount = new AtomicLong()
def map = findThirdPartyPrice("上海")
final CountDownLatch latch = new CountDownLatch(dists.size())
def thirdLists = Collections.synchronizedList(new ArrayList());
thirdLists << ["ID","区域","名称","总量"]
for(def i=0;i<dists.size();i++) {
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def count = idMaps.get(d._id.toString())
if (!count) {
def dist = CommonDatautils.findDistrictById(d._id.toString())
def priceMap = map.get(dist.region + "##" + dist.name)
if (priceMap) {
thirdLists << [dist._id.toString(), dist.region, dist.name, priceMap.size().toString()]
println dist._id.toString()+"@"+ dist.region+"@"+ dist.name+"@"+ priceMap.size()
}
}
// println totalCount.get() + " " + d.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\重庆农商行\\上海第三方.xlsx",thirdLists, "房源")
}
def run(){
@@ -60,6 +124,40 @@ class DistrictDelete {
}
}
def distHouseCountStat(){
def city = "重庆"
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def distCount = districts.size()
def rets = Collections.synchronizedList(new ArrayList());
rets << ["ID","区域","名称","总量"]
for(def i=0;i<districts.size();i++) {
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def count = findHouseCount(city, dist.region, dist.name, dist.alias)
def count = findHouses(city, dist.region, "sell", 20210701,20210801, dist.name, dist.alias)
println totalCount.get() +" "+city +" "+dist.region+" "+dist.name+" "+count
if(count){
rets << [dist._id.toString(), dist.region, dist.name, count]
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆7月份的房源统计.xlsx",rets, "房源")
}
def checkDist(city,dists){
def rets = Collections.synchronizedList(new ArrayList());
@@ -259,4 +357,82 @@ class DistrictDelete {
sqlInstance.execute(sql)
println "删除:"+tableName+" "+commid
}
def findThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city",city)
q.put("s_date", new BasicDBObject('$gte', 202112).append('$lte',202112))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.region+"##"+db.name
def prices = map.get(key)
if(!prices){
prices = []
map.put(key, prices)
}
if( db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
prices << db.zhugezhaofang_esf_price
}
if(db.anjuke_esf_price && db.anjuke_esf_price>3000){
prices << db.anjuke_esf_price
}
if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
prices << db.get("58tc_esf_price")
}
if(db.soufang_esf_price && db.soufang_esf_price>3000){
prices << db.soufang_esf_price
}
if(db.lianjia_esf_price && db.lianjia_esf_price>3000){
prices << db.lianjia_esf_price
}
if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
prices << db.cityhouse_esf_price
}
if(db.maitian_esf_price && db.maitian_esf_price>3000){
prices << db.maitian_esf_price
}
if(db.beike_esf_price && db.beike_esf_price>3000){
prices << db.beike_esf_price
}
if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
prices << db.get("365house_esf_price")
}
if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
prices << db.zhongyuan_esf_price
}
if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
prices << db.tuboshi_esf_price
}
if( db.qfang_esf_price && db.qfang_esf_price>3000){
prices << db.qfang_esf_price
}
if(db.fangtan_esf_price && db.fangtan_esf_price>3000){
prices << db.fangtan_esf_price
}
if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
prices << db.fangduoduo_esf_price
}
if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
prices << db.get("5i5j_esf_price")
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
}
return map
}
}
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
def prices = CcbCommonUtils.findGuidePrice(city, "2022-02-1", "2022-03-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -21,26 +21,25 @@ class ExportOracleData {
}
def run() {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// //不会变
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
//
// exportGuidedPrice()//随人工改变
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
//
// //处理小区情况
// exportBigdataCommunity()
// exportCommunityStaging()
// exportCommunityBuileType()
//不会变
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
// exportThirdPartyPrice2(lastMonth)
// exportBigdataForsale("2021-12-19")
// exportBigdataDeal("2021-12-01")
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
// //处理小区情况
exportBigdataCommunity()
exportCommunityStaging()
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-01-19")
exportBigdataDeal("2022-01-01")
//计算结果相关
@@ -56,11 +55,11 @@ class ExportOracleData {
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// def curDate = curMonth.toString()+"28"
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
//ok文件
exportLastFile()
// exportLastFile()
}
@@ -0,0 +1,416 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new ExportOracleEvaluationData()
task.run()
}
def run() {
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
//计算结果相关
exportAvmEvaluationGJNB(curMonth)
exportAVMEvaluationItemGJNB(curMonth)
exportCityPrice(curMonth)
exportRegionPrice(curMonth)
exportCommunityBasePriceGj(curMonth)//已合下面的合并
exportCommunityBasePrice(curMonth)
exportCommunityPdescribeGj(curMonth)
exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// ok文件
exportOkFile()
}
def exportAvmEvaluationGJNB(yearMonth){
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.COMMUNITY_NUM
data << r.HAS_PRICE_COMMUNITY_NUM
data << r.PRICE_COVERAGE
data << r.ARTIFICIAL_CALC_NUM
data << r.ARTIFICIAL_CALC_COVERAGE
data << r.SAMPLE_CALC_NUM
data << r.SAMPLE_CALC_COVERAGE
data << r.ANN_CALC_NUM
data << r.ANN_CALC_COVERAGE
data << r.TEST_NUM
data << r.TEST_COMM_NUM
data << r.TEST_COMM_COVERAGE
data << r.PPE15_COVERAGE
data << r.RATE_MEDIAN
data << r.COEFFICIENT
data << r.DEAL_TEST_CASE_NUM
data << r.DEAL_TEST_COMM_NUM
data << r.DEAL_TEST_COMM_COVERAGE
data << r.DEAL_PPE15_COVERAGE
data << r.DEAL_COEFFICIENT
data << r.DEAL_RATE_MEDIAN
data << r.THIRD_TEST_CASE_NUM
data << r.THIRD_TEST_COMM_NUM
data << r.THIRD_TEST_COMM_COVERAGE
data << r.THIRD_PPE15_COVERAGE
data << r.THIRD_COEFFICIENT
data << r.THIRD_RATE_MEDIAN
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
}
def exportAVMEvaluationItemGJNB(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.CONFIDENCE
data << r.TARGET_AVG_PRICE
data << r.DEALDATE
data << r.UNIT_PRICE
data << r.BUILDAREA
data << r.FITMENT
data << r.HOUSEASPECT
data << r.FLOORNUM
data << r.FLOORCNT
data << r.DATA_TYPE
data << r.RATE
data << r.PE
data << r.CREATETIME
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
}
def exportCommunityBasePrice(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityBasePriceGj(yearMonth){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.COMMID
data << r.COMMNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.AVG_PRICE
data << r.CALC_TYPE
data << r.FAKE_RATE
data << r.DIFF_RATE
data << r.CONFI_RATE
data << r.CORR_RATE
data << r.CONFIDENCE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
}
def exportCityPrice(month){
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "CITY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "CITY_PRICE_GJ 上传Ftp完成......"
}
def exportRegionPrice(month){
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.YEAR_MONTH
data << r.CALC_AVG_PRICE
data << r.PRICE
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
}
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
}
def exportCommunityPdescribeGj(month){
// def citys = CcbCommonUtils.citys;
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
//
// for(def city : citys) {
// def cityCode = cityCodeDataMap.get(city)
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.ID
data << r.COMMUNITY_ID
data << r.CITYID
data << r.CITY_NAME
data << r.DISTRICT_CODE
data << r.DISTRICT_NAME
data << r.COMMUNITY_NAME
data << r.PQ_NAME
data << r.BASE_PRICE
data << r.CONFIDENCE_LEVEL
data << r.CNT
data << r.MIN_PRICE
data << r.MEDIANPRICE
data << r.MEANPRICE
data << r.MAX_PRICE
data << r.TIME
data << r.FIRSTDATAT
data << r.THIRDDATA
data << r.RELATIVEERROR
data << r.SKEWNESS
data << r.PQ_PRICE_RANGE
data << r.DISTRICT_PRICE_RANGE
data << r.DESCRIPTION
data << r.PRIV_BASE_PRICE
data << r.IS_ARTTFICIAL
data << r.PRIV_FINAL_PRICE
data << r.CREATE_DATE
data << r.PRICE
data << r.PRICE_SOURCE
data << r.CASE_MD_PRICE
data << r.CASE_AVG_PRICE
data << r.INDEX_LABEL
data << r.MAIN_PROPERTY
data << r.COMMTYPE
data << r.NEW_DEAL_PRICE
data << r.NEW_DEAL_DATE
data << r.PRICE_MOM_RATIO
data << r.MAX_MOM_RATIO
data << r.PRICE_YOY_RATIO
data << r.SATUR_RATIO
data << r.NEIGHBOR_PRICE
allRows << data
});
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
// }
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
}
def exportCommCaseDetailGj(month){
def citys = CcbCommonUtils.citys;
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
for(def city : citys){
def cityCode = cityCodeDataMap.get(city)
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.CITYID
data << r.COMMUNITY_ID
data << r.NETLISTING_ID
data << r.PRICE_DATE
data << r.CREATE_DATE
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
allRows.clear()
}
});
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
println "comm_case_detail_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "comm_case_detail_gj 上传Ftp完成......"
}
def exportOkFile(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def dataCodeFileName = "/ccb/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
}
}
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-02-1", "2022-03-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -23,9 +23,10 @@ class LoadGuide {
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
}
def match(def filename){
@@ -48,15 +49,15 @@ class LoadGuide {
def datas = ExcelUtils.read(filePath,0, 1)
def rets = []
for(def d : datas){
if(d['E']){
if(d['D']){
def o = new BasicDBObject()
o.city = d['E']
o.city = d['D']
o.region = d['A']
o.name = d['B']
o.avg_price = d['C'].toString().toDouble().toInteger()
o.match_city = d['E']
o.match_region = d['F']
o.match_name = d['G']
o.match_city = d['D']
o.match_region = d['E']
o.match_name = d['F']
o.source_type = 1 //1:成交参考价 2:参考价
o.c_date = new Date()
rets << o
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
}
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
def testMap = findCaluateCityPPE(sqlInstance236, 202201)
ExcelUtils.modify(fileName,{d->
if(d['C']){
def oldcount = dealMap.get(d['C'])
d['T'] = oldcount
def casecount = testMap.get(d['C'])
d['U'] = casecount
println d
}
},0)
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['V'] = oldcount
// def casecount = testMap.get(d['C'])
// d['W'] = casecount
// println d
// }
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -271,19 +271,24 @@ ExcelUtils.modify(fileName,{d->
//},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d->
// if(d['A']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d->
// def flag = d['M']
// if(flag==0.0){
// if(d['D']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['A']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['D']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// }
// }
//
//},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
// if(d['A']){
// def region = d['B']
@@ -777,6 +782,23 @@ def getStreetInfo(def city, def glocation){
//}
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据")
def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1)
def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1)
def keys = []
for(def d : datas2){
keys << d['A']+"##"+d['C']+"##"+d['B']
}
def index = 0
for(def d : datas1){
def k = d['A']+"##"+d['C']+"##"+d['B']
if(!keys.contains(k)){
index ++
println index +" "+k
}
}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202110
def endDate = 202112
def sDate= 202201//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202111
def endDate = 202201
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -9,6 +9,7 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import groovy.sql.Sql
import org.bson.types.ObjectId
@@ -40,7 +41,7 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -48,6 +49,7 @@ class TradeStatLog {
// task.deleteDealData()
// task.deleteDealData2()
// task.statTableCount()
task.filterTrade()
}
@@ -73,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","11月总量","12月总量","1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-10-01"
def date = "2021-11-01"
def endDate = "2021-12-01"
def newDate = "2022-01-01"
def startDate = "2021-11-01"
def date = "2021-12-01"
def endDate = "2022-01-01"
def newDate = "2022-02-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -129,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-01成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -479,8 +481,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2021-12-01"
def endDate = "2022-01-01"
def startDate = "2022-01-01"
def endDate = "2022-02-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -561,7 +563,7 @@ class TradeStatLog {
}
println city +" "+rets.size()
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
}
def deleteDistrict(){
@@ -706,4 +708,24 @@ class TradeStatLog {
}
def filterTrade(){
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx",{d->
if(!d['J']){
def count = findTradeById(d['A'])
if(count){
d['J'] = "其它原因过滤"
}
}
},0)
}
def findTradeById(id){
def q = new BasicDBObject()
q.put("_id", new ObjectId(id))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
}
@@ -12,9 +12,90 @@ import com.fangjia.core.util.BaiDuApiUtils
class DistrictMarge {
static main(args) {
def task = new DistrictMarge()
task.run()
//task.run()
task.repeatAddress()
}
def repeatAddress(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "lng","address","name_cat"])
findRepead(c, districts)
}
}
def findRepead(city, districts){
def map = [:]
for(def d : districts){
if(d.name_cat == 0 && d.lng){
if(d.address && d.address.contains("号")){
def list = map.get(d.address)
if(!list){
list = []
map.put(d.address, list)
}
list << d
}
}
}
def newMap = [:]
map.each {k,vals->
if(vals.size()>=2){
vals.sort{o1,o2->
return o1.name.size() <=> o2.name.size()
}
newMap.put(k,vals)
}
}
newMap.each {k,vals->
def names = []
for(def d : vals){
def newKey = d.name
if(d.name.contains("小区")){
newKey = d.name.toString().replace("小区", "")
}
if(!names.contains(newKey)){
names << newKey
}else{
d.flag = "删除"
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId( d._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
def values = newMap.values()
def allRets = []
for(def list : values){
allRets.addAll(list)
}
def results = []
def delcount = 0
for(def r : allRets){
def rows = []
rows << r._id.toString()
rows << r.city
rows << r.name
rows << r.address
rows << r.name_cat
if(r.flag){
rows << r.flag
delcount++
}
results << rows
}
// ExcelUtils.write("d:\\数据\\小区\\删除重复地址(${city}).xlsx", results, "sheet")
println "${city} 共:"+map.size()+" 2个以上:"+newMap.size()+" delcount:"+delcount
}
def run(){
def rets = getDatas()
def deleteIndex = 0
@@ -433,23 +433,43 @@ def city = "
// }
//})
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
if(dist){
def alias = dist.mapping_alias
if(!alias){
alias = []
}
if(alias && !alias.contains(d['C'])){
index++
println index
alias << d['C']
def udata = new BasicDBObject()
udata.put("mapping_alias",alias)
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
}
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
// if (d['B']) {
// def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
// if(dist){
// def alias = dist.mapping_alias
// if(!alias){
// alias = []
// }
// if(alias && !alias.contains(d['C'])){
// index++
// println index
// alias << d['C']
// def udata = new BasicDBObject()
// udata.put("mapping_alias",alias)
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// }
// }
// }
//})
ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
if(d['A']){
try{
def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
if(dist){
d['B'] = dist.city
d['C'] = dist.region
d['D'] = dist.name
def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
d['E'] = price
}
}catch(Exception e){
println d['A']
}
}
})
@@ -0,0 +1,89 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
class MapAddressMatch {
static main(args) {
def task = new MapAddressMatch()
task.run()
}
def run(){
def city = "北京"
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\北京地址重复数据.xlsx", { d ->
index++
println index +" "+ d['D']+" "+d['E']
if(!d['G']){
// def gdData = FangjiaClientUtils.getMapData(city, d['D'],d['E'], d['K'],"", "高德")
// if(gdData){
// d['F'] = gdData.district
// d['G'] = gdData.name
// d['H'] = gdData.address
// d['I'] = gdData.type
// d['J'] = "高德"
// }else{
def rets = OpenApiUtils.getGaoDeSearchLocation(city,d['D'],d['E'])
def map = findStandardObject(city,d['D'],d['E'], rets);
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(city,d['D'],d['E']);
map = findStandardObject(city,d['D'],d['E'], rets);
}
if(map){
if(map.adname){
d['F'] = map.adname
}else if(map.region){
d['F'] = map.region
}
d['G'] = map.name
d['H'] = map.address
if(map.type){
d['I'] = map.type
}else if(map.estateType){
d['I'] = map.estateType
}
d['J'] = map.source
}
// }
}
},0)
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
continue;
}
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
}
@@ -55,6 +55,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// addBigdataDistrictFJW()
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
def index = 0
for (def city : citys) {
index++
@@ -88,20 +90,20 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
}
}
println index +" "+city +" 小区加载完成 "+districts.size()
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
println index +" "+city +" 1.小区表完成"
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 2.小区建筑结构表"
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 3.小区房屋类型表"
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
println index +" "+city +" 4.小区建筑年代表"
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
println index +" "+city +" 5.小区费用表"
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
println index +" "+city +" 6.小区房屋用途表"
addBigdataPositionGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 小区加载完成 "+districts.size()
// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 1.小区表完成"
// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 2.小区建筑结构表"
// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 3.小区房屋类型表"
// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 4.小区建筑年代表"
// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 5.小区费用表"
// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
// println index +" "+city +" 6.小区房屋用途表"
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
println index +" "+ city + " 7.地理位置表"
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
@@ -552,18 +554,17 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
// }
}
def addBigdataPositionGJ(districts, def cityid){
def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){
def rets = []
def index = 0
for(def d : districts) {
index++
def ret = []
def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap);
ret << d.cityid
ret << "5"
ret << d.commid
ret << "1"
ret << index
ret << seqno
ret << d.lng
ret << d.lat
ret << ""
@@ -589,49 +590,29 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
def headers = []
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
println "BIGDATA_POSITION_FJW 数据文件输出完成......"
}
// sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'")
//
// def sqls = []
// def index = 1
// for(def d : districts){
// def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark},"
// index++
// if(d.createtime){
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
// }else{
// sql = sql + "null,"
// }
// if(d.updatetime){
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
// }else{
// sql = sql + "null,'${d.sourcecode}')"
// }
// sqls << sql
//
// if (sqls.size() >= batchsize) {
// try {
// sqlInstance.withBatch(batchsize, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
//
// } catch (Exception e) {
// e.printStackTrace()
// }
// sqls.clear()
// }
//
// }
//
// if (sqls.size() > 0) {
// sqlInstance.withBatch(100, { stmt ->
// sqls.each { def s ->
// stmt.addBatch s
// }
// })
// }
def findSeqno(cityid, commid, citySeqnoMap){
def detailMap = citySeqnoMap.get(cityid)
if(detailMap){
def seqno = detailMap.get(commid)
if(seqno){
return seqno
}else{
seqno = CcbCommonUtils.findMaxSeqno(cityid)
if(seqno){
seqno = seqno + 1
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
def insertSql = "INSERT INTO bigdata_position_seqno(CITYID,COMMID,SEQNO,CREATETIME) VALUES ('${cityid}','${commid}',${seqno},to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'))"
sqlInstance.withBatch(batchsize, { stmt ->
stmt.addBatch insertSql
})
return seqno
}
}
}else{
println cityid+" 没有该城市的小区序号"
}
}
def addBigdataDistrictFJW(){
@@ -0,0 +1,742 @@
package com.fangjia.script.ccbevaluation.ronghe
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.sql.ResultSet
import java.sql.ResultSetMetaData
@BatchOwner(Watcher.GaoJuan)
class ExportDistrictRongHeTest extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new ExportDistrictRongHeTest(), args);
}
@Override
void process(Object args) {
def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
exportBigdataCommunityGJRH()
exportBigdataBuildingFJW()
exportBigdataCommfeeGj()
exportBigdataCommbuildstruGJ()
exportBigdataCommhousetypeGJ()
exportBigdataCommbuildyearGj()
exportBigdataCommhouseuseGJ()
exportBigdataPositionGJ()
exportBigdataDistrictFJW()
exportBigdataAliasFJW()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportEvaluateRate()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
exportOk(curDate)
}
//
// def runByCity(){
// def citys = CcbCommonUtils.citys;
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//
//
//// //检查ok文件是否上传
//// exportAllBigdataDealFJW()
//// for(def city : citys) {
//// def cityCode = cityCodeDataMap.get(city)
////
// //按天导(一个文件夹下)
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
// //按天导(每天一个文件夹)
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
//
//// exportBigdataCommunityGJRH(cityCode)
//// exportBigdataBuildingFJW(cityCode)
//// exportBigdataCommfeeGj(cityCode)
////
//// exportBigdataCommbuildstruGJ(cityCode)
//// exportBigdataCommhousetypeGJ(cityCode)
//// exportBigdataCommbuildyearGj(cityCode)
//// exportBigdataCommhouseuseGJ(cityCode)
//// exportBigdataPositionGJ(cityCode)
//// exportBigdataDistrictFJW(cityCode)
//// exportBigdataAliasFJW(cityCode)
////
//// exportBigdataBuildprojFjw(cityCode)
//// exportBigdataCommlayoutpicFjw(cityCode)
//// }
//// exportEvaluateRate()
//
////
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
//
// exportOk()
// }
def exportBigdataCommunityGJRH(cityCode){
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMMNAME
data << r.COMMALIAS
data << r.DISTID
data << r.STREET
data << r.STREETNO
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.COMMTYPE
data << r.IFNEW
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << ''
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDSTRU
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
}
def exportBigdataCommhousetypeGJ(cityCode){
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildyearGj(){
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDYEAR
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
}
def exportBigdataCommfeeGj(){
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMPID
data << r.FEE
data << r.HOUSEUSE
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.FEETYPE
data << r.NOTE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
}
def exportBigdataCommhouseuseGJ(){
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSEUSE
data << r.HOUSECNT
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
}
def exportBigdataPositionGJ(){
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.OBJECTTYPE
data << r.OBJECTID
data << r.POSITIONTYPE
data << r.SEQNO
data << r.LONGITUDE
data << r.LATITUDE
data << r.ALTITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
}
def exportBigdataDistrictFJW(){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
}
def exportBigdataAliasFJW(){
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.city_id
data << r.ca_id
data << r.address_id
data << r.address_name
data << r.status
data << r.exp_dt
data << r.version
data << r.is_key_name
data << r.data_source
data << r.description
data << r.alias_grade
data << r.addr_type_cd
data << r.ifexamine
data << r.ifdisplay
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
}
def exportBigdataDealFJW(dealdate){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def stop = false
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def index = 0
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
while(!stop){
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
def allRows = []
if(index==1){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
}
if(curStartDateStr == curSndDateStr){
stop = true
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
def curEndStr = DateUtils.add(curStartStr, 1)
curStartDateStr = curEndStr
}
}
def exportBigdataDealFJW2(curDate, dealdate){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......"
}
def exportBigdataBuildingFJW(cityCode){
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.BUILDALIAS
data << r.DISTID
data << r.COMMID
data << r.BUILDYEAR
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.FLOORCNT
data << r.UNITCNT
data << r.HOUSECNT
data << r.IFELEVATOR
data << r.ELEVATORCNT
data << r.ELEVATORHOUSE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.VALID_FLAG
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
def exportEvaluateRate(){
def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.RATE_TYPE
data << r.RATE
data << r.RANK
data << r.CITY
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "EVALUATE_RATE_FJW 上传Ftp完成......"
}
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
}
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
}
}
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","金辉伯爵山")
// q.put("name","玉泉路甲46号院")
DBObject keys = new BasicDBObject();
if(fields){
@@ -829,6 +829,7 @@ class CommonDatautils {
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -23,8 +23,8 @@ class FzDataResultStat {
// task.regionPriceTrendRun()
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:]
@@ -53,7 +53,7 @@ class FzDataResultStat {
}
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
}
//ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
@@ -94,7 +94,7 @@ class FzDataResultStat {
}
def prices = exportRegion(city, months)
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据")
ExcelUtils.write("D:\\数据\\福州\\福州6区均价半年走势_${curMonth}.xlsx", prices, "数据")
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202201.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -143,8 +143,8 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
}
if(!retMap.anjuke_esf_price && d.anjuke_esf_price){
retMap.anjuke_esf_price = d.anjuke_esf_price
if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){
retMap.anjuke_esf_price = d.get("58tc_esf_price")
}
if(!retMap.soufang_esf_price && d.soufang_esf_price){
retMap.soufang_esf_price = d.soufang_esf_price
@@ -152,6 +152,9 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.lianjia_esf_price && d.lianjia_esf_price){
retMap.lianjia_esf_price = d.lianjia_esf_price
}
if(!retMap.lianjia_esf_price && d.beike_esf_price){
retMap.lianjia_esf_price = d.beike_esf_price
}
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
retMap.cityhouse_esf_price = d.cityhouse_esf_price
@@ -45,7 +45,7 @@ class EsHouseStat {
public static void main(String[] args){
def task = new EsHouseStat();
//task.run();
task.run();
// task.testRun("ÉîÛÚ")
// task.exportOpenLog();
// task.exportMongoLog()
@@ -59,7 +59,7 @@ class EsHouseStat {
// task.szRentHouseStat();
// task.addPrice();
task.statNoHouseCCBData();
// task.statNoHouseCCBData();
}
def addPrice(){
@@ -337,21 +337,6 @@ class EsHouseStat {
return total;
}
def findHouse(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def total = 0;
while (rs.next()){
total = rs.getInt("count")
}
return total;
}
def findHouseSource(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT * FROM \"seawater-20*\" WHERE 1=1")
@@ -373,6 +358,21 @@ class EsHouseStat {
return statMap;
}
def findHouse(def city, def startDate, def endDate){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def total = 0;
while (rs.next()){
total = rs.getInt("count")
}
return total;
}
def testRun(def city){
@@ -149,6 +149,7 @@ class OpenApiUtils {
ret.lng2 = gmap.get("lng")
ret.lat2 = gmap.get("lat")
ret.estateType = ret.type
ret.source = "高德"
}catch (Exception e){
}
@@ -250,14 +251,14 @@ class OpenApiUtils {
}
if(type && type.contains("出入口")){
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
}else{
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
poiList.add(jo);
}
}else{
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
}
}
}else{
@@ -302,7 +303,7 @@ class OpenApiUtils {
def distname = formatName(ret.get("name"))
address = ret.get("address")
if((distname && distname.contains(name)) || (distname && name.contains(distname)) || (address && address.contains(name)) || ( address && name.contains(address))){
if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
// if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
lng2 = location.lng
lat2 = location.lat
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
@@ -311,7 +312,7 @@ class OpenApiUtils {
def mapRegion = ret.get("area")
distRegion = mapRegion
districtName = distname
if(address && address.contains(mapRegion)){
if(address && mapRegion && address.contains(mapRegion)){
address = address.substring(address.indexOf(mapRegion)+mapRegion.size());
}
if(ret.get("detail_info")){
@@ -330,6 +331,7 @@ class OpenApiUtils {
map.name = districtName
map.region = distRegion
map.estateType = estateType
map.source = "百度"
rets << map
lat = null
@@ -340,12 +342,18 @@ class OpenApiUtils {
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
}
}else{
println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
}
// }else{
// println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
// }
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
def score1 = 0
if(distname && name){
score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
}
def score2 = 0
if(address && name){
score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
}
if(score1 || score2){
rets << map
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】(score:"+score1+"#"+score2+")"
@@ -31,7 +31,7 @@ class RegionPointCrawl {
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
@@ -7,11 +7,11 @@ email.setSSL(true);
email.setHostName("smtp.exmail.qq.com");
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码
email.addTo("gaojuan@fangjia.com"); //接收方
email.setFrom("service@fangjia.com"); //发送方
email.setSubject("test"); //标题
email.setContent("你?", "text/html");
email.setContent("测试", "text/html");
//email.setHostName("smtp.exmail.qq.com");
//email.setSSL(true);
@@ -7,13 +7,17 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.script.util.DbUtils
/**
* 模糊地址经纬度更新
@@ -44,13 +48,16 @@ class VagueAddressLocationBatch extends BaseBatch {
void process(Object args) {
//"北京","常州",
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
def citys =["上海"] //CommonDatautils.findAllCity();
def citys =["北京"] //CommonDatautils.findAllCity();
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
//市谊大厦 华三新村 阳光家园
for (def city : citys) {
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat2", "lat", "lng2", "lng"]);
def rets = findVagueAddress(city, districts);
findAndUpdateLocation(rets);
writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
// writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
writeCSV(results, "d:\\数据\\结果\\" + city + "地址模糊.csv")
// ExcelUtils.write("d:\\数据\\结果\\" + city + "地址模糊.xlsx", rets, "数据")
}
@@ -58,6 +65,7 @@ class VagueAddressLocationBatch extends BaseBatch {
def findVagueAddress(def city, def districts) {
def rets = []
def rows = []
for (def district : districts) {
def address = district.address;
if (address) {
@@ -66,10 +74,13 @@ class VagueAddressLocationBatch extends BaseBatch {
//没地址
if (!address) {
rets << district;
// rows << [district.city, district.region, district.name, district.address]
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
} else {
if (!findPattern(address)) {
rets << district;
// rows << [district.city, district.region, district.name, district.address]
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
} else {
println district.city + " " + district.address + " 精确地址"
@@ -97,7 +108,7 @@ class VagueAddressLocationBatch extends BaseBatch {
} else if (address.indexOf("路") == -1 && address.indexOf("街") == -1 && address.indexOf("大道") == -1 && address.indexOf("巷") == -1 && address.indexOf("北里") == -1 && address.indexOf("南里") == -1 && address.indexOf("东里") == -1 && address.indexOf("西里") == -1) {
return null;
} else {
pattern = Pattern.compile("[路|街|大道|巷]\\d+");
pattern = Pattern.compile("[路甲|路|街|大道|巷|弄|里]\\d+");
matcher = pattern.matcher(address)
if(matcher.find()){
return s;
@@ -123,16 +134,14 @@ class VagueAddressLocationBatch extends BaseBatch {
for (def ret : rets) {
index++;
if(ret.name!="药师新村"){
continue
}
def locationMap = findLatAndLng(ret);
if (locationMap) {
updateLocation(locationMap, ret);
} else {
println "[没找到] " + ret.city + " " + ret.region + " " + ret.name + " " + ret.address
}
println "【${index}/${rets.size()}】************************************************************************************"
println "【${index}/${rets.size()}】**************【${ret.region}---${ret.name}----${ret.address}】**********************************************************************"
}
}
@@ -148,14 +157,14 @@ class VagueAddressLocationBatch extends BaseBatch {
msgMap.db_lng2 = ret.lng2
BasicDBObject udata = new BasicDBObject()
if (ret.lat2 != locationMap.lat2) {
udata.put('lat2', locationMap.lat2)
udata.put('lng2', locationMap.lng2)
if (ret.lat2 && ret.lat2 != locationMap.lat2) {
msgMap.m_lat2 = locationMap.lat2
msgMap.m_lng2 = locationMap.lng2
def distance = GeoHelper.GetDistance(ret.lat2, ret.lng2, locationMap.lat2, locationMap.lng2)
msgMap.updateLngFlag = true
msgMap.distance = distance
udata.put('lat2', locationMap.lat2)
udata.put('lng2', locationMap.lng2)
}
if (ret.lat != locationMap.lat) {
@@ -171,31 +180,39 @@ class VagueAddressLocationBatch extends BaseBatch {
if (udata) {
println ret.city + " " + ret.region + " " + ret.name + " [ 修改其经纬度!]"
udata.put("m_date", new Date())
// DBUtils.seaweed.update(new BasicDBObject('_id',locationMap._id),new BasicDBObject('$set',udata))
DbUtils.seaweed.update(new BasicDBObject('_id',locationMap.id),new BasicDBObject('$set',udata))
}
results << msgMap
}
def findStandardObject(def city, def name, def rets) {
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() > 1) {
//相似里找有精确地址
for (def mapObj : rets) {
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
if (mapObj.address && findPattern(mapObj.address)) {
map = mapObj;
break;
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name.contains(name) || name.contains(mapObj.name)){
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
if (mapObj.address && findPattern(mapObj.address)) {
map = mapObj;
break;
}
}
}
}
if(!map){
//交带有交叉口地址
for (def mapObj : rets) {
if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) {
if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){
map = mapObj;
break;
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name.contains(name) || name.contains(mapObj.name)) {
if (mapObj.address && (mapObj.address.contains("交汇") || mapObj.address.contains("交叉"))) {
if (!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")) {
map = mapObj;
break;
}
}
}
}
}
@@ -211,18 +228,18 @@ class VagueAddressLocationBatch extends BaseBatch {
def findLatAndLng(ret) {
//https://restapi.amap.com/v3/place/text?keywords=
def rets = OpenApiUtils.getGaoDeSearchLocation(ret.city, ret.region, ret.name)
def map = findStandardObject(ret.city, ret.name, rets);
def map = findStandardObject(ret.city, ret.region, ret.name, rets);
def locationMap = [:]
if (!map || !map.lat) {
//http://api.map.baidu.com/place/v2/search
rets = OpenApiUtils.getBiDuLocation(ret.city, ret.region, ret.name);
map = findStandardObject(ret.city, ret.name, rets);
map = findStandardObject(ret.city, ret.region, ret.name, rets);
}
if (!map || !map.lat) {
//https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap
def code = codeMap.get(ret.city);
rets = OpenApiUtils.searchBaiDuMap(ret.name, ret.region, code.toInteger());
map = findStandardObject(ret.city, ret.name, rets);
map = findStandardObject(ret.city, ret.region, ret.name, rets);
}
// if((!map || !map.lat)){
@@ -237,7 +254,7 @@ class VagueAddressLocationBatch extends BaseBatch {
locationMap.lng2 = (map.lng2.toDouble() * 10000000000).longValue()
locationMap.id = ret._id
locationMap.city = ret.city
if (ret.address.toString().endsWith("路")) {
if (map.address) {
locationMap.address = map.address.replace("(","").replace(")","");
}
@@ -48,7 +48,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-12-01"
static def statDate = "2022-01-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2022年2月\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -12,7 +12,19 @@ class TempMatchDistrict {
static main(args) {
def task = new TempMatchDistrict()
// task.run()
task.regionConver()
// task.regionConver()
task.stat()
}
def stat(){
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
def total = 0
for(def city : citys){
def count = CommonDatautils.findAllDistrictCount("district", city)
total = total + count
println city +" "+count
}
println "共:"+total
}
def regionConver(){
@@ -29,7 +41,7 @@ class TempMatchDistrict {
}
def run(){
def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据"
def files = ["合肥.xlsx","湖北.xlsx","陕西.xlsx","上海.xlsx","深圳.xlsx","苏州.xlsx","天津.xlsx","雄安.xlsx","浙江.xlsx","重庆.xlsx"]
def files = ["杭州.xlsx"]
for(def f : files){
def filePath = path+"\\"+f;
match(filePath);
@@ -10,7 +10,7 @@ class FangjiaClientUtils {
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
JSONObject jsonObj = null
try {
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"&region="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
if(address){
url = url + "&address="+address
@@ -0,0 +1,340 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import com.fangjia.core.util.BaiDuApiUtils
import java.sql.ResultSet
class ZsNoExistsToDB {
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
def task = new ZsNoExistsToDB()
task.run()
// task.addDistrictToDB()
}
def addDistrictToDB(){
def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1)
def keys = []
def addIndex = 0
for(def d : datas){
if(d['J'] && d['J']=="新增"){
def city = d['A']
def region = d['B']
def name = d['F'].toString().replace("·","")
def address = d['G']
def dist = CommonDatautils.findDistrict(city,region,name,"district")
if(!dist){
dist = CommonDatautils.findDistrict(city,region,d['C'],"district")
}
if(!dist){
keys << city+"##"+region+"##"+name
def alias = []
def str = d['H']
if(str){
def arrs = str.split(",")
for(def s : arrs){
if(!alias.contains(s)){
alias << s
}
}
}
if(address){
address = AddressUtil.formatAddress(city, address)
}
def lng = d['M']
def lat = d['L']
def lng2 = null
def lat2 = null
long lng_new = 0
long lat_new = 0
long lng2_new = 0
long lat2_new = 0
if(lng){
lng = lng.toString().toDouble()
lat = lat.toString().toDouble()
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
lng2 = baidu.x
lat2 = baidu.y
lng_new = lng *10000000000L
lat_new = lat *10000000000L
if(lng2){
lng2_new = lng2 *10000000000L
lat2_new = lat2 *10000000000L
}
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", address)
if(lng_new){
data.put("lat",lat_new)
data.put("lng",lng_new)
data.put("lat2",lat2_new)
data.put("lng2",lng2_new)
}
if(alias){
data.put("alias",alias)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
DbUtils.seaweed.save(data)
FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12);
addIndex++
println "[新增] "+addIndex +" "+ data
}else{
println dist.city +" "+dist.region+" "+dist.name+" 小区已存在"
}
}
}
}
def run(){
def rets = findOpenLog2()
// ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
println rets.size()
// matchMap2()
}
def matchMap2() {
ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d ->
if(!d['E']){
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject(d['A'],d['B'],d['C'], rets);
def locationMap = [:]
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
map = findStandardObject(d['A'],d['B'],d['C'], rets);
}
if(map){
println map
if(map.adname){
d['E'] = map.adname
}else if(map.region){
d['E'] = map.region
}
d['F'] = map.name
d['G'] = map.address
d['K'] = map.estateType
d['L'] = map.lat
d['M'] = map.lng
d['N'] = map.lat2
d['O'] = map.lng2
d['P'] = map.source
d['Q'] = RegionData.formatRegion(d['A'], d['E'])
}
}
})
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
def matchMap(){
ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data->
def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德")
if(!gdData){
gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德")
}
if(gdData){
data['E'] = gdData.district
data['F'] = gdData.name
data['G'] = gdData.address
data['H'] = gdData.type
data['I'] = gdData.lng
data['J'] = gdData.lat
data['K'] = gdData.score
}
def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度")
if(!bdData){
bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度")
}
if(bdData){
data['L'] = bdData.district
data['M'] = bdData.name
data['N'] = bdData.address
data['O'] = bdData.type
data['P'] = bdData.lng
data['Q'] = bdData.lat
data['R'] = bdData.score
}
},0)
}
def findOpenLog2(){
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18")
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate)
def rets = []
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def index = 0
def count = 0
def keys = []
for(def d : datas){
if(d.code == 400){
index++
def msg = d.msg
if(msg.contains("该楼盘不存在")){
def arrs = msg.split(" ")
if(arrs){
def rMap = [:]
for(def s : arrs){
if(s.contains(":")){
def ars = s.split(":")
if(ars.size()==2){
rMap.put(ars[0], ars[1])
}else{
rMap.put(ars[0], null)
}
}
}
if(rMap){
count++
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
if(!keys.contains(key)){
keys << key
def params = d.parameter_map
def address = null
if(params && params.address){
address = params.address
}
def r = [rMap.city, rMap.region, rMap.name, address]
if(!rets.contains(r)){
rets << r
}
}
}
}
}else{
println msg
}
}
}
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
return rets;
}
def findOpenLog(){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1")
sqlBuffer.append(" and user_id = '1007569'")
sqlBuffer.append(" and request_uri = '/czbank/evaluation'")
sqlBuffer.append(" and c_date >= '2022-01-01'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def rets = []
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
def index = 0
def count = 0
while (rs.next()){
def code = rs.getInt("code")
if(code == 400){
index++
def msg = rs.getString("msg")
if(msg && msg.contains("该楼盘不存在")){
def arrs = msg.split(" ")
if(arrs){
def rMap = [:]
for(def s : arrs){
if(s.contains(":")){
def ars = s.split(":")
if(ars.size()==2){
rMap.put(ars[0], ars[1])
}else{
rMap.put(ars[0], null)
}
}
}
if(rMap){
count++
def params = rs.getObject("parameter_map")
println params
def r = [rMap.city, rMap.region, rMap.name]
if(!rets.contains(r)){
rets << r
}
}
}
}else{
println msg
}
}
}
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
return rets;
}
}
@@ -1,11 +1,20 @@
package com.fangjia.script.gj.location
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.codehaus.jackson.sym.NameN
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import com.fangjia.script.gj.district.FangjiaClientUtils
class DistrictLocationUpdate {
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
@@ -19,20 +28,214 @@ class DistrictLocationUpdate {
static main(args) {
def task = new DistrictLocationUpdate()
task.run()
// task.run()
task.mapMatch()
// task.updateLocationToDB()
}
def run(){
def citys = CommonDatautils.findAllCity()
def citys = ["福州"]//CommonDatautils.findAllCity()
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
updateLocation(city, dists);
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"])
def rets = []
rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"]
for(def d : dists){
def lng = null
def lat = null
if(d.lng){
lng = d.lng/10000000000.0
lat = d.lat/10000000000.0
}
rets << [d.city, d.region, d.name, d.address, lng,lat]
}
ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据")
}
}
def mapMatch(){
def index = 0
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
if(!d['H'] && d['C']=="华侨城龙居御景二期"){
index++
// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德")
// println d['A']+" "+d['B']+" "+d['C']
// if(gdData){
// d['G'] = gdData.district
// d['H'] = gdData.name
// d['I'] = gdData.address
// d['J'] = gdData.type
// d['K'] = gdData.lng
// d['L'] = gdData.lat
// d['M'] = gdData.score
// if(d['E'] && d['K']){
// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
// d['N'] = distance
// }
// }
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
if (!map || !map.lat) {
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
}
if(map){
if(map.adname){
d['G'] = map.adname
}else if(map.region){
d['G'] = map.region
}
d['H'] = map.name
d['I'] = map.address
if(map.type){
d['J'] = map.type
}else if(map.estateType){
d['J'] = map.estateType
}
d['K'] = map.lng
d['L'] = map.lat
d['M'] = map.score
if(d['E'] && d['K']){
def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
d['N'] = distance
}
}
}
},0)
}
def updateLocationToDB(){
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
if(d['N'] && d['N']>1000){
if(d['M']>=0.4 && d['K']){
def query = new BasicDBObject()
query.put("city", d['A'])
query.put("region", d['B'])
query.put("name", d['C'])
def udata = new BasicDBObject()
udata.put("m_date", new Date())
Long lng0 = d['K'].toString().toDouble()*10000000000L
Long lat0 = d['L'].toString().toDouble()*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
def gmap = GeocodeUtils.getGaodeToBaidu(d['K'].toString().toDouble(), d['L'].toString().toDouble())
Long lng2 = gmap.get("lng")*10000000000L
Long lat2 = gmap.get("lat")*10000000000L
udata.put("lat2", lat2)
udata.put("lng2", lng2)
DbUtils.seaweed.update(query, new BasicDBObject('$set', udata))
}
}
},0)
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
def findStandardObject(def city, def region, def name, def rets) {
def map;
if (rets) {
if (rets.size() >= 1) {
for (def mapObj : rets) {
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
continue;
}
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
map = mapObj;
break;
}else if(mapObj.address && mapObj.address.contains(name)){
map = mapObj;
break;
}else if(mapObj.name && name){
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
if(score1){
map = mapObj;
break;
}
}
// }
}
}
}
return map;
}
def queryPoiMap(def city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
@@ -40,9 +40,9 @@ class LocationTest {
// System.out.println(str);
def task = new LocationTest()
// task.run()
task.run()
// task.run2()
task.run3()
// task.run3()
// task.run4()
// task.run5()
}
@@ -0,0 +1,383 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.gj.paimai.ChromeDriverPool
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
static main(args) {
def task = new BeiKeCrawlByChrome();
task.run()
// task.run1()
// task.addDealRegion();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
getLogin(driver)
crawlData(driver, cityMap)
}
def addDealRegion(){
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
for(def d : dbcursor){
keys << ["city":d.city, "name":d.district_name]
}
for(def m : keys){
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
if(list && list.size()==1){
def dist = list[0]
def query = new BasicDBObject()
query.put("city", m.city)
query.put("region", null)
query.put("district_name", m.name)
// query.put("source", "链家在线")
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
def data = new BasicDBObject()
data.put("region", dist.region)
def ds = DEAL_PRICE_COLL.count(query)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
}
def crawlData(driver, cityMap){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
def getPageUrls(driver, cityUrl){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "chengjiao/pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
stop = true;
break;
}
}
page++;
}else{
stop = true;
}
}
return detailUrls
}
def getPageDeail(driver, city, cityUrl){
def stop = false
def page = 1
def detailUrls = []
while(!stop){
def url = cityUrl + "/chengjiao/pg${page}/"
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def elements = JQueryUtils.select(".listContent li", doc)
if(elements){
for(def e : elements){
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
}else{
stop = true;
break;
}
}
page++;
}else{
stop = true;
}
}
return detailUrls
}
def parse(city,e){
try {
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
def arrs = title.split(" ")
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
if (roomStr.indexOf("室") > 0) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
def price = e.select(".totalPrice .number").text()
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = e.select(".dealDate").text()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "贝壳找房"
def dbo = findTrade(detail)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
detail.direction = houseInfos[0]
detail.fitment = houseInfos[1]
def floorStr = e.select(".positionInfo").text()
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.url = e.select(".title a").attr("href")
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
}
}
}catch(Exception ex){
ex.printStackTrace()
println e.select(".title a").attr("href")
}
}
def findRegionByThirdPary(city, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("name", district_name)
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
def dbo = null;
for(def d : dbcursor){
dbo = d
}
if(dbo){
return dbo.region
}
return null;
}
def getContent(driver, detailUrl){
driver.get(detailUrl)
def str = driver.getPageSource()
if(str && str.contains("<title>登录</title>")){
getLogin(driver)
return getContent(driver, detailUrl)
}
return str;
}
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
if(detailUrls){
for(def i=0;i<detailUrls.size();i++){
def detailUrl = detailUrls[i]
try{
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
} catch (Exception e) {
e.printStackTrace()
}
}
}
}
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
def content = getContent(driver, detailUrl)
def doc = Jsoup.parse(content);
def eles = JQueryUtils.select(".deal-bread a", doc)
def detail = new BasicDBObject()
if(eles && eles.size()>=4){
detail.city = eles[0].text().replace("房产网","")
detail.region = eles[2].text().replace("二手房成交","")
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
}
if(detail.district_name) {
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
if (price && price.isNumber()) {
detail.total_price = price.toInteger() * 10000
}
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
if (areaStr) {
detail.area = areaStr.toDouble()
}
if (detail.total_price && detail.area) {
detail.avg_price = (int) (detail.total_price / detail.area)
}
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
if (tradeStr) {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "链家在线"
def dbo = findTrade(DEAL_PRICE_COLL, detail)
if (!dbo) {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
if (floorStr && floorStr.isNumber()) {
detail.height = floorStr.toInteger()
}
}
detail.url = detailUrl
detail.c_date = new Date()
detail.m_date = new Date()
DEAL_PRICE_COLL.save(detail)
}
}
return detail;
}
def findTrade(def detail){
def q = new BasicDBObject()
q.put("city", detail.city)
q.put("region", detail.region)
q.put("district_name", detail.district_name)
q.put("area", detail.area)
q.put("total_price", detail.total_price)
q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q)
return dbo
}
def getAllCityUrls(driver){
driver.get("https://www.ke.com/city/")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
// if(e.text().contains("天津")){
cityMap.put(e.text(), "https:"+href)
// }
}
}
return cityMap
}
def getLogin(driver){
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
WebDriverWait wait = new WebDriverWait(driver,20)
Thread.sleep(10000)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
@@ -33,9 +34,10 @@ public class LianjiaCrawlByChrome {
static main(args) {
def task = new LianjiaCrawlByChrome();
task.run()
// task.run1()
task.addDealRegion();
// task.run()
// task.run1()
// task.addDealRegion();
task.standardRegion()
}
def run1(){
@@ -94,22 +96,44 @@ public class LianjiaCrawlByChrome {
}
def run(){
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
// System.setProperty("http.nonProxyHosts", "localhost");
// WebDriver driver = new ChromeDriver()
// def cityMap = getAllCityUrls(driver)
// getLogin(driver)
// crawlData(driver, cityMap)
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
getLogin(driver)
crawlData(driver, cityMap)
}
def standardRegion(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
def dbcursor = DEAL_PRICE_COLL.find(q)
for(def d : dbcursor){
if(d.region){
def newRegion = RegionData.formatRegion(d.city, d.region)
if(newRegion && newRegion!=d.region){
def query = new BasicDBObject()
query.put("city", d.city)
query.put("region", d.region)
def data = new BasicDBObject()
data.put("region", newRegion)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
}
}
}
def addDealRegion(){
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
lastMonth = lastMonth +"-01"
def q = new BasicDBObject()
q.put("region", null)
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
// q.put("source", new BasicDBObject('$in', ["Á´¼ÒÔÚÏß","±´¿ÇÕÒ·¿"]))
def dbcursor = DEAL_PRICE_COLL.find(q)
def keys = []
@@ -131,6 +155,7 @@ public class LianjiaCrawlByChrome {
def data = new BasicDBObject()
data.put("region", dist.region)
println m.city+ " "+m.name +" "+dist.region
def ds = DEAL_PRICE_COLL.count(query)
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
}
@@ -193,7 +218,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -219,6 +244,9 @@ public class LianjiaCrawlByChrome {
detail.city = city
detail.district_name = arrs[0]
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
def roomStr = arrs[1]
if (roomStr) {
@@ -391,9 +419,9 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(!href.contains("fang.lianjia")){
// if(!href.contains("fang.lianjia") && e.text().contains("¸£ÖÝ")){
cityMap.put(e.text(), href)
}
// }
}
}
return cityMap
@@ -37,7 +37,10 @@ class CrawlDistrictIdSelect {
}
def run(){
def datas = findCrawlData()
def source = "58同城"
def root = "58.com"
def shortCityMap = findCitySeeds(root)
def datas = findCrawlData(source, shortCityMap, root)
def list = []
datas.each {k,v->
if(v){
@@ -58,15 +61,32 @@ class CrawlDistrictIdSelect {
}
}
def findCrawlData(){
def findCitySeeds(root){
def q = new BasicDBObject()
q.put("root", root)
q.put("city", "福州")
def dbcursor = districtseedsColl.find(q)
def cityMap = [:]
for(def d : dbcursor){
if(!cityMap.keySet().contains(d.city)){
cityMap.put(d.city, d.city_short)
}
}
return cityMap
}
def findCrawlData(def source, shortCityMap, def root){
def q = new BasicDBObject()
q.put("city","¸£ÖÝ")
q.put("source", "³ÇÊз¿²ú")
q.put("s_date",202012)
q.put("source", source)
q.put("s_date",202112)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = ".cityhouse.cn/ha/"
def prefix = "xiaoqu/"
def keyList =new HashSet()
def index = 0
for(def db : dbcursor){
def url = null
if(!db.url){
@@ -76,33 +96,42 @@ class CrawlDistrictIdSelect {
}else{
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
if(url){
def district_id = ""
if(url.contains(prefix)){
district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1)
}else{
district_id = url;
}
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
def ids = retMap.get(db.city)
if(!ids){
ids = []
retMap.put(key, ids)
retMap.put(db.city, ids)
}
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "cityhouse.com"
map.city_short= shortCity
map.city = db.city
map.region = db.region
map.district_id = district_id
map.source_url = url
ids << map
keyList << key
if(!keyList.contains(db.city+"#"+district_id)){
def count = findIsExists(db.city, district_id, root)
if(!count){
def map = new BasicDBObject()
map.root = "58.com"
map.city_short= shortCityMap.get(db.city)
map.city = db.city
map.region = db.region
map.district_id = district_id
map.source_url = url
map.name = db.name
ids << map
}else{
index++
println "[${index}]已存在:"+db.city +" "+db.name
}
keyList << db.city+"#"+district_id
}else{
println "ÖØ¸´£º"+key
println "重复:"+db.city+"#"+db.name
}
}else{
println url +" Á´½Ó²»ºÏÒªÇó"
@@ -110,4 +139,13 @@ class CrawlDistrictIdSelect {
}
return retMap
}
def findIsExists(city, district_id, root){
def q = new BasicDBObject()
q.put("city",city)
q.put("district_id", district_id)
q.put("root",root)
def count = districtseedsColl.count(q)
return count
}
}
@@ -68,7 +68,7 @@ class FZloationUpdate {
Long lng0 = lng*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[ÐÞ¸Ä-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
}
@@ -153,7 +153,7 @@ class FZloationUpdate {
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(dist.city, addr);
if(reps && reps.count.toInteger()>=1){
if(reps && reps.count && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
@@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch {
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "G00jia123456");
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
email.addTo(mail.receiverEmail);
email.setFrom("service@fangjia.com");//·¢ËÍ·½
email.setSubject(mail.subject); //±êÌâ
@@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch {
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "G00jia123456");
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
email.addTo(mail.receiverEmail);
email.setFrom("service@fangjia.com");
email.setSubject(mail.subject);
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "58ͬ³Ç"//
def citys = findCrawlCity(source, 202111);
def source = "城市房产"//
def citys = findCrawlCity(source, 202112);
for(def city : citys){
def count1 = findCount(source, city, 202111);
def count2 = findCount(source, city, 202112);
def count1 = findCount(source, city, 202112);
def count2 = findCount(source, city, 202201);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
File diff suppressed because one or more lines are too long
@@ -315,11 +315,11 @@ class AddMetadataBatchQfang{
private static void saveThirdPartyPrice(def saveDate){
def q = new BasicDBObject()
q.put("url", saveDate.get("xqUrl"))
q.put("s_date", SHORT_DATE);
q.put("city", saveDate.get("city"));
q.put("region", saveDate.get("region"));
q.put("name", saveDate.get("xqName"));
q.put("source", saveDate.get("source"));
def udata = THIRD_PARTY_PRICE_COLL.findOne(q)
if(!udata){
@@ -336,6 +336,14 @@ class AddMetadataBatchQfang{
obj.put("source", saveDate.get("source"));
obj.put("c_date", new Date());
THIRD_PARTY_PRICE_COLL.save(obj);
println "[save] "+saveDate.toString()
}else{
def updata = new BasicDBObject()
updata.put("avg_price", saveDate.get("price"))
updata.put("url", saveDate.get("xqUrl"));
updata.put("m_date", new Date());
THIRD_PARTY_PRICE_COLL.update(q,new BasicDBObject('$set',updata))
println "[update] "+saveDate.toString()
}
}
@@ -87,7 +87,7 @@ class AddSeedBatchQfang extends BaseBatch{
href = "https:"+href;
}
if(href&&!CITYS.contains(city)){
ret.put(city, href);
ret.put(city, href);
}
}
}catch(Exception e){
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
//"fang.com":"搜房",
def sourceMap = ["fang.com":"搜房"]
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -94,11 +94,16 @@ class SynchDistrictPrice {
for(def r : rets){
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
if(obj){
updateIndex++
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
addIndex++
thirdPartyColl.save(r)
}else{
thirdPartyColl.save(r)
try{
addIndex++
thirdPartyColl.save(r)
}catch(Exception e){
e.printStackTrace()
}
}
// if(!obj){
// addIndex++
@@ -112,7 +117,7 @@ class SynchDistrictPrice {
// }
// }
}
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 已存在:"+updateIndex
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
def count1 = findTotalCount(source, 202111);
def count2 = findTotalCount(source, 202112);
def count1 = findTotalCount(source, 202112);
def count2 = findTotalCount(source, 202201);
println source +" "+count1 +" "+count2
}
}
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20211201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220101", "yyyyMMdd")
def startDate = DateUtil.parseDate("20220201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20220301", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0