Merge remote-tracking branch 'origin/script_gj' into script_gj
This commit is contained in:
@@ -13,7 +13,7 @@ import java.util.*;
|
||||
*/
|
||||
public class BaiDuApiUtils {
|
||||
|
||||
private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5";
|
||||
private static String AK = "ajgWIBd9rYsXed1hcDQqYvkX";
|
||||
public static String[] AKS = new String[]{"hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg",
|
||||
"4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784",
|
||||
"9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224",
|
||||
|
||||
@@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils {
|
||||
|
||||
public static void main(String[] args) {
|
||||
for (int i = 0; i < 1000; i++) {
|
||||
System.out.println(random(0, 10));
|
||||
System.out.println(random(-9, 9));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -54,10 +54,10 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
def citys = CcbCommonUtils.citys;
|
||||
|
||||
for(def city : citys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
println citys.unique().size()
|
||||
// for(def city : citys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// println citys.unique().size()
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
+1
-1
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220118, 20220219);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
|
||||
+27
-4
@@ -149,8 +149,8 @@ class CcbCommonUtils {
|
||||
|
||||
public static def findTradesDetailById(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
// q.put("city", dist.city)
|
||||
// q.put("region", dist.region)
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
@@ -161,7 +161,7 @@ class CcbCommonUtils {
|
||||
def lists = []
|
||||
println dist.name +" "+total
|
||||
for(def d : dbcursor){
|
||||
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
if(!rets.contains(key)){
|
||||
rets << key
|
||||
d.dist_id = dist._id.toString()
|
||||
@@ -367,7 +367,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","长隆小区")
|
||||
// q.put("name","特房锦绣一方")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
@@ -488,4 +488,27 @@ class CcbCommonUtils {
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
public static def findDistrictSeqno(){
|
||||
def sql = "select cityid,commid,seqno from bigdata_position_seqno";
|
||||
def map = new HashMap()
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def detailMap = map.get(r.cityid)
|
||||
if(!detailMap){
|
||||
detailMap = [:]
|
||||
map.put(r.cityid, detailMap)
|
||||
}
|
||||
detailMap.put(r.commid, r.seqno)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
public static def findMaxSeqno(def cityid){
|
||||
def sql = "select max(seqno) as max_seqno from bigdata_position_seqno where cityid='${cityid}'"
|
||||
def maxSeqno = null;
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
maxSeqno = r.max_seqno
|
||||
});
|
||||
return maxSeqno
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jiangnanblank.SFTP
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class CcbFtpTestUtils {
|
||||
|
||||
static def ftpServer = "192.168.0.86"
|
||||
static def ftpPort = 17495
|
||||
static def ftpUser = "ccb"
|
||||
static def ftpPassword = "ccb!@#"
|
||||
static def ftpPath = "/home/ccb/output/"
|
||||
|
||||
static def ftpPath_rh = "/home/ccb_rh/output/"
|
||||
|
||||
static SFTP sftp = new SFTP();
|
||||
static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
|
||||
|
||||
static def ftpUpload(type,outputPath_dist, month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
if(type && type=="rh"){
|
||||
sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel);
|
||||
}else{
|
||||
sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
static def reConnect(){
|
||||
if(sftpChannel == null){
|
||||
println "sftpChannel 为空,需重连....."
|
||||
if(sftp == null){
|
||||
sftp = new SFTP();
|
||||
}
|
||||
sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword);
|
||||
}else{
|
||||
println "sftpChannel 正常....."
|
||||
}
|
||||
return sftpChannel;
|
||||
}
|
||||
|
||||
static def uploadSha512Data(def sb, def dataCodeFileName, def charset){
|
||||
String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){
|
||||
|
||||
def data = new FileInputStream(new File(dataFileName))
|
||||
String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码
|
||||
DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
+178
-2
@@ -1,15 +1,20 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
@@ -37,15 +42,74 @@ class DistrictDelete {
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
// task.run()
|
||||
// task.gzRun();
|
||||
// task.deleteDistrict()
|
||||
task.deleteCommPrice()
|
||||
// task.deleteCommPrice()
|
||||
// task.distHouseCountStat()
|
||||
task.addThirdPartyStat()
|
||||
// task.processRegion()
|
||||
}
|
||||
def processRegion(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d->
|
||||
if(d['B']){
|
||||
def region = RegionData.formatRegion("重庆",d['B'])
|
||||
if(region){
|
||||
d['B'] = region
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addThirdPartyStat(){
|
||||
def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1)
|
||||
def idMaps = [:]
|
||||
for(def d : datas){
|
||||
def id = d['A']
|
||||
idMaps.put(id, d['D'])
|
||||
}
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
|
||||
def totalCount = new AtomicLong()
|
||||
def map = findThirdPartyPrice("上海")
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def thirdLists = Collections.synchronizedList(new ArrayList());
|
||||
thirdLists << ["ID","区域","名称","总量"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def d = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def count = idMaps.get(d._id.toString())
|
||||
if (!count) {
|
||||
def dist = CommonDatautils.findDistrictById(d._id.toString())
|
||||
def priceMap = map.get(dist.region + "##" + dist.name)
|
||||
|
||||
if (priceMap) {
|
||||
thirdLists << [dist._id.toString(), dist.region, dist.name, priceMap.size().toString()]
|
||||
println dist._id.toString()+"@"+ dist.region+"@"+ dist.name+"@"+ priceMap.size()
|
||||
}
|
||||
}
|
||||
// println totalCount.get() + " " + d.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\上海第三方.xlsx",thirdLists, "房源")
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -60,6 +124,40 @@ class DistrictDelete {
|
||||
}
|
||||
}
|
||||
|
||||
def distHouseCountStat(){
|
||||
def city = "重庆"
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address"])
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def distCount = districts.size()
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
rets << ["ID","区域","名称","总量"]
|
||||
for(def i=0;i<districts.size();i++) {
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def count = findHouseCount(city, dist.region, dist.name, dist.alias)
|
||||
def count = findHouses(city, dist.region, "sell", 20210701,20210801, dist.name, dist.alias)
|
||||
println totalCount.get() +" "+city +" "+dist.region+" "+dist.name+" "+count
|
||||
if(count){
|
||||
rets << [dist._id.toString(), dist.region, dist.name, count]
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\重庆农商行\\重庆7月份的房源统计.xlsx",rets, "房源")
|
||||
|
||||
}
|
||||
|
||||
def checkDist(city,dists){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
@@ -259,4 +357,82 @@ class DistrictDelete {
|
||||
sqlInstance.execute(sql)
|
||||
println "删除:"+tableName+" "+commid
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202112).append('$lte',202112))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.region+"##"+db.name
|
||||
def prices = map.get(key)
|
||||
if(!prices){
|
||||
prices = []
|
||||
map.put(key, prices)
|
||||
}
|
||||
if( db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
prices << db.zhugezhaofang_esf_price
|
||||
}
|
||||
if(db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
prices << db.anjuke_esf_price
|
||||
}
|
||||
if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
prices << db.get("58tc_esf_price")
|
||||
}
|
||||
if(db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
prices << db.soufang_esf_price
|
||||
}
|
||||
if(db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
prices << db.lianjia_esf_price
|
||||
}
|
||||
if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
prices << db.cityhouse_esf_price
|
||||
}
|
||||
|
||||
if(db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
prices << db.maitian_esf_price
|
||||
}
|
||||
|
||||
if(db.beike_esf_price && db.beike_esf_price>3000){
|
||||
prices << db.beike_esf_price
|
||||
}
|
||||
|
||||
if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
prices << db.get("365house_esf_price")
|
||||
}
|
||||
|
||||
if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
prices << db.zhongyuan_esf_price
|
||||
}
|
||||
|
||||
if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
prices << db.tuboshi_esf_price
|
||||
}
|
||||
|
||||
if( db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
prices << db.qfang_esf_price
|
||||
}
|
||||
|
||||
if(db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
prices << db.fangtan_esf_price
|
||||
}
|
||||
|
||||
if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
prices << db.fangduoduo_esf_price
|
||||
}
|
||||
|
||||
if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
prices << db.get("5i5j_esf_price")
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
}
|
||||
|
||||
return map
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1")
|
||||
def prices = CcbCommonUtils.findGuidePrice(city, "2022-02-1", "2022-03-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+20
-21
@@ -21,26 +21,25 @@ class ExportOracleData {
|
||||
}
|
||||
|
||||
def run() {
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
// //不会变
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()
|
||||
// exportBigdataDistrictData()
|
||||
//
|
||||
// exportGuidedPrice()//随人工改变
|
||||
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
//
|
||||
// //处理小区情况
|
||||
// exportBigdataCommunity()
|
||||
// exportCommunityStaging()
|
||||
// exportCommunityBuileType()
|
||||
//不会变
|
||||
exportEvaluationRateData()
|
||||
exportBigdataPoiData()
|
||||
exportBigdataDistrictData()
|
||||
|
||||
// exportThirdPartyPrice2(lastMonth)
|
||||
// exportBigdataForsale("2021-12-19")
|
||||
// exportBigdataDeal("2021-12-01")
|
||||
exportGuidedPrice()//随人工改变
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
|
||||
// //处理小区情况
|
||||
exportBigdataCommunity()
|
||||
exportCommunityStaging()
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-01-19")
|
||||
exportBigdataDeal("2022-01-01")
|
||||
|
||||
|
||||
//计算结果相关
|
||||
@@ -56,11 +55,11 @@ class ExportOracleData {
|
||||
// exportCommunityPdescribeGj(curMonth)
|
||||
// exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// def curDate = curMonth.toString()+"28"
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
//ok文件
|
||||
exportLastFile()
|
||||
// exportLastFile()
|
||||
}
|
||||
|
||||
|
||||
|
||||
+416
@@ -0,0 +1,416 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
|
||||
|
||||
class ExportOracleEvaluationData {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportOracleEvaluationData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run() {
|
||||
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
|
||||
//计算结果相关
|
||||
exportAvmEvaluationGJNB(curMonth)
|
||||
exportAVMEvaluationItemGJNB(curMonth)
|
||||
|
||||
exportCityPrice(curMonth)
|
||||
exportRegionPrice(curMonth)
|
||||
|
||||
exportCommunityBasePriceGj(curMonth)//已合下面的合并
|
||||
exportCommunityBasePrice(curMonth)
|
||||
|
||||
exportCommunityPdescribeGj(curMonth)
|
||||
exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// ok文件
|
||||
exportOkFile()
|
||||
}
|
||||
|
||||
def exportAvmEvaluationGJNB(yearMonth){
|
||||
def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" +
|
||||
"ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" +
|
||||
"DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.COMMUNITY_NUM
|
||||
data << r.HAS_PRICE_COMMUNITY_NUM
|
||||
data << r.PRICE_COVERAGE
|
||||
data << r.ARTIFICIAL_CALC_NUM
|
||||
data << r.ARTIFICIAL_CALC_COVERAGE
|
||||
data << r.SAMPLE_CALC_NUM
|
||||
data << r.SAMPLE_CALC_COVERAGE
|
||||
data << r.ANN_CALC_NUM
|
||||
data << r.ANN_CALC_COVERAGE
|
||||
data << r.TEST_NUM
|
||||
data << r.TEST_COMM_NUM
|
||||
data << r.TEST_COMM_COVERAGE
|
||||
data << r.PPE15_COVERAGE
|
||||
data << r.RATE_MEDIAN
|
||||
data << r.COEFFICIENT
|
||||
data << r.DEAL_TEST_CASE_NUM
|
||||
data << r.DEAL_TEST_COMM_NUM
|
||||
data << r.DEAL_TEST_COMM_COVERAGE
|
||||
data << r.DEAL_PPE15_COVERAGE
|
||||
data << r.DEAL_COEFFICIENT
|
||||
data << r.DEAL_RATE_MEDIAN
|
||||
data << r.THIRD_TEST_CASE_NUM
|
||||
data << r.THIRD_TEST_COMM_NUM
|
||||
data << r.THIRD_TEST_COMM_COVERAGE
|
||||
data << r.THIRD_PPE15_COVERAGE
|
||||
data << r.THIRD_COEFFICIENT
|
||||
data << r.THIRD_RATE_MEDIAN
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE",
|
||||
//"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE",
|
||||
//"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "AVM_EVALUATION_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "AVM_EVALUATION_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportAVMEvaluationItemGJNB(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" +
|
||||
"to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" +
|
||||
"to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.CONFIDENCE
|
||||
data << r.TARGET_AVG_PRICE
|
||||
data << r.DEALDATE
|
||||
data << r.UNIT_PRICE
|
||||
data << r.BUILDAREA
|
||||
data << r.FITMENT
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORCNT
|
||||
data << r.DATA_TYPE
|
||||
data << r.RATE
|
||||
data << r.PE
|
||||
data << r.CREATETIME
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePrice(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.FAKE_RATE
|
||||
data << r.DIFF_RATE
|
||||
data << r.CONFI_RATE
|
||||
data << r.CORR_RATE
|
||||
data << r.CONFIDENCE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
CcbFtpUtils.ftpUpload(null, fileName1, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityBasePriceGj(yearMonth){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.AVG_PRICE
|
||||
data << r.CALC_TYPE
|
||||
data << r.FAKE_RATE
|
||||
data << r.DIFF_RATE
|
||||
data << r.CONFI_RATE
|
||||
data << r.CORR_RATE
|
||||
data << r.CONFIDENCE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCityPrice(month){
|
||||
def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "CITY_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "CITY_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportRegionPrice(month){
|
||||
def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.YEAR_MONTH
|
||||
data << r.CALC_AVG_PRICE
|
||||
data << r.PRICE
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "DISTRICT_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "DISTRICT_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommunityPdescribeGj(month){
|
||||
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
//
|
||||
// for(def city : citys) {
|
||||
// def cityCode = cityCodeDataMap.get(city)
|
||||
def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}";
|
||||
// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.COMMUNITY_ID
|
||||
data << r.CITYID
|
||||
data << r.CITY_NAME
|
||||
data << r.DISTRICT_CODE
|
||||
data << r.DISTRICT_NAME
|
||||
data << r.COMMUNITY_NAME
|
||||
data << r.PQ_NAME
|
||||
data << r.BASE_PRICE
|
||||
data << r.CONFIDENCE_LEVEL
|
||||
data << r.CNT
|
||||
data << r.MIN_PRICE
|
||||
data << r.MEDIANPRICE
|
||||
data << r.MEANPRICE
|
||||
data << r.MAX_PRICE
|
||||
data << r.TIME
|
||||
data << r.FIRSTDATAT
|
||||
data << r.THIRDDATA
|
||||
data << r.RELATIVEERROR
|
||||
data << r.SKEWNESS
|
||||
data << r.PQ_PRICE_RANGE
|
||||
data << r.DISTRICT_PRICE_RANGE
|
||||
data << r.DESCRIPTION
|
||||
data << r.PRIV_BASE_PRICE
|
||||
data << r.IS_ARTTFICIAL
|
||||
data << r.PRIV_FINAL_PRICE
|
||||
data << r.CREATE_DATE
|
||||
data << r.PRICE
|
||||
data << r.PRICE_SOURCE
|
||||
data << r.CASE_MD_PRICE
|
||||
data << r.CASE_AVG_PRICE
|
||||
data << r.INDEX_LABEL
|
||||
data << r.MAIN_PROPERTY
|
||||
data << r.COMMTYPE
|
||||
data << r.NEW_DEAL_PRICE
|
||||
data << r.NEW_DEAL_DATE
|
||||
data << r.PRICE_MOM_RATIO
|
||||
data << r.MAX_MOM_RATIO
|
||||
data << r.PRICE_YOY_RATIO
|
||||
data << r.SATUR_RATIO
|
||||
data << r.NEIGHBOR_PRICE
|
||||
allRows << data
|
||||
});
|
||||
println allRows.size()
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
// }
|
||||
println "community_p_describe_gj 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "community_p_describe_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportCommCaseDetailGj(month){
|
||||
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
|
||||
for(def city : citys){
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.CITYID
|
||||
data << r.COMMUNITY_ID
|
||||
data << r.NETLISTING_ID
|
||||
data << r.PRICE_DATE
|
||||
data << r.CREATE_DATE
|
||||
allRows << data
|
||||
if(allRows.size()%200000==0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
allRows.clear()
|
||||
}
|
||||
});
|
||||
if(allRows.size()>0){
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
|
||||
}
|
||||
}
|
||||
|
||||
println "comm_case_detail_gj 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "comm_case_detail_gj 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOkFile(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def dataCodeFileName = "/ccb/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-02-1", "2022-03-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+7
-6
@@ -23,9 +23,10 @@ class LoadGuide {
|
||||
// saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
|
||||
|
||||
}
|
||||
def match(def filename){
|
||||
@@ -48,15 +49,15 @@ class LoadGuide {
|
||||
def datas = ExcelUtils.read(filePath,0, 1)
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
if(d['D']){
|
||||
def o = new BasicDBObject()
|
||||
o.city = d['E']
|
||||
o.city = d['D']
|
||||
o.region = d['A']
|
||||
o.name = d['B']
|
||||
o.avg_price = d['C'].toString().toDouble().toInteger()
|
||||
o.match_city = d['E']
|
||||
o.match_region = d['F']
|
||||
o.match_name = d['G']
|
||||
o.match_city = d['D']
|
||||
o.match_region = d['E']
|
||||
o.match_name = d['F']
|
||||
o.source_type = 1 //1:成交参考价 2:参考价
|
||||
o.c_date = new Date()
|
||||
rets << o
|
||||
|
||||
+44
-22
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
}
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01")
|
||||
def testMap = findCaluateCityPPE(sqlInstance236, 202201)
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['C']){
|
||||
def oldcount = dealMap.get(d['C'])
|
||||
d['T'] = oldcount
|
||||
def casecount = testMap.get(d['C'])
|
||||
d['U'] = casecount
|
||||
println d
|
||||
}
|
||||
},0)
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
|
||||
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
|
||||
//ExcelUtils.modify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// def oldcount = dealMap.get(d['C'])
|
||||
// d['V'] = oldcount
|
||||
// def casecount = testMap.get(d['C'])
|
||||
// d['W'] = casecount
|
||||
// println d
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
|
||||
// if(d['I'] && d['K']){
|
||||
@@ -271,19 +271,24 @@ ExcelUtils.modify(fileName,{d->
|
||||
//},0)
|
||||
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d->
|
||||
// def flag = d['M']
|
||||
// if(flag==0.0){
|
||||
// if(d['D']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", new ObjectId(d['A']))
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// index++
|
||||
// println index
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", new ObjectId(d['D']))
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// index++
|
||||
// println index
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def region = d['B']
|
||||
@@ -777,6 +782,23 @@ def getStreetInfo(def city, def glocation){
|
||||
//}
|
||||
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据")
|
||||
|
||||
def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1)
|
||||
def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1)
|
||||
|
||||
def keys = []
|
||||
for(def d : datas2){
|
||||
keys << d['A']+"##"+d['C']+"##"+d['B']
|
||||
}
|
||||
|
||||
def index = 0
|
||||
for(def d : datas1){
|
||||
def k = d['A']+"##"+d['C']+"##"+d['B']
|
||||
if(!keys.contains(k)){
|
||||
index ++
|
||||
println index +" "+k
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202110
|
||||
def endDate = 202112
|
||||
def sDate= 202201//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202111
|
||||
def endDate = 202201
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+32
-10
@@ -9,6 +9,7 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
@@ -40,7 +41,7 @@ class TradeStatLog {
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -48,6 +49,7 @@ class TradeStatLog {
|
||||
// task.deleteDealData()
|
||||
// task.deleteDealData2()
|
||||
// task.statTableCount()
|
||||
task.filterTrade()
|
||||
|
||||
|
||||
}
|
||||
@@ -73,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","11月总量","12月总量","1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-10-01"
|
||||
def date = "2021-11-01"
|
||||
def endDate = "2021-12-01"
|
||||
def newDate = "2022-01-01"
|
||||
def startDate = "2021-11-01"
|
||||
def date = "2021-12-01"
|
||||
def endDate = "2022-01-01"
|
||||
def newDate = "2022-02-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -129,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-01成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
@@ -479,8 +481,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2021-12-01"
|
||||
def endDate = "2022-01-01"
|
||||
def startDate = "2022-01-01"
|
||||
def endDate = "2022-02-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
@@ -561,7 +563,7 @@ class TradeStatLog {
|
||||
}
|
||||
println city +" "+rets.size()
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
@@ -706,4 +708,24 @@ class TradeStatLog {
|
||||
|
||||
}
|
||||
|
||||
def filterTrade(){
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx",{d->
|
||||
if(!d['J']){
|
||||
def count = findTradeById(d['A'])
|
||||
if(count){
|
||||
d['J'] = "其它原因过滤"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findTradeById(id){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(id))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+82
-1
@@ -12,9 +12,90 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
class DistrictMarge {
|
||||
static main(args) {
|
||||
def task = new DistrictMarge()
|
||||
task.run()
|
||||
//task.run()
|
||||
task.repeatAddress()
|
||||
}
|
||||
|
||||
def repeatAddress(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "lng","address","name_cat"])
|
||||
findRepead(c, districts)
|
||||
}
|
||||
}
|
||||
def findRepead(city, districts){
|
||||
def map = [:]
|
||||
for(def d : districts){
|
||||
if(d.name_cat == 0 && d.lng){
|
||||
if(d.address && d.address.contains("号")){
|
||||
def list = map.get(d.address)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(d.address, list)
|
||||
}
|
||||
list << d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def newMap = [:]
|
||||
map.each {k,vals->
|
||||
if(vals.size()>=2){
|
||||
vals.sort{o1,o2->
|
||||
return o1.name.size() <=> o2.name.size()
|
||||
}
|
||||
newMap.put(k,vals)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
newMap.each {k,vals->
|
||||
def names = []
|
||||
for(def d : vals){
|
||||
def newKey = d.name
|
||||
if(d.name.contains("小区")){
|
||||
newKey = d.name.toString().replace("小区", "")
|
||||
}
|
||||
if(!names.contains(newKey)){
|
||||
names << newKey
|
||||
}else{
|
||||
d.flag = "删除"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId( d._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def values = newMap.values()
|
||||
def allRets = []
|
||||
for(def list : values){
|
||||
allRets.addAll(list)
|
||||
}
|
||||
def results = []
|
||||
def delcount = 0
|
||||
for(def r : allRets){
|
||||
def rows = []
|
||||
rows << r._id.toString()
|
||||
rows << r.city
|
||||
rows << r.name
|
||||
rows << r.address
|
||||
rows << r.name_cat
|
||||
if(r.flag){
|
||||
rows << r.flag
|
||||
delcount++
|
||||
}
|
||||
results << rows
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\小区\\删除重复地址(${city}).xlsx", results, "sheet")
|
||||
println "${city} 共:"+map.size()+" 2个以上:"+newMap.size()+" delcount:"+delcount
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def rets = getDatas()
|
||||
def deleteIndex = 0
|
||||
|
||||
+37
-17
@@ -433,23 +433,43 @@ def city = "
|
||||
// }
|
||||
//})
|
||||
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
|
||||
if (d['B']) {
|
||||
def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
|
||||
if(dist){
|
||||
def alias = dist.mapping_alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(alias && !alias.contains(d['C'])){
|
||||
index++
|
||||
println index
|
||||
alias << d['C']
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mapping_alias",alias)
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d ->
|
||||
// if (d['B']) {
|
||||
// def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"])
|
||||
// if(dist){
|
||||
// def alias = dist.mapping_alias
|
||||
// if(!alias){
|
||||
// alias = []
|
||||
// }
|
||||
// if(alias && !alias.contains(d['C'])){
|
||||
// index++
|
||||
// println index
|
||||
// alias << d['C']
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mapping_alias",alias)
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d ->
|
||||
if(d['A']){
|
||||
try{
|
||||
def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"])
|
||||
if(dist){
|
||||
d['B'] = dist.city
|
||||
d['C'] = dist.region
|
||||
d['D'] = dist.name
|
||||
|
||||
def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name)
|
||||
d['E'] = price
|
||||
}
|
||||
}catch(Exception e){
|
||||
println d['A']
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class MapAddressMatch {
|
||||
static main(args) {
|
||||
def task = new MapAddressMatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "北京"
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\北京地址重复数据.xlsx", { d ->
|
||||
index++
|
||||
println index +" "+ d['D']+" "+d['E']
|
||||
if(!d['G']){
|
||||
// def gdData = FangjiaClientUtils.getMapData(city, d['D'],d['E'], d['K'],"", "高德")
|
||||
// if(gdData){
|
||||
// d['F'] = gdData.district
|
||||
// d['G'] = gdData.name
|
||||
// d['H'] = gdData.address
|
||||
// d['I'] = gdData.type
|
||||
// d['J'] = "高德"
|
||||
// }else{
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(city,d['D'],d['E'])
|
||||
def map = findStandardObject(city,d['D'],d['E'], rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(city,d['D'],d['E']);
|
||||
map = findStandardObject(city,d['D'],d['E'], rets);
|
||||
}
|
||||
if(map){
|
||||
if(map.adname){
|
||||
d['F'] = map.adname
|
||||
}else if(map.region){
|
||||
d['F'] = map.region
|
||||
}
|
||||
d['G'] = map.name
|
||||
d['H'] = map.address
|
||||
if(map.type){
|
||||
d['I'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['I'] = map.estateType
|
||||
}
|
||||
|
||||
d['J'] = map.source
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
|
||||
continue;
|
||||
}
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
}
|
||||
+41
-60
@@ -55,6 +55,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
// addBigdataDistrictFJW()
|
||||
|
||||
def citySeqnoMap = CcbCommonUtils.findDistrictSeqno();
|
||||
|
||||
def index = 0
|
||||
for (def city : citys) {
|
||||
index++
|
||||
@@ -88,20 +90,20 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2)
|
||||
}
|
||||
}
|
||||
println index +" "+city +" 小区加载完成 "+districts.size()
|
||||
addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 1.小区表完成"
|
||||
addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 2.小区建筑结构表"
|
||||
addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 3.小区房屋类型表"
|
||||
addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 4.小区建筑年代表"
|
||||
addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 5.小区费用表"
|
||||
addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
|
||||
println index +" "+city +" 6.小区房屋用途表"
|
||||
addBigdataPositionGJ(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 小区加载完成 "+districts.size()
|
||||
// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 1.小区表完成"
|
||||
// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 2.小区建筑结构表"
|
||||
// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 3.小区房屋类型表"
|
||||
// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 4.小区建筑年代表"
|
||||
// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 5.小区费用表"
|
||||
// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city))
|
||||
// println index +" "+city +" 6.小区房屋用途表"
|
||||
addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap)
|
||||
println index +" "+ city + " 7.地理位置表"
|
||||
|
||||
// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行
|
||||
@@ -552,18 +554,17 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
// }
|
||||
}
|
||||
|
||||
def addBigdataPositionGJ(districts, def cityid){
|
||||
def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){
|
||||
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def d : districts) {
|
||||
index++
|
||||
def ret = []
|
||||
def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap);
|
||||
ret << d.cityid
|
||||
ret << "5"
|
||||
ret << d.commid
|
||||
ret << "1"
|
||||
ret << index
|
||||
ret << seqno
|
||||
ret << d.lng
|
||||
ret << d.lat
|
||||
ret << ""
|
||||
@@ -589,49 +590,29 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
def headers = []
|
||||
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
|
||||
println "BIGDATA_POSITION_FJW 数据文件输出完成......"
|
||||
}
|
||||
|
||||
// sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'")
|
||||
//
|
||||
// def sqls = []
|
||||
// def index = 1
|
||||
// for(def d : districts){
|
||||
// def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark},"
|
||||
// index++
|
||||
// if(d.createtime){
|
||||
// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss'),"
|
||||
// }else{
|
||||
// sql = sql + "null,"
|
||||
// }
|
||||
// if(d.updatetime){
|
||||
// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')"
|
||||
// }else{
|
||||
// sql = sql + "null,'${d.sourcecode}')"
|
||||
// }
|
||||
// sqls << sql
|
||||
//
|
||||
// if (sqls.size() >= batchsize) {
|
||||
// try {
|
||||
// sqlInstance.withBatch(batchsize, { stmt ->
|
||||
// sqls.each { def s ->
|
||||
// stmt.addBatch s
|
||||
// }
|
||||
// })
|
||||
//
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// }
|
||||
// sqls.clear()
|
||||
// }
|
||||
//
|
||||
// }
|
||||
//
|
||||
// if (sqls.size() > 0) {
|
||||
// sqlInstance.withBatch(100, { stmt ->
|
||||
// sqls.each { def s ->
|
||||
// stmt.addBatch s
|
||||
// }
|
||||
// })
|
||||
// }
|
||||
def findSeqno(cityid, commid, citySeqnoMap){
|
||||
def detailMap = citySeqnoMap.get(cityid)
|
||||
if(detailMap){
|
||||
def seqno = detailMap.get(commid)
|
||||
if(seqno){
|
||||
return seqno
|
||||
}else{
|
||||
seqno = CcbCommonUtils.findMaxSeqno(cityid)
|
||||
if(seqno){
|
||||
seqno = seqno + 1
|
||||
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
def insertSql = "INSERT INTO bigdata_position_seqno(CITYID,COMMID,SEQNO,CREATETIME) VALUES ('${cityid}','${commid}',${seqno},to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'))"
|
||||
sqlInstance.withBatch(batchsize, { stmt ->
|
||||
stmt.addBatch insertSql
|
||||
})
|
||||
return seqno
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println cityid+" 没有该城市的小区序号"
|
||||
}
|
||||
}
|
||||
|
||||
def addBigdataDistrictFJW(){
|
||||
|
||||
+742
@@ -0,0 +1,742 @@
|
||||
package com.fangjia.script.ccbevaluation.ronghe
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.sql.ResultSetMetaData
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ExportDistrictRongHeTest extends BaseBatch{
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
static main(args) {
|
||||
runBatch(new ExportDistrictRongHeTest(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
|
||||
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
|
||||
|
||||
exportBigdataCommunityGJRH()
|
||||
exportBigdataBuildingFJW()
|
||||
exportBigdataCommfeeGj()
|
||||
|
||||
exportBigdataCommbuildstruGJ()
|
||||
exportBigdataCommhousetypeGJ()
|
||||
exportBigdataCommbuildyearGj()
|
||||
exportBigdataCommhouseuseGJ()
|
||||
exportBigdataPositionGJ()
|
||||
exportBigdataDistrictFJW()
|
||||
exportBigdataAliasFJW()
|
||||
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
exportEvaluateRate()
|
||||
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
|
||||
exportOk(curDate)
|
||||
|
||||
}
|
||||
|
||||
//
|
||||
// def runByCity(){
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//
|
||||
//
|
||||
//// //检查ok文件是否上传
|
||||
//// exportAllBigdataDealFJW()
|
||||
//// for(def city : citys) {
|
||||
//// def cityCode = cityCodeDataMap.get(city)
|
||||
////
|
||||
// //按天导(一个文件夹下)
|
||||
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
||||
// //按天导(每天一个文件夹)
|
||||
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
|
||||
//
|
||||
//// exportBigdataCommunityGJRH(cityCode)
|
||||
//// exportBigdataBuildingFJW(cityCode)
|
||||
//// exportBigdataCommfeeGj(cityCode)
|
||||
////
|
||||
//// exportBigdataCommbuildstruGJ(cityCode)
|
||||
//// exportBigdataCommhousetypeGJ(cityCode)
|
||||
//// exportBigdataCommbuildyearGj(cityCode)
|
||||
//// exportBigdataCommhouseuseGJ(cityCode)
|
||||
//// exportBigdataPositionGJ(cityCode)
|
||||
//// exportBigdataDistrictFJW(cityCode)
|
||||
//// exportBigdataAliasFJW(cityCode)
|
||||
////
|
||||
//// exportBigdataBuildprojFjw(cityCode)
|
||||
//// exportBigdataCommlayoutpicFjw(cityCode)
|
||||
//// }
|
||||
//// exportEvaluateRate()
|
||||
//
|
||||
////
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
// SSHHelper jschCommand = new SSHHelper();
|
||||
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
|
||||
//
|
||||
// exportOk()
|
||||
// }
|
||||
|
||||
def exportBigdataCommunityGJRH(cityCode){
|
||||
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.COMMALIAS
|
||||
data << r.DISTID
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.ADDRESS
|
||||
data << r.LANDAREA
|
||||
data << r.BUILDAREA
|
||||
data << r.COMMTYPE
|
||||
data << r.IFNEW
|
||||
data << r.GREENINGRATE
|
||||
data << r.PLOTRATIO
|
||||
data << r.PARKINGCNT
|
||||
data << ''
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
|
||||
}
|
||||
def exportBigdataCommbuildstruGJ(){
|
||||
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDSTRU
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommhousetypeGJ(cityCode){
|
||||
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.HOUSETYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommbuildyearGj(){
|
||||
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.BUILDYEAR
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommfeeGj(){
|
||||
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.COMPID
|
||||
data << r.FEE
|
||||
data << r.HOUSEUSE
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.FEETYPE
|
||||
data << r.NOTE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataCommhouseuseGJ(){
|
||||
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.COMMID
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSECNT
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataPositionGJ(){
|
||||
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.OBJECTTYPE
|
||||
data << r.OBJECTID
|
||||
data << r.POSITIONTYPE
|
||||
data << r.SEQNO
|
||||
data << r.LONGITUDE
|
||||
data << r.LATITUDE
|
||||
data << r.ALTITUDE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDistrictFJW(){
|
||||
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.DISTSPELL
|
||||
data << r.CITYID
|
||||
data << r.DISTCODE
|
||||
data << r.SEQ
|
||||
data << r.DISTTYPE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataAliasFJW(){
|
||||
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.city_id
|
||||
data << r.ca_id
|
||||
data << r.address_id
|
||||
data << r.address_name
|
||||
data << r.status
|
||||
data << r.exp_dt
|
||||
data << r.version
|
||||
data << r.is_key_name
|
||||
data << r.data_source
|
||||
data << r.description
|
||||
data << r.alias_grade
|
||||
data << r.addr_type_cd
|
||||
data << r.ifexamine
|
||||
data << r.ifdisplay
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataDealFJW(dealdate){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def stop = false
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def index = 0
|
||||
|
||||
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
|
||||
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
|
||||
|
||||
while(!stop){
|
||||
index++
|
||||
def curStartStr = curStartDateStr
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
if(index==1){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
}
|
||||
if(curStartDateStr == curSndDateStr){
|
||||
stop = true
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
|
||||
def curEndStr = DateUtils.add(curStartStr, 1)
|
||||
curStartDateStr = curEndStr
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataDealFJW2(curDate, dealdate){
|
||||
|
||||
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
def allRows = []
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
|
||||
}
|
||||
|
||||
def exportAllBigdataDealFJW(){
|
||||
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.DEALID
|
||||
data << r.CITYNAME
|
||||
data << r.DISTID
|
||||
data << r.DISTNAME
|
||||
data << r.STREET
|
||||
data << r.STREETNO
|
||||
data << r.COMMID
|
||||
data << r.COMMNAME
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.UNITID
|
||||
data << r.UNITNO
|
||||
data << r.UNITNAME
|
||||
data << r.HOUSEID
|
||||
data << r.ADDRESS
|
||||
data << r.BUILDYEAR
|
||||
data << r.PROPTYPE
|
||||
data << r.PROPYEARS
|
||||
data << r.PERIODNUM
|
||||
data << r.FLOORNUM
|
||||
data << r.FLOORTYPE
|
||||
data << r.FLOORCNT
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.BUILDAREA
|
||||
data << r.LIVINGAREA
|
||||
data << r.HOUSESTYLE
|
||||
data << r.ROOMCNT
|
||||
data << r.LIVINGROOMCNT
|
||||
data << r.KITCHENCNT
|
||||
data << r.BATHROOMCNT
|
||||
data << r.HOUSEUSE
|
||||
data << r.HOUSESTRU
|
||||
data << r.HOUSEASPECT
|
||||
data << r.FITMENT
|
||||
data << r.EVALUATENO
|
||||
data << r.DEALUNITPRICE
|
||||
data << r.DEALTOTALPRICE
|
||||
data << r.DECLAREPRICE
|
||||
data << r.EVALUATEPRICE
|
||||
data << r.CONTRACTNO
|
||||
data << r.DEALDATE
|
||||
data << r.DEALTYPE
|
||||
data << r.PAYTYPE
|
||||
data << r.DEALPERIOD
|
||||
data << r.IFELEVATOR
|
||||
data << r.COMPAREAGARDEN
|
||||
data << r.COMPAREATERRACE
|
||||
data << r.COMPAREABALCONY
|
||||
data << r.COMPAREABAYWINDOW
|
||||
data << r.COMPAREAATTIC
|
||||
data << r.COMPAREAGARAGE
|
||||
data << r.COMPAREABASEMENT
|
||||
data << r.COMPAREAOTHER
|
||||
data << r.DATASOURCE
|
||||
data << r.AGENCYID
|
||||
data << r.AGENCYNAME
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME1
|
||||
data << r.UPDATETIME1
|
||||
data << r.SOURCECODE
|
||||
data << r.WORKDATE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
println "BIGDATA_DEAL_FJW 导出完成......"
|
||||
}
|
||||
|
||||
|
||||
def exportBigdataBuildingFJW(cityCode){
|
||||
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.CITYID
|
||||
data << r.BUILDID
|
||||
data << r.BUILDNO
|
||||
data << r.BUILDNAME
|
||||
data << r.BUILDALIAS
|
||||
data << r.DISTID
|
||||
data << r.COMMID
|
||||
data << r.BUILDYEAR
|
||||
if(r.BUILDTYPE){
|
||||
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
|
||||
data << arrs[0].trim()
|
||||
}else{
|
||||
data << ''
|
||||
}
|
||||
data << r.BUILDSTRU
|
||||
data << r.FLOORCNT
|
||||
data << r.UNITCNT
|
||||
data << r.HOUSECNT
|
||||
data << r.IFELEVATOR
|
||||
data << r.ELEVATORCNT
|
||||
data << r.ELEVATORHOUSE
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
data << r.VALID_FLAG
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def exportEvaluateRate(){
|
||||
def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.ID
|
||||
data << r.RATE_TYPE
|
||||
data << r.RATE
|
||||
data << r.RANK
|
||||
data << r.CITY
|
||||
data << r.REMARK
|
||||
data << r.CREATETIME
|
||||
data << r.UPDATETIME
|
||||
data << r.SOURCECODE
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "EVALUATE_RATE_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportOk(curDate){
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
|
||||
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
}
|
||||
+2
-1
@@ -218,7 +218,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","金辉伯爵山")
|
||||
// q.put("name","玉泉路甲46号院")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -829,6 +829,7 @@ class CommonDatautils {
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
|
||||
+4
-4
@@ -23,8 +23,8 @@ class FzDataResultStat {
|
||||
// task.regionPriceTrendRun()
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
def map1 = [:]
|
||||
@@ -53,7 +53,7 @@ class FzDataResultStat {
|
||||
}
|
||||
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup]
|
||||
}
|
||||
//ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
|
||||
|
||||
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
@@ -94,7 +94,7 @@ class FzDataResultStat {
|
||||
}
|
||||
def prices = exportRegion(city, months)
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")
|
||||
ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据")
|
||||
ExcelUtils.write("D:\\数据\\福州\\福州6区均价半年走势_${curMonth}.xlsx", prices, "数据")
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202201.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+5
-2
@@ -143,8 +143,8 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
|
||||
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
|
||||
}
|
||||
if(!retMap.anjuke_esf_price && d.anjuke_esf_price){
|
||||
retMap.anjuke_esf_price = d.anjuke_esf_price
|
||||
if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){
|
||||
retMap.anjuke_esf_price = d.get("58tc_esf_price")
|
||||
}
|
||||
if(!retMap.soufang_esf_price && d.soufang_esf_price){
|
||||
retMap.soufang_esf_price = d.soufang_esf_price
|
||||
@@ -152,6 +152,9 @@ class ThirdPriceExportBatch extends BaseBatch{
|
||||
if(!retMap.lianjia_esf_price && d.lianjia_esf_price){
|
||||
retMap.lianjia_esf_price = d.lianjia_esf_price
|
||||
}
|
||||
if(!retMap.lianjia_esf_price && d.beike_esf_price){
|
||||
retMap.lianjia_esf_price = d.beike_esf_price
|
||||
}
|
||||
|
||||
if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){
|
||||
retMap.cityhouse_esf_price = d.cityhouse_esf_price
|
||||
|
||||
@@ -45,7 +45,7 @@ class EsHouseStat {
|
||||
|
||||
public static void main(String[] args){
|
||||
def task = new EsHouseStat();
|
||||
//task.run();
|
||||
task.run();
|
||||
// task.testRun("ÉîÛÚ")
|
||||
// task.exportOpenLog();
|
||||
// task.exportMongoLog()
|
||||
@@ -59,7 +59,7 @@ class EsHouseStat {
|
||||
// task.szRentHouseStat();
|
||||
// task.addPrice();
|
||||
|
||||
task.statNoHouseCCBData();
|
||||
// task.statNoHouseCCBData();
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
@@ -337,21 +337,6 @@ class EsHouseStat {
|
||||
return total;
|
||||
}
|
||||
|
||||
def findHouse(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def total = 0;
|
||||
while (rs.next()){
|
||||
total = rs.getInt("count")
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
|
||||
def findHouseSource(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM \"seawater-20*\" WHERE 1=1")
|
||||
@@ -373,6 +358,21 @@ class EsHouseStat {
|
||||
return statMap;
|
||||
}
|
||||
|
||||
|
||||
def findHouse(def city, def startDate, def endDate){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and cat = 'sell'")
|
||||
sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate);
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def total = 0;
|
||||
while (rs.next()){
|
||||
total = rs.getInt("count")
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
def testRun(def city){
|
||||
|
||||
|
||||
|
||||
@@ -149,6 +149,7 @@ class OpenApiUtils {
|
||||
ret.lng2 = gmap.get("lng")
|
||||
ret.lat2 = gmap.get("lat")
|
||||
ret.estateType = ret.type
|
||||
ret.source = "高德"
|
||||
}catch (Exception e){
|
||||
|
||||
}
|
||||
@@ -250,14 +251,14 @@ class OpenApiUtils {
|
||||
}
|
||||
|
||||
if(type && type.contains("出入口")){
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}"
|
||||
}else{
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
poiList.add(jo);
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
@@ -302,7 +303,7 @@ class OpenApiUtils {
|
||||
def distname = formatName(ret.get("name"))
|
||||
address = ret.get("address")
|
||||
if((distname && distname.contains(name)) || (distname && name.contains(distname)) || (address && address.contains(name)) || ( address && name.contains(address))){
|
||||
if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
|
||||
// if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){
|
||||
lng2 = location.lng
|
||||
lat2 = location.lat
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
|
||||
@@ -311,7 +312,7 @@ class OpenApiUtils {
|
||||
def mapRegion = ret.get("area")
|
||||
distRegion = mapRegion
|
||||
districtName = distname
|
||||
if(address && address.contains(mapRegion)){
|
||||
if(address && mapRegion && address.contains(mapRegion)){
|
||||
address = address.substring(address.indexOf(mapRegion)+mapRegion.size());
|
||||
}
|
||||
if(ret.get("detail_info")){
|
||||
@@ -330,6 +331,7 @@ class OpenApiUtils {
|
||||
map.name = districtName
|
||||
map.region = distRegion
|
||||
map.estateType = estateType
|
||||
map.source = "百度"
|
||||
rets << map
|
||||
|
||||
lat = null
|
||||
@@ -340,12 +342,18 @@ class OpenApiUtils {
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
|
||||
}else{
|
||||
println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
}
|
||||
// }else{
|
||||
// println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
|
||||
// }
|
||||
}else{
|
||||
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
|
||||
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
|
||||
def score1 = 0
|
||||
if(distname && name){
|
||||
score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name);
|
||||
}
|
||||
def score2 = 0
|
||||
if(address && name){
|
||||
score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name);
|
||||
}
|
||||
if(score1 || score2){
|
||||
rets << map
|
||||
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】(score:"+score1+"#"+score2+")"
|
||||
|
||||
@@ -31,7 +31,7 @@ class RegionPointCrawl {
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
|
||||
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
|
||||
for(def p : provinces){
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
task.run(p);
|
||||
|
||||
@@ -7,11 +7,11 @@ email.setSSL(true);
|
||||
email.setHostName("smtp.exmail.qq.com");
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码
|
||||
email.addTo("gaojuan@fangjia.com"); //接收方
|
||||
email.setFrom("service@fangjia.com"); //发送方
|
||||
email.setSubject("test"); //标题
|
||||
email.setContent("你?", "text/html");
|
||||
email.setContent("测试", "text/html");
|
||||
|
||||
//email.setHostName("smtp.exmail.qq.com");
|
||||
//email.setSSL(true);
|
||||
|
||||
+42
-25
@@ -7,13 +7,17 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
/**
|
||||
* 模糊地址经纬度更新
|
||||
@@ -44,13 +48,16 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
//"北京","常州",
|
||||
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
|
||||
def citys =["上海"] //CommonDatautils.findAllCity();
|
||||
|
||||
def citys =["北京"] //CommonDatautils.findAllCity();
|
||||
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
|
||||
//市谊大厦 华三新村 阳光家园
|
||||
for (def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat2", "lat", "lng2", "lng"]);
|
||||
def rets = findVagueAddress(city, districts);
|
||||
findAndUpdateLocation(rets);
|
||||
writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
|
||||
// writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv")
|
||||
writeCSV(results, "d:\\数据\\结果\\" + city + "地址模糊.csv")
|
||||
// ExcelUtils.write("d:\\数据\\结果\\" + city + "地址模糊.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +65,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
|
||||
def findVagueAddress(def city, def districts) {
|
||||
def rets = []
|
||||
def rows = []
|
||||
for (def district : districts) {
|
||||
def address = district.address;
|
||||
if (address) {
|
||||
@@ -66,10 +74,13 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
//没地址
|
||||
if (!address) {
|
||||
rets << district;
|
||||
|
||||
// rows << [district.city, district.region, district.name, district.address]
|
||||
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
|
||||
} else {
|
||||
if (!findPattern(address)) {
|
||||
rets << district;
|
||||
// rows << [district.city, district.region, district.name, district.address]
|
||||
println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址"
|
||||
} else {
|
||||
println district.city + " " + district.address + " 精确地址"
|
||||
@@ -97,7 +108,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
} else if (address.indexOf("路") == -1 && address.indexOf("街") == -1 && address.indexOf("大道") == -1 && address.indexOf("巷") == -1 && address.indexOf("北里") == -1 && address.indexOf("南里") == -1 && address.indexOf("东里") == -1 && address.indexOf("西里") == -1) {
|
||||
return null;
|
||||
} else {
|
||||
pattern = Pattern.compile("[路|街|大道|巷]\\d+");
|
||||
pattern = Pattern.compile("[路甲|路|街|大道|巷|弄|里]\\d+");
|
||||
matcher = pattern.matcher(address)
|
||||
if(matcher.find()){
|
||||
return s;
|
||||
@@ -123,16 +134,14 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
for (def ret : rets) {
|
||||
index++;
|
||||
|
||||
if(ret.name!="药师新村"){
|
||||
continue
|
||||
}
|
||||
|
||||
def locationMap = findLatAndLng(ret);
|
||||
if (locationMap) {
|
||||
updateLocation(locationMap, ret);
|
||||
} else {
|
||||
println "[没找到] " + ret.city + " " + ret.region + " " + ret.name + " " + ret.address
|
||||
}
|
||||
println "【${index}/${rets.size()}】************************************************************************************"
|
||||
println "【${index}/${rets.size()}】**************【${ret.region}---${ret.name}----${ret.address}】**********************************************************************"
|
||||
|
||||
}
|
||||
}
|
||||
@@ -148,14 +157,14 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
msgMap.db_lng2 = ret.lng2
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
|
||||
if (ret.lat2 != locationMap.lat2) {
|
||||
udata.put('lat2', locationMap.lat2)
|
||||
udata.put('lng2', locationMap.lng2)
|
||||
if (ret.lat2 && ret.lat2 != locationMap.lat2) {
|
||||
msgMap.m_lat2 = locationMap.lat2
|
||||
msgMap.m_lng2 = locationMap.lng2
|
||||
def distance = GeoHelper.GetDistance(ret.lat2, ret.lng2, locationMap.lat2, locationMap.lng2)
|
||||
msgMap.updateLngFlag = true
|
||||
msgMap.distance = distance
|
||||
udata.put('lat2', locationMap.lat2)
|
||||
udata.put('lng2', locationMap.lng2)
|
||||
}
|
||||
|
||||
if (ret.lat != locationMap.lat) {
|
||||
@@ -171,31 +180,39 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
if (udata) {
|
||||
println ret.city + " " + ret.region + " " + ret.name + " [ 修改其经纬度!]"
|
||||
udata.put("m_date", new Date())
|
||||
// DBUtils.seaweed.update(new BasicDBObject('_id',locationMap._id),new BasicDBObject('$set',udata))
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',locationMap.id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
results << msgMap
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def name, def rets) {
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() > 1) {
|
||||
//相似里找有精确地址
|
||||
for (def mapObj : rets) {
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
if (mapObj.address && findPattern(mapObj.address)) {
|
||||
map = mapObj;
|
||||
break;
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name.contains(name) || name.contains(mapObj.name)){
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
if (mapObj.address && findPattern(mapObj.address)) {
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!map){
|
||||
//交带有交叉口地址
|
||||
for (def mapObj : rets) {
|
||||
if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) {
|
||||
if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){
|
||||
map = mapObj;
|
||||
break;
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name.contains(name) || name.contains(mapObj.name)) {
|
||||
if (mapObj.address && (mapObj.address.contains("交汇") || mapObj.address.contains("交叉"))) {
|
||||
if (!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")) {
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,18 +228,18 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
def findLatAndLng(ret) {
|
||||
//https://restapi.amap.com/v3/place/text?keywords=
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(ret.city, ret.region, ret.name)
|
||||
def map = findStandardObject(ret.city, ret.name, rets);
|
||||
def map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
def locationMap = [:]
|
||||
if (!map || !map.lat) {
|
||||
//http://api.map.baidu.com/place/v2/search
|
||||
rets = OpenApiUtils.getBiDuLocation(ret.city, ret.region, ret.name);
|
||||
map = findStandardObject(ret.city, ret.name, rets);
|
||||
map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
}
|
||||
if (!map || !map.lat) {
|
||||
//https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap
|
||||
def code = codeMap.get(ret.city);
|
||||
rets = OpenApiUtils.searchBaiDuMap(ret.name, ret.region, code.toInteger());
|
||||
map = findStandardObject(ret.city, ret.name, rets);
|
||||
map = findStandardObject(ret.city, ret.region, ret.name, rets);
|
||||
}
|
||||
|
||||
// if((!map || !map.lat)){
|
||||
@@ -237,7 +254,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
locationMap.lng2 = (map.lng2.toDouble() * 10000000000).longValue()
|
||||
locationMap.id = ret._id
|
||||
locationMap.city = ret.city
|
||||
if (ret.address.toString().endsWith("路")) {
|
||||
if (map.address) {
|
||||
locationMap.address = map.address.replace("(","").replace(")","");
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -48,7 +48,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-12-01"
|
||||
static def statDate = "2022-01-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年2月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+14
-2
@@ -12,7 +12,19 @@ class TempMatchDistrict {
|
||||
static main(args) {
|
||||
def task = new TempMatchDistrict()
|
||||
// task.run()
|
||||
task.regionConver()
|
||||
// task.regionConver()
|
||||
task.stat()
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"]
|
||||
def total = 0
|
||||
for(def city : citys){
|
||||
def count = CommonDatautils.findAllDistrictCount("district", city)
|
||||
total = total + count
|
||||
println city +" "+count
|
||||
}
|
||||
println "共:"+total
|
||||
}
|
||||
|
||||
def regionConver(){
|
||||
@@ -29,7 +41,7 @@ class TempMatchDistrict {
|
||||
}
|
||||
def run(){
|
||||
def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据"
|
||||
def files = ["合肥.xlsx","湖北.xlsx","陕西.xlsx","上海.xlsx","深圳.xlsx","苏州.xlsx","天津.xlsx","雄安.xlsx","浙江.xlsx","重庆.xlsx"]
|
||||
def files = ["杭州.xlsx"]
|
||||
for(def f : files){
|
||||
def filePath = path+"\\"+f;
|
||||
match(filePath);
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ class FangjiaClientUtils {
|
||||
public static def getMapData(def city, def region, def name, def address, def types, def mapCode){
|
||||
JSONObject jsonObj = null
|
||||
try {
|
||||
def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
//def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode
|
||||
if(address){
|
||||
url = url + "&address="+address
|
||||
|
||||
+340
@@ -0,0 +1,340 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class ZsNoExistsToDB {
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new ZsNoExistsToDB()
|
||||
task.run()
|
||||
// task.addDistrictToDB()
|
||||
}
|
||||
|
||||
def addDistrictToDB(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1)
|
||||
def keys = []
|
||||
def addIndex = 0
|
||||
for(def d : datas){
|
||||
if(d['J'] && d['J']=="新增"){
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def name = d['F'].toString().replace("·","")
|
||||
def address = d['G']
|
||||
def dist = CommonDatautils.findDistrict(city,region,name,"district")
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findDistrict(city,region,d['C'],"district")
|
||||
}
|
||||
if(!dist){
|
||||
keys << city+"##"+region+"##"+name
|
||||
def alias = []
|
||||
def str = d['H']
|
||||
if(str){
|
||||
def arrs = str.split(",")
|
||||
for(def s : arrs){
|
||||
if(!alias.contains(s)){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(address){
|
||||
address = AddressUtil.formatAddress(city, address)
|
||||
}
|
||||
def lng = d['M']
|
||||
def lat = d['L']
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
long lng_new = 0
|
||||
long lat_new = 0
|
||||
long lng2_new = 0
|
||||
long lat2_new = 0
|
||||
if(lng){
|
||||
lng = lng.toString().toDouble()
|
||||
lat = lat.toString().toDouble()
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
lng2 = baidu.x
|
||||
lat2 = baidu.y
|
||||
lng_new = lng *10000000000L
|
||||
lat_new = lat *10000000000L
|
||||
if(lng2){
|
||||
lng2_new = lng2 *10000000000L
|
||||
lat2_new = lat2 *10000000000L
|
||||
}
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",region)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", address)
|
||||
if(lng_new){
|
||||
data.put("lat",lat_new)
|
||||
data.put("lng",lng_new)
|
||||
data.put("lat2",lat2_new)
|
||||
data.put("lng2",lng2_new)
|
||||
}
|
||||
if(alias){
|
||||
data.put("alias",alias)
|
||||
}
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
DbUtils.seaweed.save(data)
|
||||
FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12);
|
||||
addIndex++
|
||||
println "[新增] "+addIndex +" "+ data
|
||||
|
||||
}else{
|
||||
println dist.city +" "+dist.region+" "+dist.name+" 小区已存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findOpenLog2()
|
||||
// ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据")
|
||||
println rets.size()
|
||||
|
||||
// matchMap2()
|
||||
}
|
||||
|
||||
def matchMap2() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d ->
|
||||
if(!d['E']){
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject(d['A'],d['B'],d['C'], rets);
|
||||
|
||||
def locationMap = [:]
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
map = findStandardObject(d['A'],d['B'],d['C'], rets);
|
||||
}
|
||||
if(map){
|
||||
println map
|
||||
if(map.adname){
|
||||
d['E'] = map.adname
|
||||
}else if(map.region){
|
||||
d['E'] = map.region
|
||||
}
|
||||
d['F'] = map.name
|
||||
d['G'] = map.address
|
||||
d['K'] = map.estateType
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.lng
|
||||
d['N'] = map.lat2
|
||||
d['O'] = map.lng2
|
||||
d['P'] = map.source
|
||||
d['Q'] = RegionData.formatRegion(d['A'], d['E'])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address);
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def matchMap(){
|
||||
ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data->
|
||||
def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德")
|
||||
if(!gdData){
|
||||
gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德")
|
||||
}
|
||||
if(gdData){
|
||||
data['E'] = gdData.district
|
||||
data['F'] = gdData.name
|
||||
data['G'] = gdData.address
|
||||
data['H'] = gdData.type
|
||||
data['I'] = gdData.lng
|
||||
data['J'] = gdData.lat
|
||||
data['K'] = gdData.score
|
||||
}
|
||||
|
||||
def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度")
|
||||
if(!bdData){
|
||||
bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度")
|
||||
}
|
||||
if(bdData){
|
||||
data['L'] = bdData.district
|
||||
data['M'] = bdData.name
|
||||
data['N'] = bdData.address
|
||||
data['O'] = bdData.type
|
||||
data['P'] = bdData.lng
|
||||
data['Q'] = bdData.lat
|
||||
data['R'] = bdData.score
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findOpenLog2(){
|
||||
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18")
|
||||
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate)
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
|
||||
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
|
||||
def index = 0
|
||||
def count = 0
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
if(d.code == 400){
|
||||
index++
|
||||
def msg = d.msg
|
||||
if(msg.contains("该楼盘不存在")){
|
||||
def arrs = msg.split(" ")
|
||||
if(arrs){
|
||||
def rMap = [:]
|
||||
for(def s : arrs){
|
||||
if(s.contains(":")){
|
||||
def ars = s.split(":")
|
||||
if(ars.size()==2){
|
||||
rMap.put(ars[0], ars[1])
|
||||
}else{
|
||||
rMap.put(ars[0], null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
count++
|
||||
def key = rMap.city+"#"+rMap.region+"#"+rMap.name
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def params = d.parameter_map
|
||||
def address = null
|
||||
if(params && params.address){
|
||||
address = params.address
|
||||
}
|
||||
def r = [rMap.city, rMap.region, rMap.name, address]
|
||||
if(!rets.contains(r)){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println msg
|
||||
}
|
||||
}
|
||||
}
|
||||
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findOpenLog(){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1")
|
||||
sqlBuffer.append(" and user_id = '1007569'")
|
||||
sqlBuffer.append(" and request_uri = '/czbank/evaluation'")
|
||||
sqlBuffer.append(" and c_date >= '2022-01-01'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def rets = []
|
||||
rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"]
|
||||
def index = 0
|
||||
def count = 0
|
||||
while (rs.next()){
|
||||
def code = rs.getInt("code")
|
||||
if(code == 400){
|
||||
index++
|
||||
def msg = rs.getString("msg")
|
||||
if(msg && msg.contains("该楼盘不存在")){
|
||||
def arrs = msg.split(" ")
|
||||
if(arrs){
|
||||
def rMap = [:]
|
||||
for(def s : arrs){
|
||||
if(s.contains(":")){
|
||||
def ars = s.split(":")
|
||||
if(ars.size()==2){
|
||||
rMap.put(ars[0], ars[1])
|
||||
}else{
|
||||
rMap.put(ars[0], null)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rMap){
|
||||
count++
|
||||
def params = rs.getObject("parameter_map")
|
||||
println params
|
||||
def r = [rMap.city, rMap.region, rMap.name]
|
||||
if(!rets.contains(r)){
|
||||
rets << r
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println msg
|
||||
}
|
||||
}
|
||||
}
|
||||
println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+210
-7
@@ -1,11 +1,20 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.codehaus.jackson.sym.NameN
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
class DistrictLocationUpdate {
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
@@ -19,20 +28,214 @@ class DistrictLocationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationUpdate()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.mapMatch()
|
||||
// task.updateLocationToDB()
|
||||
}
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"])
|
||||
|
||||
|
||||
updateLocation(city, dists);
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"])
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"]
|
||||
for(def d : dists){
|
||||
def lng = null
|
||||
def lat = null
|
||||
if(d.lng){
|
||||
lng = d.lng/10000000000.0
|
||||
lat = d.lat/10000000000.0
|
||||
}
|
||||
rets << [d.city, d.region, d.name, d.address, lng,lat]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def mapMatch(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
|
||||
if(!d['H'] && d['C']=="华侨城龙居御景二期"){
|
||||
index++
|
||||
// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德")
|
||||
// println d['A']+" "+d['B']+" "+d['C']
|
||||
// if(gdData){
|
||||
// d['G'] = gdData.district
|
||||
// d['H'] = gdData.name
|
||||
// d['I'] = gdData.address
|
||||
// d['J'] = gdData.type
|
||||
// d['K'] = gdData.lng
|
||||
// d['L'] = gdData.lat
|
||||
// d['M'] = gdData.score
|
||||
// if(d['E'] && d['K']){
|
||||
// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
|
||||
// d['N'] = distance
|
||||
// }
|
||||
// }
|
||||
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
if (!map || !map.lat) {
|
||||
rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
}
|
||||
if(map){
|
||||
if(map.adname){
|
||||
d['G'] = map.adname
|
||||
}else if(map.region){
|
||||
d['G'] = map.region
|
||||
}
|
||||
d['H'] = map.name
|
||||
d['I'] = map.address
|
||||
if(map.type){
|
||||
d['J'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['J'] = map.estateType
|
||||
}
|
||||
d['K'] = map.lng
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.score
|
||||
if(d['E'] && d['K']){
|
||||
def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble())
|
||||
d['N'] = distance
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def updateLocationToDB(){
|
||||
ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d->
|
||||
if(d['N'] && d['N']>1000){
|
||||
if(d['M']>=0.4 && d['K']){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", d['A'])
|
||||
query.put("region", d['B'])
|
||||
query.put("name", d['C'])
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
|
||||
Long lng0 = d['K'].toString().toDouble()*10000000000L
|
||||
Long lat0 = d['L'].toString().toDouble()*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
|
||||
def gmap = GeocodeUtils.getGaodeToBaidu(d['K'].toString().toDouble(), d['L'].toString().toDouble())
|
||||
Long lng2 = gmap.get("lng")*10000000000L
|
||||
Long lat2 = gmap.get("lat")*10000000000L
|
||||
udata.put("lat2", lat2)
|
||||
udata.put("lng2", lng2)
|
||||
DbUtils.seaweed.update(query, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
def findStandardObject(def city, def region, def name, def rets) {
|
||||
def map;
|
||||
if (rets) {
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){
|
||||
continue;
|
||||
}
|
||||
// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.address && mapObj.address.contains(name)){
|
||||
map = mapObj;
|
||||
break;
|
||||
}else if(mapObj.name && name){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name);
|
||||
if(score1){
|
||||
map = mapObj;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def queryPoiMap(def city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
|
||||
|
||||
+2
-2
@@ -40,9 +40,9 @@ class LocationTest {
|
||||
// System.out.println(str);
|
||||
|
||||
def task = new LocationTest()
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run2()
|
||||
task.run3()
|
||||
// task.run3()
|
||||
// task.run4()
|
||||
// task.run5()
|
||||
}
|
||||
|
||||
+383
@@ -0,0 +1,383 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.gj.paimai.ChromeDriverPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
|
||||
public class BeiKeCrawlByChrome {
|
||||
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.addDealRegion();
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
for(def d : dbcursor){
|
||||
keys << ["city":d.city, "name":d.district_name]
|
||||
}
|
||||
for(def m : keys){
|
||||
//def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district")
|
||||
def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district")
|
||||
if(list && list.size()==1){
|
||||
def dist = list[0]
|
||||
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", m.city)
|
||||
query.put("region", null)
|
||||
query.put("district_name", m.name)
|
||||
// query.put("source", "链家在线")
|
||||
query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd")))
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
def ds = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
|
||||
def getPageUrls(driver, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
|
||||
def getPageDeail(driver, city, cityUrl){
|
||||
def stop = false
|
||||
def page = 1
|
||||
def detailUrls = []
|
||||
while(!stop){
|
||||
def url = cityUrl + "/chengjiao/pg${page}/"
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def elements = JQueryUtils.select(".listContent li", doc)
|
||||
if(elements){
|
||||
for(def e : elements){
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = e.select(".dealDate").text()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
detail.direction = houseInfos[0]
|
||||
detail.fitment = houseInfos[1]
|
||||
def floorStr = e.select(".positionInfo").text()
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
ex.printStackTrace()
|
||||
println e.select(".title a").attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findRegionByThirdPary(city, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("name", district_name)
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1)
|
||||
def dbo = null;
|
||||
for(def d : dbcursor){
|
||||
dbo = d
|
||||
}
|
||||
if(dbo){
|
||||
return dbo.region
|
||||
}
|
||||
return null;
|
||||
}
|
||||
def getContent(driver, detailUrl){
|
||||
driver.get(detailUrl)
|
||||
def str = driver.getPageSource()
|
||||
if(str && str.contains("<title>登录</title>")){
|
||||
getLogin(driver)
|
||||
return getContent(driver, detailUrl)
|
||||
}
|
||||
return str;
|
||||
}
|
||||
def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){
|
||||
if(detailUrls){
|
||||
for(def i=0;i<detailUrls.size();i++){
|
||||
def detailUrl = detailUrls[i]
|
||||
try{
|
||||
def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl)
|
||||
println ${i}+" "+detail.city + " " + detail.region + " " + detail.district_name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def crawling(driver, DEAL_PRICE_COLL, detailUrl){
|
||||
def content = getContent(driver, detailUrl)
|
||||
def doc = Jsoup.parse(content);
|
||||
def eles = JQueryUtils.select(".deal-bread a", doc)
|
||||
def detail = new BasicDBObject()
|
||||
if(eles && eles.size()>=4){
|
||||
detail.city = eles[0].text().replace("房产网","")
|
||||
detail.region = eles[2].text().replace("二手房成交","")
|
||||
detail.district_name = eles[eles.size()-1].text().replace("二手房成交","")
|
||||
}
|
||||
if(detail.district_name) {
|
||||
def price = JQueryUtils.selectText(".dealTotalPrice i", doc)
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = price.toInteger() * 10000
|
||||
}
|
||||
def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "")
|
||||
if (areaStr) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
if (detail.total_price && detail.area) {
|
||||
detail.avg_price = (int) (detail.total_price / detail.area)
|
||||
}
|
||||
def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim()
|
||||
if (tradeStr) {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "链家在线"
|
||||
def dbo = findTrade(DEAL_PRICE_COLL, detail)
|
||||
if (!dbo) {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
|
||||
}
|
||||
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
|
||||
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
|
||||
def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "")
|
||||
if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) {
|
||||
floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)"))
|
||||
if (floorStr && floorStr.isNumber()) {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = detailUrl
|
||||
detail.c_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
q.put("region", detail.region)
|
||||
q.put("district_name", detail.district_name)
|
||||
q.put("area", detail.area)
|
||||
q.put("total_price", detail.total_price)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
|
||||
def getAllCityUrls(driver){
|
||||
driver.get("https://www.ke.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
// if(e.text().contains("天津")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+41
-13
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
@@ -33,9 +34,10 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.run()
|
||||
// task.run1()
|
||||
// task.addDealRegion();
|
||||
task.standardRegion()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
@@ -94,22 +96,44 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run(){
|
||||
// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
// System.setProperty("http.nonProxyHosts", "localhost");
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
// def cityMap = getAllCityUrls(driver)
|
||||
// getLogin(driver)
|
||||
// crawlData(driver, cityMap)
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def standardRegion(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) )
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
if(d.region){
|
||||
def newRegion = RegionData.formatRegion(d.city, d.region)
|
||||
if(newRegion && newRegion!=d.region){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city", d.city)
|
||||
query.put("region", d.region)
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", newRegion)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def addDealRegion(){
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
lastMonth = lastMonth +"-01"
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd")))
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")))
|
||||
// q.put("source", new BasicDBObject('$in', ["Á´¼ÒÔÚÏß","±´¿ÇÕÒ·¿"]))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def keys = []
|
||||
@@ -131,6 +155,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
def data = new BasicDBObject()
|
||||
data.put("region", dist.region)
|
||||
println m.city+ " "+m.name +" "+dist.region
|
||||
def ds = DEAL_PRICE_COLL.count(query)
|
||||
DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true)
|
||||
}
|
||||
@@ -193,7 +218,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -219,6 +244,9 @@ public class LianjiaCrawlByChrome {
|
||||
detail.city = city
|
||||
detail.district_name = arrs[0]
|
||||
detail.region = findRegionByThirdPary(detail.city, detail.district_name)
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
@@ -391,9 +419,9 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(!href.contains("fang.lianjia")){
|
||||
// if(!href.contains("fang.lianjia") && e.text().contains("¸£ÖÝ")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
+62
-24
@@ -37,7 +37,10 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findCrawlData()
|
||||
def source = "58同城"
|
||||
def root = "58.com"
|
||||
def shortCityMap = findCitySeeds(root)
|
||||
def datas = findCrawlData(source, shortCityMap, root)
|
||||
def list = []
|
||||
datas.each {k,v->
|
||||
if(v){
|
||||
@@ -58,15 +61,32 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
}
|
||||
|
||||
def findCrawlData(){
|
||||
def findCitySeeds(root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("root", root)
|
||||
q.put("city", "福州")
|
||||
|
||||
def dbcursor = districtseedsColl.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(!cityMap.keySet().contains(d.city)){
|
||||
cityMap.put(d.city, d.city_short)
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
}
|
||||
|
||||
def findCrawlData(def source, shortCityMap, def root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","¸£ÖÝ")
|
||||
q.put("source", "³ÇÊз¿²ú")
|
||||
q.put("s_date",202012)
|
||||
q.put("source", source)
|
||||
q.put("s_date",202112)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = ".cityhouse.cn/ha/"
|
||||
def prefix = "xiaoqu/"
|
||||
def keyList =new HashSet()
|
||||
def index = 0
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
@@ -76,33 +96,42 @@ class CrawlDistrictIdSelect {
|
||||
}else{
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
|
||||
|
||||
if(url){
|
||||
def district_id = ""
|
||||
if(url.contains(prefix)){
|
||||
district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1)
|
||||
}else{
|
||||
district_id = url;
|
||||
}
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
def ids = retMap.get(db.city)
|
||||
if(!ids){
|
||||
ids = []
|
||||
retMap.put(key, ids)
|
||||
retMap.put(db.city, ids)
|
||||
}
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "cityhouse.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
if(!keyList.contains(db.city+"#"+district_id)){
|
||||
def count = findIsExists(db.city, district_id, root)
|
||||
if(!count){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "58.com"
|
||||
map.city_short= shortCityMap.get(db.city)
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
map.name = db.name
|
||||
ids << map
|
||||
}else{
|
||||
index++
|
||||
println "[${index}]已存在:"+db.city +" "+db.name
|
||||
}
|
||||
keyList << db.city+"#"+district_id
|
||||
}else{
|
||||
println "ÖØ¸´£º"+key
|
||||
println "重复:"+db.city+"#"+db.name
|
||||
}
|
||||
}else{
|
||||
println url +" Á´½Ó²»ºÏÒªÇó"
|
||||
@@ -110,4 +139,13 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findIsExists(city, district_id, root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("district_id", district_id)
|
||||
q.put("root",root)
|
||||
def count = districtseedsColl.count(q)
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -68,7 +68,7 @@ class FZloationUpdate {
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[ÐÞ¸Ä-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
@@ -153,7 +153,7 @@ class FZloationUpdate {
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
if(reps && reps.count && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
|
||||
@@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch {
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "G00jia123456");
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
|
||||
email.addTo(mail.receiverEmail);
|
||||
email.setFrom("service@fangjia.com");//·¢ËÍ·½
|
||||
email.setSubject(mail.subject); //±êÌâ
|
||||
@@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch {
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "G00jia123456");
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
|
||||
email.addTo(mail.receiverEmail);
|
||||
email.setFrom("service@fangjia.com");
|
||||
email.setSubject(mail.subject);
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "58ͬ³Ç"//
|
||||
def citys = findCrawlCity(source, 202111);
|
||||
def source = "城市房产"//
|
||||
def citys = findCrawlCity(source, 202112);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202111);
|
||||
def count2 = findCount(source, city, 202112);
|
||||
def count1 = findCount(source, city, 202112);
|
||||
def count2 = findCount(source, city, 202201);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
+131
-129
File diff suppressed because one or more lines are too long
+9
-1
@@ -315,11 +315,11 @@ class AddMetadataBatchQfang{
|
||||
private static void saveThirdPartyPrice(def saveDate){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", saveDate.get("xqUrl"))
|
||||
q.put("s_date", SHORT_DATE);
|
||||
q.put("city", saveDate.get("city"));
|
||||
q.put("region", saveDate.get("region"));
|
||||
q.put("name", saveDate.get("xqName"));
|
||||
q.put("source", saveDate.get("source"));
|
||||
|
||||
def udata = THIRD_PARTY_PRICE_COLL.findOne(q)
|
||||
if(!udata){
|
||||
@@ -336,6 +336,14 @@ class AddMetadataBatchQfang{
|
||||
obj.put("source", saveDate.get("source"));
|
||||
obj.put("c_date", new Date());
|
||||
THIRD_PARTY_PRICE_COLL.save(obj);
|
||||
println "[save] "+saveDate.toString()
|
||||
}else{
|
||||
def updata = new BasicDBObject()
|
||||
updata.put("avg_price", saveDate.get("price"))
|
||||
updata.put("url", saveDate.get("xqUrl"));
|
||||
updata.put("m_date", new Date());
|
||||
THIRD_PARTY_PRICE_COLL.update(q,new BasicDBObject('$set',updata))
|
||||
println "[update] "+saveDate.toString()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -87,7 +87,7 @@ class AddSeedBatchQfang extends BaseBatch{
|
||||
href = "https:"+href;
|
||||
}
|
||||
if(href&&!CITYS.contains(city)){
|
||||
ret.put(city, href);
|
||||
ret.put(city, href);
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
|
||||
+9
-4
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
|
||||
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["fang.com":"搜房"]
|
||||
def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
@@ -94,11 +94,16 @@ class SynchDistrictPrice {
|
||||
for(def r : rets){
|
||||
def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source)
|
||||
if(obj){
|
||||
updateIndex++
|
||||
thirdPartyColl.remove(new BasicDBObject('_id',obj._id))
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}else{
|
||||
thirdPartyColl.save(r)
|
||||
try{
|
||||
addIndex++
|
||||
thirdPartyColl.save(r)
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
// if(!obj){
|
||||
// addIndex++
|
||||
@@ -112,7 +117,7 @@ class SynchDistrictPrice {
|
||||
// }
|
||||
// }
|
||||
}
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex
|
||||
println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 已存在:"+updateIndex
|
||||
}
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202111);
|
||||
def count2 = findTotalCount(source, 202112);
|
||||
def count1 = findTotalCount(source, 202112);
|
||||
def count2 = findTotalCount(source, 202201);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20211201", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220101", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20220201", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20220301", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
Reference in New Issue
Block a user