Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+1
@@ -1837,6 +1837,7 @@ public class AddressController {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity());
|
||||
params.put("address", model.getAddress());
|
||||
params.put("showCode", model.getShowCode());
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
|
||||
@@ -22,6 +22,8 @@ public class AddesssMatchModel extends FangjiaApiModel {
|
||||
private String address_1;
|
||||
private String address_2;
|
||||
|
||||
private String showCode;
|
||||
|
||||
public String getAddress_1() {
|
||||
return address_1;
|
||||
}
|
||||
@@ -134,4 +136,12 @@ public class AddesssMatchModel extends FangjiaApiModel {
|
||||
public void setDebugName(String debugName) {
|
||||
this.debugName = debugName;
|
||||
}
|
||||
|
||||
public String getShowCode() {
|
||||
return showCode;
|
||||
}
|
||||
|
||||
public void setShowCode(String showCode) {
|
||||
this.showCode = showCode;
|
||||
}
|
||||
}
|
||||
|
||||
+37
-8
@@ -9,10 +9,41 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
//没有成交时间,随机生成
|
||||
//for(def d : datas){
|
||||
// def map = new BasicDBObject()
|
||||
// map.city = "深圳"
|
||||
// map.district_name = d['D']
|
||||
// map.region = d['B']
|
||||
// def roomStr = d['E']
|
||||
// if(roomStr){
|
||||
// def arrs = roomStr.split("-")
|
||||
// try{
|
||||
// map.room = arrs[0].toString().toInteger()
|
||||
// }catch(Exception e){
|
||||
// println "room error"
|
||||
// }
|
||||
// map.hall = arrs[1].toString().toInteger()
|
||||
// map.toilet = arrs[3].toString().toInteger()
|
||||
// }
|
||||
//
|
||||
// map.area = d['F'].toString().toDouble()
|
||||
// map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
// map.avg_price = (int)(d['H'].toString().toDouble()*10000)
|
||||
// def trade_date = DateUtils.getRandomDate("2023-04-01", "2023-04-30")
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd")
|
||||
// map.source = "深房家族"
|
||||
// map.create_date = new Date()
|
||||
// map.m_date = new Date()
|
||||
// DEAL_PRICE_COLL.save(map)
|
||||
//}
|
||||
|
||||
|
||||
|
||||
for(def d : datas){
|
||||
def map = new BasicDBObject()
|
||||
map.city = "深圳"
|
||||
@@ -20,17 +51,15 @@ for(def d : datas){
|
||||
map.region = d['B']
|
||||
def roomStr = d['E']
|
||||
if(roomStr){
|
||||
def arrs = roomStr.split("-")
|
||||
map.room = arrs[0].toString().toInteger()
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
map.area = d['F'].toString().toDouble()
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble()*10000)
|
||||
def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd")
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
def trade_date = d['J']
|
||||
map.trade_date = trade_date
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
map.m_date = new Date()
|
||||
|
||||
+6
@@ -56,6 +56,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false);
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
delPgData()
|
||||
saveToPgSql(rets)
|
||||
|
||||
}
|
||||
@@ -110,6 +111,11 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
|
||||
return rets;
|
||||
}
|
||||
|
||||
def delPgData(){
|
||||
sqlInstance.execute("delete from base.t0831_bigdata_commbuildtype_gj_h")
|
||||
println "base.t0831_bigdata_commbuildtype_gj_h 数据已删除"
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
def sqls = []
|
||||
|
||||
+6
-6
@@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
|
||||
def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用
|
||||
def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
@@ -78,10 +78,10 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
|
||||
// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
// delPgData()
|
||||
// saveToPgSql2(datas)
|
||||
@@ -259,7 +259,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
sqlInstance.withBatch(1, sql){ ps->
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
|
||||
+42
-43
@@ -65,52 +65,51 @@ class BigdataDealBatch extends BaseBatch{
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
|
||||
println "curDate:"+curDate
|
||||
for(def city : citys){
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def index = 0
|
||||
// for(def city : citys){
|
||||
// index ++
|
||||
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
// def ppeDists = [:]
|
||||
//
|
||||
// def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
// if(city=="上海"){
|
||||
// def youDataTrade = getYouDaData(statMap)
|
||||
// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
// rows.addAll(youdaDatas)
|
||||
// println city +" "+youdaDatas.size()
|
||||
// }
|
||||
//
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
//
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
// }
|
||||
//
|
||||
// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
// if(CommonDatautils.isWindow()){
|
||||
// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
// }
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
//
|
||||
// saveToPgSql(rows)
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
|
||||
+3
-2
@@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230318, 20230419, houseIdsfilter);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
}
|
||||
@@ -203,7 +203,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
}
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
@@ -1127,6 +1127,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
|
||||
+2
-2
@@ -430,8 +430,8 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉
|
||||
// q.put("name","锦南浦三小区")
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","金地玺湾")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-26")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
saveToPgSql(rows)
|
||||
|
||||
//更新静态数据表
|
||||
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
|
||||
|
||||
}
|
||||
|
||||
|
||||
+15
-3
@@ -1,5 +1,8 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -12,15 +15,24 @@ class TestRepeat {
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
private DSCollection seabird = new DataStore("192.168.0.100", 37017).getCollection("fangjia.seabird")
|
||||
|
||||
static main(args) {
|
||||
def task = new TestRepeat()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat", "sell")
|
||||
q.put("city", "ÏåÑô")
|
||||
def count = seabird.count(q)
|
||||
println count
|
||||
// for(def d : dbcursor){
|
||||
// println d.name
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202303//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202301
|
||||
def endDate = 202303
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304//
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+5
-5
@@ -92,13 +92,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-01-01"
|
||||
def date = "2023-02-01"
|
||||
def endDate = "2023-03-01"
|
||||
def newDate = "2023-04-01"
|
||||
def startDate = "2023-02-01"
|
||||
def date = "2023-03-01"
|
||||
def endDate = "2023-04-01"
|
||||
def newDate = "2023-05-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+22
-1
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
@@ -11,7 +13,26 @@ class UpdateData {
|
||||
|
||||
static main(args) {
|
||||
def task = new UpdateData();
|
||||
task.run()
|
||||
// task.run()
|
||||
task.updateBlock()
|
||||
}
|
||||
|
||||
def updateBlock(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def c : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "block","name"])
|
||||
for(def d : dists){
|
||||
if(d.block){
|
||||
def newBlock = FiledStrand.processBlock(d.city, d.region, d.block)
|
||||
if(newBlock!= d.block){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("block", newBlock)
|
||||
println d.city+" "+d.block +" "+newBlock
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+1040
File diff suppressed because it is too large
Load Diff
+7
-9
@@ -39,6 +39,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
|
||||
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
|
||||
def new_dists = []//["350604","350605","410307","410308","450181"]
|
||||
|
||||
@@ -66,9 +67,6 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
exportBigdataCityPrice(queryDate)
|
||||
exportBigdataDistPrice(queryDate)
|
||||
exportBigdataCommPrice(queryDate)
|
||||
|
||||
|
||||
|
||||
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
|
||||
// exportCommunityBasePrice(lastMonth)
|
||||
// exportCommunityPdescribeGj(lastMonth)
|
||||
@@ -81,15 +79,15 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
if(curDate.toString().endsWith("28")){
|
||||
exportPgBigdataSupplierStatistics(lastMonth)
|
||||
exportPgBigdataSupplierStatistics(curMonth)
|
||||
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
|
||||
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
// exportThirdPartyPrice2(thirdPartyMonth, new_dists)
|
||||
exportPgThirdPartyPrice2(thirdPartyMonth, new_dists)
|
||||
exportPgCommunityBasePrice(lastMonth)
|
||||
exportPgCommunityPdescribeGj(lastMonth)
|
||||
exportPgBigdataModelingBase(lastMonth)
|
||||
exportPgGuideStatistics(lastMonth)
|
||||
exportPgCommunityBasePrice(curMonth)
|
||||
exportPgCommunityPdescribeGj(curMonth)
|
||||
exportPgBigdataModelingBase(curMonth)
|
||||
exportPgGuideStatistics(curMonth)
|
||||
}
|
||||
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
|
||||
+13
-16
@@ -69,27 +69,24 @@ class CityInfoAnaBatch extends BaseBatch {
|
||||
alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator',
|
||||
township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart'
|
||||
]
|
||||
//out_pics : 'outPics', pics : 'pics',household_count :'householdCount',build_structure :'buildStructure', facilities : 'facilities',characteristic:'characteristic',licence:'licence',sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany',
|
||||
//conditioner_time:'conditionerTime',network_way:'networkWay',fitment_level:'fitmentLevel',lighting_ventilation:'lightingVentilation',
|
||||
public static final def OFFICE_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate',
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate',
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent',
|
||||
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure',
|
||||
facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo',licence:'licence',
|
||||
sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany',estate_charge:'estateCharge',
|
||||
floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife',
|
||||
conditioner_time:'conditionerTime',elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount',
|
||||
goods_elevator_count:'goodsElevatorCount',network_way:'networkWay',fitment_level:'fitmentLevel',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea',
|
||||
is_foreign:'foreign',is_register:'register',lighting_ventilation:'lightingVentilation',lobby_height:'lobbyHeight'
|
||||
car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',
|
||||
car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias',lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo',
|
||||
estate_charge:'estateCharge',floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife',
|
||||
elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount',goods_elevator_count:'goodsElevatorCount',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea',
|
||||
is_foreign:'foreign',is_register:'register',lobby_height:'lobbyHeight',township:'township',divisible :'divisible',land_floor:'landFloor',underground_floor:'undergroundFloor'
|
||||
]
|
||||
|
||||
// out_pics : 'outPics', pics : 'pics',
|
||||
public static final def SHOP_FIELDS_MAP = [
|
||||
name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate',
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',
|
||||
rent_type :'rentType',lease_term :'leaseTerm',car_site :'carSite',estate_charge :'estateCharge',divisible :'divisible',floor :'floor',
|
||||
finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate',
|
||||
bay_area :'bayArea', property_life :'propertyLife',unified_manage :'unifiedManage',licence :'licence',business_status :'businessStatus',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics'
|
||||
|
||||
name :'name',region :'region',block :'block',address :'address', alias:'alias',addr_alias:'addrAlias',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate',
|
||||
all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',car_site :'carSite',estate_charge :'estateCharge',floor :'floor',
|
||||
finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate',bay_area :'bayArea', property_life :'propertyLife',business_status :'businessStatus',parking_fee:'parkingFee',
|
||||
lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2',township:'township',loop_line_info :'loopLineInfo',land_floor:'landFloor',underground_floor:'undergroundFloor',real_use_percent:'realUsePercent'
|
||||
]
|
||||
|
||||
public static CITY_GRADE_MAP = [1000: "一线" , 1500 : "新一线" , 2000 : "二线" , 3000:"三线" ,4000 : "四线" ,5000 : "五线"]
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.BlockData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
@@ -337,11 +339,12 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processBlock(def block){
|
||||
static def processBlock(city, region, def block){
|
||||
if(block){
|
||||
if(block.contains("ÎÞ") || block.toString().length()>10){
|
||||
return null
|
||||
}
|
||||
// block = BlockData.formatBlock(city, region, block);
|
||||
return block
|
||||
}
|
||||
return null
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-03-01"
|
||||
static def statDate = "2023-04-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
@@ -83,7 +83,7 @@ class FileUtils {
|
||||
def f = new File(file)
|
||||
def ret = [];
|
||||
if(f.exists()){
|
||||
def lines = org.apache.commons.io.FileUtils.readLines(f)
|
||||
def lines = org.apache.commons.io.FileUtils.readLines(f,"utf-8")
|
||||
for(line in lines) {
|
||||
ret << line;
|
||||
}
|
||||
|
||||
+2
-2
@@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
def citys = ["ÕÄÖÝ"]//CommonDatautils.findAllCity().reverse()
|
||||
|
||||
def analysRets = []
|
||||
|
||||
@@ -279,7 +279,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
newVal = FiledStrand.processAddress(esVal, dist.city)
|
||||
}
|
||||
if(seaweedfield =="block"){
|
||||
newVal = FiledStrand.processBlock(esVal)
|
||||
newVal = FiledStrand.processBlock(dist.city, dist.region, esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_type"){
|
||||
|
||||
+4
-2
@@ -26,8 +26,10 @@ class SeaweedDataMatchProcess {
|
||||
rMap.region = obj.region
|
||||
rMap.alias = obj.alias
|
||||
rMap.name = obj.name
|
||||
rMap.lat = obj.lat/10000000000.0
|
||||
rMap.lng = obj.lng/10000000000.0
|
||||
if(obj.lat){
|
||||
rMap.lat = obj.lat/10000000000.0
|
||||
rMap.lng = obj.lng/10000000000.0
|
||||
}
|
||||
rMap.id = obj._id.toString()
|
||||
rMap.address = obj.address
|
||||
}
|
||||
|
||||
+28
-33
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
@@ -34,6 +35,7 @@ public class BeiKeCrawlByChrome {
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
|
||||
//广州,佛山 重庆
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
task.run()
|
||||
@@ -49,27 +51,8 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def citys = getAllCityUrls(driver)
|
||||
// citys = citys.reverse()
|
||||
// def index = 0
|
||||
// for(def c : citys){
|
||||
// index++
|
||||
// if(c.city=="合肥"){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// citys = citys[index..citys.size()-1]
|
||||
def start = 0
|
||||
// for(def i=0;i<citys.size();i++){
|
||||
// def m = citys[i]
|
||||
// println i+" "+ m.city
|
||||
// if(m.city=="天津"){
|
||||
// start = i
|
||||
// break;
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
driver = getLogin(driver)
|
||||
|
||||
driver = getLogin(driver, "https://sh.ke.com/")
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
@@ -113,7 +96,7 @@ public class BeiKeCrawlByChrome {
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && (s.contains("无法访问此网站") || s.contains("<title>登录</title>"))){
|
||||
driver = getLogin(driver)
|
||||
driver = getLogin(driver, url)
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
@@ -127,8 +110,14 @@ public class BeiKeCrawlByChrome {
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\贝壳已抓城市.txt")
|
||||
|
||||
for(def m : citys){
|
||||
def city = m[0]
|
||||
if(runedCitys.contains(city)){
|
||||
println city +" 已运行"
|
||||
continue
|
||||
}
|
||||
def url = m[1]
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
@@ -145,10 +134,11 @@ public class BeiKeCrawlByChrome {
|
||||
println url +" 详情链接:"+detailUrls.size()
|
||||
}
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true);
|
||||
// def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
// println m.city +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -434,27 +424,32 @@ public class BeiKeCrawlByChrome {
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
def getLogin(driver, url){
|
||||
// WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4)
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
try{
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(10000)
|
||||
}
|
||||
return driver
|
||||
}
|
||||
}
|
||||
|
||||
+14
-16
@@ -118,6 +118,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
@@ -523,24 +524,21 @@ public class LianjiaCrawlByChrome {
|
||||
Thread.sleep(10000)
|
||||
try{
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username"))));
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(60000)
|
||||
getLogin(driver, url)
|
||||
Thread.sleep(10000)
|
||||
}
|
||||
|
||||
WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框
|
||||
|
||||
WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(20000)
|
||||
return driver
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user