+17
-18
@@ -58,17 +58,17 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
|
||||
// println "curDate:"+curDate
|
||||
// for(def city : citys){
|
||||
//// if(!ccbCitys.contains(city)){
|
||||
// addDealPriceMatch(city);
|
||||
//// }
|
||||
// }
|
||||
println "curDate:"+curDate
|
||||
for(def city : citys){
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
@@ -86,17 +86,16 @@ class BigdataDealBatch extends BaseBatch{
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
// def trades = getCityData(city, statMap, ppeDists)
|
||||
// if(trades){
|
||||
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
|
||||
// rows.addAll(rets)
|
||||
// println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
//
|
||||
// def cityDatas = []
|
||||
// cityDatas << headers
|
||||
// cityDatas.addAll(rows)
|
||||
// }
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -105,7 +104,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true);
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成 " + rows.size()
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -75,7 +75,7 @@ class FjwPricePkCcbPrice {
|
||||
def cityId = d['D']
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202211)
|
||||
def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209)
|
||||
if(map){
|
||||
d['O'] = map.avg_price
|
||||
d['P'] = map.calc_type
|
||||
|
||||
+3
-3
@@ -38,16 +38,16 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
// CcbCommonUtils.deleteGuidedPrice(curDate)
|
||||
CcbCommonUtils.deleteGuidedPrice(curDate)
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-25", "2022-12-19")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+59
-35
@@ -47,41 +47,41 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
|
||||
|
||||
|
||||
|
||||
def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def rets = []
|
||||
for(def c : citys){
|
||||
def citycode = cityCodeDataMap.get(c)
|
||||
def datas = findAllData(sqlInstance236, citycode)
|
||||
def totalCount = datas.size()
|
||||
def landareaCount = 0;
|
||||
def buildareaCount = 0
|
||||
def greeningrateCount = 0
|
||||
def plotratioCount = 0
|
||||
for(def m : datas){
|
||||
if(m.landarea){
|
||||
landareaCount ++
|
||||
}
|
||||
if(m.buildarea){
|
||||
buildareaCount++
|
||||
}
|
||||
if(m.greeningrate){
|
||||
greeningrateCount++
|
||||
}
|
||||
if(m.plotratio){
|
||||
plotratioCount++
|
||||
}
|
||||
}
|
||||
|
||||
def landAreaPre = landareaCount*100/totalCount
|
||||
def buildAreaPre = buildareaCount*100/totalCount
|
||||
def greeningratePre = greeningrateCount*100/totalCount
|
||||
def plotratioPre = plotratioCount*100/totalCount
|
||||
rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre]
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率")
|
||||
//def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
|
||||
//
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def rets = []
|
||||
//for(def c : citys){
|
||||
// def citycode = cityCodeDataMap.get(c)
|
||||
// def datas = findAllData(sqlInstance236, citycode)
|
||||
// def totalCount = datas.size()
|
||||
// def landareaCount = 0;
|
||||
// def buildareaCount = 0
|
||||
// def greeningrateCount = 0
|
||||
// def plotratioCount = 0
|
||||
// for(def m : datas){
|
||||
// if(m.landarea){
|
||||
// landareaCount ++
|
||||
// }
|
||||
// if(m.buildarea){
|
||||
// buildareaCount++
|
||||
// }
|
||||
// if(m.greeningrate){
|
||||
// greeningrateCount++
|
||||
// }
|
||||
// if(m.plotratio){
|
||||
// plotratioCount++
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// def landAreaPre = landareaCount*100/totalCount
|
||||
// def buildAreaPre = buildareaCount*100/totalCount
|
||||
// def greeningratePre = greeningrateCount*100/totalCount
|
||||
// def plotratioPre = plotratioCount*100/totalCount
|
||||
// rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre]
|
||||
//}
|
||||
//
|
||||
//ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率")
|
||||
|
||||
def findAllData(sqlInstance236, citycode){
|
||||
def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'"
|
||||
@@ -1149,6 +1149,30 @@ def findThirdCommCount(cityid, sqlInstance){
|
||||
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\二期\\付.xlsx",{d->
|
||||
def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district")
|
||||
if(obj){
|
||||
def count = findGuidedPrice(sqlInstance236, obj._id.toString())
|
||||
if(count){
|
||||
d['M'] = "存在"
|
||||
}
|
||||
d['N'] = obj.name_cat
|
||||
d['O'] = obj.estate_type2
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
def findGuidedPrice(sqlInstance, id){
|
||||
def count = 0
|
||||
def sql = "select count(*) as count from guided_price_gj where commid='${id}'"
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+7
-7
@@ -50,7 +50,7 @@ class TradeStatLog {
|
||||
// task.deleteDealData2()
|
||||
// task.statTableCount()
|
||||
// task.filterTrade()
|
||||
task.updateDealPriceType()
|
||||
// task.updateDealPriceType()
|
||||
|
||||
}
|
||||
|
||||
@@ -90,15 +90,15 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_92;
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2022-08-01"
|
||||
def date = "2022-09-01"
|
||||
def endDate = "2022-10-01"
|
||||
def newDate = "2022-11-01"
|
||||
def startDate = "2022-09-01"
|
||||
def date = "2022-10-01"
|
||||
def endDate = "2022-11-01"
|
||||
def newDate = "2022-12-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
@@ -5,9 +5,36 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
|
||||
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
|
||||
def rets = []
|
||||
for(def d : datas){
|
||||
def arrs = d.split("@")
|
||||
rets << ["commId":arrs[0], "avg_price":arrs[4]]
|
||||
}
|
||||
|
||||
for(def r : rets){
|
||||
if(r.avg_price){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("date", 202210)
|
||||
udata.put("avg", r.avg_price.toInteger())
|
||||
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
|
||||
}else{
|
||||
println r.commId +" 不存在"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
//List<String> list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv"))
|
||||
//println "过滤前:"+list.size()+ " "+new Date().toLocaleString()
|
||||
//def ret = []
|
||||
@@ -48,29 +75,29 @@ import org.bson.types.ObjectId
|
||||
// return location;
|
||||
//}
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
map1.put(key, row[5])
|
||||
}
|
||||
|
||||
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
|
||||
def map2 = [:]
|
||||
for(def s : list2) {
|
||||
def row = s.split("@\\|@")
|
||||
def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
map2.put(key, row[5])
|
||||
}
|
||||
def rows = []
|
||||
map1.each {k,v->
|
||||
def row = []
|
||||
def keys = k.split("##")
|
||||
def lastPrice = map2.get(k)
|
||||
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
|
||||
//List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
|
||||
//def map1 = [:]
|
||||
//for(def s : list1){
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2]+"##"+row[3]+"##"+row[4]
|
||||
// map1.put(key, row[5])
|
||||
//}
|
||||
//
|
||||
//List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
|
||||
//def map2 = [:]
|
||||
//for(def s : list2) {
|
||||
// def row = s.split("@\\|@")
|
||||
// def key = row[2] + "##" + row[3] + "##" + row[4]
|
||||
// map2.put(key, row[5])
|
||||
//}
|
||||
//def rows = []
|
||||
//map1.each {k,v->
|
||||
// def row = []
|
||||
// def keys = k.split("##")
|
||||
// def lastPrice = map2.get(k)
|
||||
// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
|
||||
// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
|
||||
// if(obj){
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202210.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+343
-51
File diff suppressed because one or more lines are too long
+1
-1
@@ -52,7 +52,7 @@ class YouDaDataParser {
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2022年12月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+1
-1
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia")){
|
||||
if (!href.contains("fang.lianjia") ){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+29
@@ -247,6 +247,35 @@ class GaoDeUtils {
|
||||
return Collections.emptyList()
|
||||
}
|
||||
|
||||
|
||||
static List<AmapPoi> getAmapById2(id){
|
||||
def params = [:]
|
||||
params.id = id
|
||||
params.output = 'json'
|
||||
params.key = "66e6771767ab293ace441553c36621c1"
|
||||
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params);
|
||||
if (json == null) {
|
||||
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params);
|
||||
}
|
||||
if (json && json.info && json.info == "OK") {
|
||||
try {
|
||||
List<AmapPoi> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
|
||||
return ret
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
}
|
||||
} else if (json && json.infocode && json.infocode == "10044") {
|
||||
System.exit(1)
|
||||
} else if (json && json.infocode && json.infocode == "10004") {
|
||||
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
|
||||
Thread.sleep(60000)
|
||||
} else if (json && json.infocode && json.infocode == "10017") {
|
||||
// 所请求的资源不可用
|
||||
System.exit(1)
|
||||
}
|
||||
return Collections.emptyList()
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取详情
|
||||
*/
|
||||
|
||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
||||
|
||||
|
||||
def run(){
|
||||
def source = "链家"//
|
||||
def citys = findCrawlCity(source, 202209);
|
||||
def source = "365淘房"//
|
||||
def citys = findCrawlCity(source, 202210);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202209);
|
||||
def count2 = findCount(source, city, 202210);
|
||||
def count1 = findCount(source, city, 202210);
|
||||
def count2 = findCount(source, city, 202211);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
@@ -65,6 +65,8 @@ class MongoUtils {
|
||||
if (domainUrl == MongoDB_Domain_mongo) {
|
||||
port == 28017;
|
||||
}
|
||||
|
||||
|
||||
if ((domainUrl == "192.168.0.136") && (port == 27017 || port == 28017)) {
|
||||
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user