Merge branch 'script_gj' into 'master'

成交修改

See merge request !499
This commit is contained in:
2022-12-21 18:05:20 +08:00
13 changed files with 518 additions and 145 deletions
@@ -58,17 +58,17 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
// println "curDate:"+curDate
// for(def city : citys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
println "curDate:"+curDate
for(def city : citys){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
// }
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -86,17 +86,16 @@ class BigdataDealBatch extends BaseBatch{
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
// def trades = getCityData(city, statMap, ppeDists)
// if(trades){
// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
// rows.addAll(rets)
// println "【${index}】"+city +" 成交共:"+rets.size()
//
// def cityDatas = []
// cityDatas << headers
// cityDatas.addAll(rows)
// }
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
@@ -105,7 +104,7 @@ class BigdataDealBatch extends BaseBatch{
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true);
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
}
@@ -75,7 +75,7 @@ class FjwPricePkCcbPrice {
def cityId = d['D']
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202211)
def map = getCommunityBasePriceGjByCommid(cityId, dist._id.toString(), 202209)
if(map){
d['O'] = map.avg_price
d['P'] = map.calc_type
@@ -38,16 +38,16 @@ class GuiDingPriceBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = ["上海"]//CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
// CcbCommonUtils.deleteGuidedPrice(curDate)
CcbCommonUtils.deleteGuidedPrice(curDate)
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-24", "2022-11-25")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-11-25", "2022-12-19")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -47,41 +47,41 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business
def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def rets = []
for(def c : citys){
def citycode = cityCodeDataMap.get(c)
def datas = findAllData(sqlInstance236, citycode)
def totalCount = datas.size()
def landareaCount = 0;
def buildareaCount = 0
def greeningrateCount = 0
def plotratioCount = 0
for(def m : datas){
if(m.landarea){
landareaCount ++
}
if(m.buildarea){
buildareaCount++
}
if(m.greeningrate){
greeningrateCount++
}
if(m.plotratio){
plotratioCount++
}
}
def landAreaPre = landareaCount*100/totalCount
def buildAreaPre = buildareaCount*100/totalCount
def greeningratePre = greeningrateCount*100/totalCount
def plotratioPre = plotratioCount*100/totalCount
rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre]
}
ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率")
//def citys = ['苏州','杭州','宁波','温州','嘉兴','湖州','绍兴','金华','衢州','舟山','台州','丽水']
//
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def rets = []
//for(def c : citys){
// def citycode = cityCodeDataMap.get(c)
// def datas = findAllData(sqlInstance236, citycode)
// def totalCount = datas.size()
// def landareaCount = 0;
// def buildareaCount = 0
// def greeningrateCount = 0
// def plotratioCount = 0
// for(def m : datas){
// if(m.landarea){
// landareaCount ++
// }
// if(m.buildarea){
// buildareaCount++
// }
// if(m.greeningrate){
// greeningrateCount++
// }
// if(m.plotratio){
// plotratioCount++
// }
// }
//
// def landAreaPre = landareaCount*100/totalCount
// def buildAreaPre = buildareaCount*100/totalCount
// def greeningratePre = greeningrateCount*100/totalCount
// def plotratioPre = plotratioCount*100/totalCount
// rets << [c, totalCount, landAreaPre, buildAreaPre, greeningratePre, plotratioPre]
//}
//
//ExcelUtils.write("D:\\数据\\建行\\测试\\12城覆盖率统计.xlsx",rets, "覆盖率")
def findAllData(sqlInstance236, citycode){
def sql = "select landarea,buildarea,greeningrate,plotratio from bigdata_community_fjw where cityid='${citycode}'"
@@ -1149,6 +1149,30 @@ def findThirdCommCount(cityid, sqlInstance){
ExcelUtils.modify("D:\\数据\\建行\\二期\\付.xlsx",{d->
def obj = CommonDatautils.findDistrict(d['D'],d['L'], d['K'], "district")
if(obj){
def count = findGuidedPrice(sqlInstance236, obj._id.toString())
if(count){
d['M'] = "存在"
}
d['N'] = obj.name_cat
d['O'] = obj.estate_type2
}
})
def findGuidedPrice(sqlInstance, id){
def count = 0
def sql = "select count(*) as count from guided_price_gj where commid='${id}'"
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
@@ -50,7 +50,7 @@ class TradeStatLog {
// task.deleteDealData2()
// task.statTableCount()
// task.filterTrade()
task.updateDealPriceType()
// task.updateDealPriceType()
}
@@ -90,15 +90,15 @@ class TradeStatLog {
}
def run(){
def citys = CcbCommonUtils.citys_92;
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","8月总量","9月总量","10月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","9月总量","10月总量","11月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2022-08-01"
def date = "2022-09-01"
def endDate = "2022-10-01"
def newDate = "2022-11-01"
def startDate = "2022-09-01"
def date = "2022-10-01"
def endDate = "2022-11-01"
def newDate = "2022-12-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -5,9 +5,36 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
def datas = FileUtils.readLines("D:\\数据\\其它\\FJW_COMM_20221015.csv")
def rets = []
for(def d : datas){
def arrs = d.split("@")
rets << ["commId":arrs[0], "avg_price":arrs[4]]
}
for(def r : rets){
if(r.avg_price){
def udata = new BasicDBObject()
udata.put("date", 202210)
udata.put("avg", r.avg_price.toInteger())
DISTRICT_SELL_STAT.update(new BasicDBObject("commid", r.commId) , new BasicDBObject('$set', data),false,false)
}else{
println r.commId +" 不存在"
}
}
//List<String> list = FileUtils.readLines(new File("C:\\Users\\gaojuan\\Desktop\\FJW_HOUSE_201911.csv"))
//println "过滤前:"+list.size()+ " "+new Date().toLocaleString()
//def ret = []
@@ -48,29 +75,29 @@ import org.bson.types.ObjectId
// return location;
//}
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
map1.put(key, row[5])
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row[5])
}
def rows = []
map1.each {k,v->
def row = []
def keys = k.split("##")
def lastPrice = map2.get(k)
rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
}
ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
//List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv"))
//def map1 = [:]
//for(def s : list1){
// def row = s.split("@\\|@")
// def key = row[2]+"##"+row[3]+"##"+row[4]
// map1.put(key, row[5])
//}
//
//List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
//def map2 = [:]
//for(def s : list2) {
// def row = s.split("@\\|@")
// def key = row[2] + "##" + row[3] + "##" + row[4]
// map2.put(key, row[5])
//}
//def rows = []
//map1.each {k,v->
// def row = []
// def keys = k.split("##")
// def lastPrice = map2.get(k)
// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()]
//}
//ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d->
// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd"))
// if(obj){
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202210.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202211.csv")
def count0 = 0
def count1 = 0
def count2 = 0
File diff suppressed because one or more lines are too long
@@ -52,7 +52,7 @@ class YouDaDataParser {
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年11月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2022年12月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia")){
if (!href.contains("fang.lianjia") ){
cityMap.put(e.text(), href)
}
}
@@ -247,6 +247,35 @@ class GaoDeUtils {
return Collections.emptyList()
}
static List<AmapPoi> getAmapById2(id){
def params = [:]
params.id = id
params.output = 'json'
params.key = "66e6771767ab293ace441553c36621c1"
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v5/place/detail", params);
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoi> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
System.exit(1)
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
}
return Collections.emptyList()
}
/**
* 获取详情
*/
@@ -40,11 +40,11 @@ class DataStat {
def run(){
def source = "链家"//
def citys = findCrawlCity(source, 202209);
def source = "365淘房"//
def citys = findCrawlCity(source, 202210);
for(def city : citys){
def count1 = findCount(source, city, 202209);
def count2 = findCount(source, city, 202210);
def count1 = findCount(source, city, 202210);
def count2 = findCount(source, city, 202211);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
@@ -65,6 +65,8 @@ class MongoUtils {
if (domainUrl == MongoDB_Domain_mongo) {
port == 28017;
}
if ((domainUrl == "192.168.0.136") && (port == 27017 || port == 28017)) {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
}