diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 639400868..e2ae36e0d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221118, 20221219); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); @@ -429,6 +429,13 @@ class BigdataForsaleBatch extends BaseBatch{ def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) + def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) + rets1.addAll(rets2) + return rets1; + } + + def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); @@ -586,7 +593,201 @@ class BigdataForsaleBatch extends BaseBatch{ def fitment = parseFitment(seawater.fitment) rowMap.fitment = fitment=="null"?null:fitment + if(seawater.p_date instanceof String){ + seawater.p_date = seawater.p_date.toInteger() + } + if(seawater.p_date<=20200101){ + //seawater.p_date.toString().replace("2019","2020")// + rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31") + rowMap.c_date = rowMap.p_date + rowMap.m_date = rowMap.p_date + }else{ + rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") + rowMap.c_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) + } + rets << rowMap + } + } + + println name +" "+houses.size()+" last:"+rets.size() + return rets; + } + + + def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){ + def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd") + def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd") + def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd") + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + + QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range).must(date_range_mdate) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + if(houses){ + + def ids = new HashSet() + for(def h : rets1){ + ids << h.houseId + } + for (int k=0;k60){ + continue; + } + +// if(!seawater.room){ +// continue; +// } + if(!seawater.room){ + seawater.room = null + } + + if(seawater.room){ + if(seawater.room>=11 || seawater.room<1){ + continue; + } + } + + if(!seawater.hall){ + seawater.hall = null + } + if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){ + continue; + } + if(!seawater.toilet){ + seawater.toilet = null + } + if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){ + continue; + } + + + def floor = parseFloor(seawater.floor_info) + if(seawater.height && floor && floor.isNumber()){ + if(floor.toInteger()>seawater.height){ + floor = null + seawater.height = null + } + + } + + if(!floor){ + seawater.floor = null + } + if(floor && floor.toInteger()<1){ + floor = null + } + + def rowMap = [:]; + if(cat == "rent"){ + rowMap.rent_type = seawater.rent_type + if(seawater.avg_price && seawater.avg_price>1000){ + continue + } + + if(seawater.avg_price && seawater.avg_price<10){ + continue + } + + } + rowMap.houseId = seawater.id + rowMap.city = CcbCommonUtils.getCcbCity(city) + //rowMap.region =seawater.region + //rowMap.address = seawater.address + + + rowMap.area = seawater.area + rowMap.total_price = seawater.total_price + rowMap.avg_price = seawater.avg_price + rowMap.height = seawater.height=="null"?null:seawater.height + if(rowMap.height && rowMap.height<1){ + rowMap.height = null + } + rowMap.floor = floor=="null"?null:floor + rowMap.room = seawater.room=="null"?null:seawater.room + rowMap.hall = seawater.hall=="null"?null:seawater.hall + rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet + if(CcbCommonUtils.rootMap.get(seawater.root)){ + rowMap.root = CcbCommonUtils.rootMap.get(seawater.root) + }else{ + rowMap.root = "99" + } + + def direction = parseDirection(seawater.direction) + rowMap.direction = direction=="null"?null:direction + + def fitment = parseFitment(seawater.fitment) + rowMap.fitment = fitment=="null"?null:fitment if(seawater.p_date instanceof String){ seawater.p_date = seawater.p_date.toInteger() @@ -599,7 +800,7 @@ class BigdataForsaleBatch extends BaseBatch{ }else{ rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") rowMap.c_date = DateUtils.getConverDate(seawater.m_date) - rowMap.m_date = DateUtils.getConverDate(seawater.m_date) + rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date) } rets << rowMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index a551c50b3..b17d97a59 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -65,6 +65,8 @@ class CcbCommonUtils { } }else{ citys = CcbCommonUtils.citys_92 + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") } println "运行城市数量:"+citys.size() return citys; @@ -422,7 +424,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","云山尚城") +// q.put("name","鸣翠蓝湾") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 0722c4464..917ddfc5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -111,8 +111,8 @@ class FzDataResultStat { def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; def datas1 = [] def datas2 = [] - datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] - datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] + datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] + datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] def row_num = 0 for(def row : rows){ if(row_num>0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 7f99588f5..0ecdddeb1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") ){ + if (!href.contains("fang.lianjia")){ cityMap.put(e.text(), href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy index 2e4cfff4a..3b8227240 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy @@ -1,10 +1,12 @@ package com.fangjia.script.gj.test +import com.fangjia.core.util.DataUtil import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import net.sf.json.JSONObject import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -24,17 +26,34 @@ class DongGuangApiTest { } def run(){ - def rets = findEsData(); + // def rets = findEsData(); + def rets = readTxt("D:\\数据\\东莞银行\\东莞银行 (1).csv") println "ES 加载完成" def datas = evaluation(rets) println "重估完成" // ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据") - FileUtils.batchCreateFile(datas, "/home/goojia/东莞重估.txt", [], "utf-8", true); + FileUtils.batchCreateFile(datas, "D:\\数据\\东莞银行\\东莞重估.txt", [], "utf-8", true); println "txt 生成" - exportCsv("/home/goojia/东莞重估.csv", datas) + exportCsv("D:\\数据\\东莞银行\\东莞重估.csv", datas) println "csv 生成" } + def readTxt(def fileName){ + File file = new File(fileName); + def rows = []; + + file.eachLine{ + def row = it.split("#",-1) + if(row && row.size()>=2){ + def s = row[1] + JSONObject result = JSONObject.fromObject(s); + rows << result + } + + } + return rows; + } + def evaluation(rets){ def datas = new CopyOnWriteArrayList() def totalCount = new AtomicLong() @@ -45,7 +64,7 @@ class DongGuangApiTest { void run() { try{ totalCount.addAndGet(1) - def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, dist.size) + def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, 100) if(ret && ret.avgPrice){ def row = [] row << dist.city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 0ceed1225..62e2524e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -41,10 +41,10 @@ class DataStat { def run(){ def source = "365淘房"// - def citys = findCrawlCity(source, 202210); + def citys = findCrawlCity(source, 202211); for(def city : citys){ - def count1 = findCount(source, city, 202210); - def count2 = findCount(source, city, 202211); + def count1 = findCount(source, city, 202211); + def count2 = findCount(source, city, 202212); println city +" "+count1+" "+count2 // println count1 // println "--- ----"