Merge branch 'script_gj' into 'master'
Script gj See merge request !215
This commit is contained in:
+151
-4
@@ -67,15 +67,17 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def index = 0
|
def index = 0
|
||||||
def citys = CcbCommonUtils.citys;
|
def citys = ["苏州","杭州"]//CcbCommonUtils.citys;
|
||||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
index++
|
index++
|
||||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211019, 20211118);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211019, 20211118);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||||
|
//测试通过匹配接口配房源效果
|
||||||
|
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,8 +97,9 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
totalCount.addAndGet(1)
|
totalCount.addAndGet(1)
|
||||||
def commId = rMap.get("_id").toString()
|
def commId = rMap.get("_id").toString()
|
||||||
|
|
||||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
//def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||||
// if(ret && ret.size()<100){
|
def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
|
||||||
|
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
|
||||||
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
||||||
// }
|
// }
|
||||||
if(ret){
|
if(ret){
|
||||||
@@ -247,6 +250,150 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def findHousesByDistId(def city, def commId, def cat, def startDate, def endDate){
|
||||||
|
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||||
|
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||||
|
|
||||||
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", cat)).must(QueryBuilders.termQuery("match_seaweed_id", commId))
|
||||||
|
.must(date_range).must(area_range)
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||||
|
|
||||||
|
if(houses){
|
||||||
|
for (int k=0;k<houses.size();k++) {
|
||||||
|
def map = houses[k];
|
||||||
|
DBObject seawater = new BasicDBObject(map)
|
||||||
|
|
||||||
|
if(!seawater.total_price){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if(!seawater.avg_price){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!seawater.height){
|
||||||
|
seawater.height = null
|
||||||
|
}
|
||||||
|
// if(seawater.height && seawater.height<1){
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
if(seawater.height && seawater.height>60){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// if(!seawater.room){
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
if(!seawater.room){
|
||||||
|
seawater.room = null
|
||||||
|
}
|
||||||
|
|
||||||
|
if(seawater.room){
|
||||||
|
if(seawater.room>=11 || seawater.room<1){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!seawater.hall){
|
||||||
|
seawater.hall = null
|
||||||
|
}
|
||||||
|
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if(!seawater.toilet){
|
||||||
|
seawater.toilet = null
|
||||||
|
}
|
||||||
|
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def floor = parseFloor(seawater.floor_info)
|
||||||
|
if(seawater.height && floor && floor.isNumber()){
|
||||||
|
if(floor.toInteger()>seawater.height){
|
||||||
|
floor = null
|
||||||
|
seawater.height = null
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!floor){
|
||||||
|
seawater.floor = null
|
||||||
|
}
|
||||||
|
if(floor && floor.toInteger()<1){
|
||||||
|
floor = null
|
||||||
|
}
|
||||||
|
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||||
|
if(vallaFlag){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
def rowMap = [:];
|
||||||
|
if(cat == "rent"){
|
||||||
|
rowMap.rent_type = seawater.rent_type
|
||||||
|
if(seawater.avg_price && seawater.avg_price>1000){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if(seawater.avg_price && seawater.avg_price<10){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
rowMap.houseId = seawater.id
|
||||||
|
rowMap.city = city
|
||||||
|
//rowMap.region =seawater.region
|
||||||
|
//rowMap.address = seawater.address
|
||||||
|
|
||||||
|
|
||||||
|
rowMap.area = seawater.area
|
||||||
|
rowMap.total_price = seawater.total_price
|
||||||
|
rowMap.avg_price = seawater.avg_price
|
||||||
|
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||||
|
if(rowMap.height && rowMap.height<1){
|
||||||
|
rowMap.height = null
|
||||||
|
}
|
||||||
|
rowMap.floor = floor=="null"?null:floor
|
||||||
|
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||||
|
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||||
|
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||||
|
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||||
|
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||||
|
}else{
|
||||||
|
rowMap.root = "99"
|
||||||
|
}
|
||||||
|
|
||||||
|
def direction = parseDirection(seawater.direction)
|
||||||
|
rowMap.direction = direction=="null"?null:direction
|
||||||
|
|
||||||
|
def fitment = parseFitment(seawater.fitment)
|
||||||
|
rowMap.fitment = fitment=="null"?null:fitment
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
if(seawater.p_date instanceof String){
|
||||||
|
seawater.p_date = seawater.p_date.toInteger()
|
||||||
|
}
|
||||||
|
if(seawater.p_date<=20200101){
|
||||||
|
//seawater.p_date.toString().replace("2019","2020")//
|
||||||
|
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||||
|
rowMap.c_date = rowMap.p_date
|
||||||
|
rowMap.m_date = rowMap.p_date
|
||||||
|
}else{
|
||||||
|
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||||
|
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||||
|
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||||
|
}
|
||||||
|
|
||||||
|
rets << rowMap
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println name +" "+houses.size()+" last:"+rets.size()
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||||
|
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
|||||||
//福州第三方均价统计,发送钉钉消息
|
//福州第三方均价统计,发送钉钉消息
|
||||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202110.csv")
|
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202111.csv")
|
||||||
def count0 = 0
|
def count0 = 0
|
||||||
def count1 = 0
|
def count1 = 0
|
||||||
def count2 = 0
|
def count2 = 0
|
||||||
|
|||||||
+4
-4
@@ -40,11 +40,11 @@ class DataStat {
|
|||||||
|
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def source = "58同城"//
|
def source = "我爱我家"//
|
||||||
def citys = findCrawlCity(source, 202107);
|
def citys = findCrawlCity(source, 202110);
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def count1 = findCount(source, city, 202107);
|
def count1 = findCount(source, city, 202110);
|
||||||
def count2 = findCount(source, city, 202110);
|
def count2 = findCount(source, city, 202111);
|
||||||
println city +" "+count1+" "+count2
|
println city +" "+count1+" "+count2
|
||||||
// println count1
|
// println count1
|
||||||
// println "--- ----"
|
// println "--- ----"
|
||||||
|
|||||||
+1
-1
@@ -44,7 +44,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
|||||||
@Override
|
@Override
|
||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
println SHORT_DATE
|
println SHORT_DATE
|
||||||
def citys = CcbCommonUtils.citys;//CommonDatautils.findAllCity();
|
def citys = CommonDatautils.findAllCity();
|
||||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||||
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
|
//def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD("2021-07-15"));
|
||||||
def cityIndex = 0;
|
def cityIndex = 0;
|
||||||
|
|||||||
Reference in New Issue
Block a user