建行房源同步修改

This commit is contained in:
2023-01-15 14:29:59 +08:00
parent 32d64dbd67
commit 3a8dbefddc
6 changed files with 235 additions and 13 deletions
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city)) // def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221118, 20221219); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
@@ -429,6 +429,13 @@ class BigdataForsaleBatch extends BaseBatch{
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
return rets1;
}
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -586,7 +593,201 @@ class BigdataForsaleBatch extends BaseBatch{
def fitment = parseFitment(seawater.fitment) def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if(houses){
def ids = new HashSet()
for(def h : rets1){
ids << h.houseId
}
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(ids && ids.contains(seawater.id)){
continue;
}
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){ if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger() seawater.p_date = seawater.p_date.toInteger()
@@ -599,7 +800,7 @@ class BigdataForsaleBatch extends BaseBatch{
}else{ }else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss") rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date) rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date) rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
} }
rets << rowMap rets << rowMap
@@ -65,6 +65,8 @@ class CcbCommonUtils {
} }
}else{ }else{
citys = CcbCommonUtils.citys_92 citys = CcbCommonUtils.citys_92
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
} }
println "运行城市数量:"+citys.size() println "运行城市数量:"+citys.size()
return citys; return citys;
@@ -422,7 +424,7 @@ class CcbCommonUtils {
q.put("visible",0) q.put("visible",0)
q.put("cat", cat); q.put("cat", cat);
q.put("city", city); q.put("city", city);
// q.put("name","云山尚城") // q.put("name","鸣翠蓝湾")
// q.put("name_cat", 0); // q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -111,8 +111,8 @@ class FzDataResultStat {
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
def datas1 = [] def datas1 = []
def datas2 = [] def datas2 = []
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
def row_num = 0 def row_num = 0
for(def row : rows){ for(def row : rows){
if(row_num>0){ if(row_num>0){
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles) { for(def e : cityEles) {
def href = e.attr("href") def href = e.attr("href")
if (!href.contains("fang.lianjia") ){ if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href) cityMap.put(e.text(), href)
} }
} }
@@ -1,10 +1,12 @@
package com.fangjia.script.gj.test package com.fangjia.script.gj.test
import com.fangjia.core.util.DataUtil
import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient import com.fangjia.script.util.OpenApiClient
import net.sf.json.JSONObject
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
@@ -24,17 +26,34 @@ class DongGuangApiTest {
} }
def run(){ def run(){
def rets = findEsData(); // def rets = findEsData();
def rets = readTxt("D:\\数据\\东莞银行\\东莞银行 (1).csv")
println "ES 加载完成" println "ES 加载完成"
def datas = evaluation(rets) def datas = evaluation(rets)
println "重估完成" println "重估完成"
// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据") // ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据")
FileUtils.batchCreateFile(datas, "/home/goojia/东莞重估.txt", [], "utf-8", true); FileUtils.batchCreateFile(datas, "D:\\数据\\东莞银行\\东莞重估.txt", [], "utf-8", true);
println "txt 生成" println "txt 生成"
exportCsv("/home/goojia/东莞重估.csv", datas) exportCsv("D:\\数据\\东莞银行\\东莞重估.csv", datas)
println "csv 生成" println "csv 生成"
} }
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("#",-1)
if(row && row.size()>=2){
def s = row[1]
JSONObject result = JSONObject.fromObject(s);
rows << result
}
}
return rows;
}
def evaluation(rets){ def evaluation(rets){
def datas = new CopyOnWriteArrayList() def datas = new CopyOnWriteArrayList()
def totalCount = new AtomicLong() def totalCount = new AtomicLong()
@@ -45,7 +64,7 @@ class DongGuangApiTest {
void run() { void run() {
try{ try{
totalCount.addAndGet(1) totalCount.addAndGet(1)
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, dist.size) def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, 100)
if(ret && ret.avgPrice){ if(ret && ret.avgPrice){
def row = [] def row = []
row << dist.city row << dist.city
@@ -41,10 +41,10 @@ class DataStat {
def run(){ def run(){
def source = "365淘房"// def source = "365淘房"//
def citys = findCrawlCity(source, 202210); def citys = findCrawlCity(source, 202211);
for(def city : citys){ for(def city : citys){
def count1 = findCount(source, city, 202210); def count1 = findCount(source, city, 202211);
def count2 = findCount(source, city, 202211); def count2 = findCount(source, city, 202212);
println city +" "+count1+" "+count2 println city +" "+count1+" "+count2
// println count1 // println count1
// println "--- ----" // println "--- ----"