Merge branch 'script_gj' into 'master'

建行房源同步修改

See merge request !510
This commit is contained in:
2023-01-15 14:31:17 +08:00
6 changed files with 235 additions and 13 deletions
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221118, 20221219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
@@ -429,6 +429,13 @@ class BigdataForsaleBatch extends BaseBatch{
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
return rets1;
}
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -586,7 +593,201 @@ class BigdataForsaleBatch extends BaseBatch{
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
if(houses){
def ids = new HashSet()
for(def h : rets1){
ids << h.houseId
}
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(ids && ids.contains(seawater.id)){
continue;
}
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
@@ -599,7 +800,7 @@ class BigdataForsaleBatch extends BaseBatch{
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
@@ -65,6 +65,8 @@ class CcbCommonUtils {
}
}else{
citys = CcbCommonUtils.citys_92
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
println "运行城市数量:"+citys.size()
return citys;
@@ -422,7 +424,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","云山尚城")
// q.put("name","鸣翠蓝湾")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -111,8 +111,8 @@ class FzDataResultStat {
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
def datas1 = []
def datas2 = []
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
def row_num = 0
for(def row : rows){
if(row_num>0){
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
if (!href.contains("fang.lianjia") ){
if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
@@ -1,10 +1,12 @@
package com.fangjia.script.gj.test
import com.fangjia.core.util.DataUtil
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import net.sf.json.JSONObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
@@ -24,17 +26,34 @@ class DongGuangApiTest {
}
def run(){
def rets = findEsData();
// def rets = findEsData();
def rets = readTxt("D:\\数据\\东莞银行\\东莞银行 (1).csv")
println "ES 加载完成"
def datas = evaluation(rets)
println "重估完成"
// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据")
FileUtils.batchCreateFile(datas, "/home/goojia/东莞重估.txt", [], "utf-8", true);
FileUtils.batchCreateFile(datas, "D:\\数据\\东莞银行\\东莞重估.txt", [], "utf-8", true);
println "txt 生成"
exportCsv("/home/goojia/东莞重估.csv", datas)
exportCsv("D:\\数据\\东莞银行\\东莞重估.csv", datas)
println "csv 生成"
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("#",-1)
if(row && row.size()>=2){
def s = row[1]
JSONObject result = JSONObject.fromObject(s);
rows << result
}
}
return rows;
}
def evaluation(rets){
def datas = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
@@ -45,7 +64,7 @@ class DongGuangApiTest {
void run() {
try{
totalCount.addAndGet(1)
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, dist.size)
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, 100)
if(ret && ret.avgPrice){
def row = []
row << dist.city
@@ -41,10 +41,10 @@ class DataStat {
def run(){
def source = "365淘房"//
def citys = findCrawlCity(source, 202210);
def citys = findCrawlCity(source, 202211);
for(def city : citys){
def count1 = findCount(source, city, 202210);
def count2 = findCount(source, city, 202211);
def count1 = findCount(source, city, 202211);
def count2 = findCount(source, city, 202212);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"