建行房源同步修改
This commit is contained in:
+203
-2
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221118, 20221219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
@@ -429,6 +429,13 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
|
||||
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
rets1.addAll(rets2)
|
||||
return rets1;
|
||||
}
|
||||
|
||||
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
@@ -586,7 +593,201 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(seawater.p_date<=20200101){
|
||||
//seawater.p_date.toString().replace("2019","2020")//
|
||||
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
}
|
||||
}
|
||||
|
||||
println name +" "+houses.size()+" last:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
|
||||
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range).must(date_range_mdate)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
|
||||
def ids = new HashSet()
|
||||
for(def h : rets1){
|
||||
ids << h.houseId
|
||||
}
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
if(ids && ids.contains(seawater.id)){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
// if(seawater.height && seawater.height<1){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = CcbCommonUtils.getCcbCity(city)
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
@@ -599,7 +800,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
|
||||
+3
-1
@@ -65,6 +65,8 @@ class CcbCommonUtils {
|
||||
}
|
||||
}else{
|
||||
citys = CcbCommonUtils.citys_92
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}
|
||||
println "运行城市数量:"+citys.size()
|
||||
return citys;
|
||||
@@ -422,7 +424,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","云山尚城")
|
||||
// q.put("name","鸣翠蓝湾")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
|
||||
+2
-2
@@ -111,8 +111,8 @@ class FzDataResultStat {
|
||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||
def datas1 = []
|
||||
def datas2 = []
|
||||
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
|
||||
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
|
||||
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||
def row_num = 0
|
||||
for(def row : rows){
|
||||
if(row_num>0){
|
||||
|
||||
+1
-1
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") ){
|
||||
if (!href.contains("fang.lianjia")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
|
||||
+23
-4
@@ -1,10 +1,12 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
@@ -24,17 +26,34 @@ class DongGuangApiTest {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findEsData();
|
||||
// def rets = findEsData();
|
||||
def rets = readTxt("D:\\数据\\东莞银行\\东莞银行 (1).csv")
|
||||
println "ES 加载完成"
|
||||
def datas = evaluation(rets)
|
||||
println "重估完成"
|
||||
// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据")
|
||||
FileUtils.batchCreateFile(datas, "/home/goojia/东莞重估.txt", [], "utf-8", true);
|
||||
FileUtils.batchCreateFile(datas, "D:\\数据\\东莞银行\\东莞重估.txt", [], "utf-8", true);
|
||||
println "txt 生成"
|
||||
exportCsv("/home/goojia/东莞重估.csv", datas)
|
||||
exportCsv("D:\\数据\\东莞银行\\东莞重估.csv", datas)
|
||||
println "csv 生成"
|
||||
}
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split("#",-1)
|
||||
if(row && row.size()>=2){
|
||||
def s = row[1]
|
||||
JSONObject result = JSONObject.fromObject(s);
|
||||
rows << result
|
||||
}
|
||||
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
def evaluation(rets){
|
||||
def datas = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
@@ -45,7 +64,7 @@ class DongGuangApiTest {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, dist.size)
|
||||
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, 100)
|
||||
if(ret && ret.avgPrice){
|
||||
def row = []
|
||||
row << dist.city
|
||||
|
||||
+3
-3
@@ -41,10 +41,10 @@ class DataStat {
|
||||
|
||||
def run(){
|
||||
def source = "365淘房"//
|
||||
def citys = findCrawlCity(source, 202210);
|
||||
def citys = findCrawlCity(source, 202211);
|
||||
for(def city : citys){
|
||||
def count1 = findCount(source, city, 202210);
|
||||
def count2 = findCount(source, city, 202211);
|
||||
def count1 = findCount(source, city, 202211);
|
||||
def count2 = findCount(source, city, 202212);
|
||||
println city +" "+count1+" "+count2
|
||||
// println count1
|
||||
// println "--- ----"
|
||||
|
||||
Reference in New Issue
Block a user