建行房源同步修改
This commit is contained in:
+203
-2
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221118, 20221219);
|
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20221218, 20230119);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210901, 20221119);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211001, 20221011);
|
||||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||||
@@ -429,6 +429,13 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
|
|
||||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||||
|
|
||||||
|
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||||
|
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||||
|
rets1.addAll(rets2)
|
||||||
|
return rets1;
|
||||||
|
}
|
||||||
|
|
||||||
|
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||||
|
|
||||||
@@ -586,7 +593,201 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
def fitment = parseFitment(seawater.fitment)
|
def fitment = parseFitment(seawater.fitment)
|
||||||
rowMap.fitment = fitment=="null"?null:fitment
|
rowMap.fitment = fitment=="null"?null:fitment
|
||||||
|
|
||||||
|
if(seawater.p_date instanceof String){
|
||||||
|
seawater.p_date = seawater.p_date.toInteger()
|
||||||
|
}
|
||||||
|
if(seawater.p_date<=20200101){
|
||||||
|
//seawater.p_date.toString().replace("2019","2020")//
|
||||||
|
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||||
|
rowMap.c_date = rowMap.p_date
|
||||||
|
rowMap.m_date = rowMap.p_date
|
||||||
|
}else{
|
||||||
|
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||||
|
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||||
|
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||||
|
}
|
||||||
|
|
||||||
|
rets << rowMap
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
println name +" "+houses.size()+" last:"+rets.size()
|
||||||
|
return rets;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||||
|
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||||
|
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
|
||||||
|
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||||
|
|
||||||
|
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||||
|
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||||
|
|
||||||
|
|
||||||
|
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||||
|
|
||||||
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||||
|
.must(date_range).must(area_range).must(date_range_mdate)
|
||||||
|
|
||||||
|
if(!alias){
|
||||||
|
alias = []
|
||||||
|
}
|
||||||
|
if(!alias.contains(name)){
|
||||||
|
alias.add(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(name.contains("弄小区")){
|
||||||
|
def newName = name.replace("弄小区","弄")
|
||||||
|
if(!alias.contains(newName)){
|
||||||
|
alias << newName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(alias && alias.size()>0){
|
||||||
|
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||||
|
for(def s : alias){
|
||||||
|
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||||
|
}
|
||||||
|
queryBuilder.must(inBuilder)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def rets = []
|
||||||
|
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||||
|
|
||||||
|
if(houses){
|
||||||
|
|
||||||
|
def ids = new HashSet()
|
||||||
|
for(def h : rets1){
|
||||||
|
ids << h.houseId
|
||||||
|
}
|
||||||
|
for (int k=0;k<houses.size();k++) {
|
||||||
|
def map = houses[k];
|
||||||
|
DBObject seawater = new BasicDBObject(map)
|
||||||
|
if(ids && ids.contains(seawater.id)){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(estate_type2 && estate_type2=="普通住宅"){
|
||||||
|
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||||
|
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||||
|
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||||
|
if(vallaFlag){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if(!seawater.total_price){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if(!seawater.avg_price){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!seawater.height){
|
||||||
|
seawater.height = null
|
||||||
|
}
|
||||||
|
// if(seawater.height && seawater.height<1){
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
if(seawater.height && seawater.height>60){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// if(!seawater.room){
|
||||||
|
// continue;
|
||||||
|
// }
|
||||||
|
if(!seawater.room){
|
||||||
|
seawater.room = null
|
||||||
|
}
|
||||||
|
|
||||||
|
if(seawater.room){
|
||||||
|
if(seawater.room>=11 || seawater.room<1){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!seawater.hall){
|
||||||
|
seawater.hall = null
|
||||||
|
}
|
||||||
|
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if(!seawater.toilet){
|
||||||
|
seawater.toilet = null
|
||||||
|
}
|
||||||
|
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def floor = parseFloor(seawater.floor_info)
|
||||||
|
if(seawater.height && floor && floor.isNumber()){
|
||||||
|
if(floor.toInteger()>seawater.height){
|
||||||
|
floor = null
|
||||||
|
seawater.height = null
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
if(!floor){
|
||||||
|
seawater.floor = null
|
||||||
|
}
|
||||||
|
if(floor && floor.toInteger()<1){
|
||||||
|
floor = null
|
||||||
|
}
|
||||||
|
|
||||||
|
def rowMap = [:];
|
||||||
|
if(cat == "rent"){
|
||||||
|
rowMap.rent_type = seawater.rent_type
|
||||||
|
if(seawater.avg_price && seawater.avg_price>1000){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if(seawater.avg_price && seawater.avg_price<10){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
rowMap.houseId = seawater.id
|
||||||
|
rowMap.city = CcbCommonUtils.getCcbCity(city)
|
||||||
|
//rowMap.region =seawater.region
|
||||||
|
//rowMap.address = seawater.address
|
||||||
|
|
||||||
|
|
||||||
|
rowMap.area = seawater.area
|
||||||
|
rowMap.total_price = seawater.total_price
|
||||||
|
rowMap.avg_price = seawater.avg_price
|
||||||
|
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||||
|
if(rowMap.height && rowMap.height<1){
|
||||||
|
rowMap.height = null
|
||||||
|
}
|
||||||
|
rowMap.floor = floor=="null"?null:floor
|
||||||
|
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||||
|
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||||
|
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||||
|
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||||
|
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||||
|
}else{
|
||||||
|
rowMap.root = "99"
|
||||||
|
}
|
||||||
|
|
||||||
|
def direction = parseDirection(seawater.direction)
|
||||||
|
rowMap.direction = direction=="null"?null:direction
|
||||||
|
|
||||||
|
def fitment = parseFitment(seawater.fitment)
|
||||||
|
rowMap.fitment = fitment=="null"?null:fitment
|
||||||
|
|
||||||
if(seawater.p_date instanceof String){
|
if(seawater.p_date instanceof String){
|
||||||
seawater.p_date = seawater.p_date.toInteger()
|
seawater.p_date = seawater.p_date.toInteger()
|
||||||
@@ -599,7 +800,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
|||||||
}else{
|
}else{
|
||||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||||
rowMap.m_date = DateUtils.getConverDate(seawater.m_date)
|
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||||
}
|
}
|
||||||
|
|
||||||
rets << rowMap
|
rets << rowMap
|
||||||
|
|||||||
+3
-1
@@ -65,6 +65,8 @@ class CcbCommonUtils {
|
|||||||
}
|
}
|
||||||
}else{
|
}else{
|
||||||
citys = CcbCommonUtils.citys_92
|
citys = CcbCommonUtils.citys_92
|
||||||
|
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||||
|
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||||
}
|
}
|
||||||
println "运行城市数量:"+citys.size()
|
println "运行城市数量:"+citys.size()
|
||||||
return citys;
|
return citys;
|
||||||
@@ -422,7 +424,7 @@ class CcbCommonUtils {
|
|||||||
q.put("visible",0)
|
q.put("visible",0)
|
||||||
q.put("cat", cat);
|
q.put("cat", cat);
|
||||||
q.put("city", city);
|
q.put("city", city);
|
||||||
// q.put("name","云山尚城")
|
// q.put("name","鸣翠蓝湾")
|
||||||
// q.put("name_cat", 0);
|
// q.put("name_cat", 0);
|
||||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -111,8 +111,8 @@ class FzDataResultStat {
|
|||||||
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"];
|
||||||
def datas1 = []
|
def datas1 = []
|
||||||
def datas2 = []
|
def datas2 = []
|
||||||
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
|
datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||||
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"]
|
datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
|
||||||
def row_num = 0
|
def row_num = 0
|
||||||
for(def row : rows){
|
for(def row : rows){
|
||||||
if(row_num>0){
|
if(row_num>0){
|
||||||
|
|||||||
+1
-1
@@ -474,7 +474,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles) {
|
for(def e : cityEles) {
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if (!href.contains("fang.lianjia") ){
|
if (!href.contains("fang.lianjia")){
|
||||||
cityMap.put(e.text(), href)
|
cityMap.put(e.text(), href)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+23
-4
@@ -1,10 +1,12 @@
|
|||||||
package com.fangjia.script.gj.test
|
package com.fangjia.script.gj.test
|
||||||
|
|
||||||
|
import com.fangjia.core.util.DataUtil
|
||||||
import com.fangjia.script.gj.ccb.FileUtils
|
import com.fangjia.script.gj.ccb.FileUtils
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.ElasticUtils
|
import com.fangjia.script.util.ElasticUtils
|
||||||
import com.fangjia.script.util.ExcelUtils
|
import com.fangjia.script.util.ExcelUtils
|
||||||
import com.fangjia.script.util.OpenApiClient
|
import com.fangjia.script.util.OpenApiClient
|
||||||
|
import net.sf.json.JSONObject
|
||||||
import org.elasticsearch.index.query.QueryBuilder
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
import org.elasticsearch.index.query.QueryBuilders
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
|
||||||
@@ -24,17 +26,34 @@ class DongGuangApiTest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def rets = findEsData();
|
// def rets = findEsData();
|
||||||
|
def rets = readTxt("D:\\数据\\东莞银行\\东莞银行 (1).csv")
|
||||||
println "ES 加载完成"
|
println "ES 加载完成"
|
||||||
def datas = evaluation(rets)
|
def datas = evaluation(rets)
|
||||||
println "重估完成"
|
println "重估完成"
|
||||||
// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据")
|
// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据")
|
||||||
FileUtils.batchCreateFile(datas, "/home/goojia/东莞重估.txt", [], "utf-8", true);
|
FileUtils.batchCreateFile(datas, "D:\\数据\\东莞银行\\东莞重估.txt", [], "utf-8", true);
|
||||||
println "txt 生成"
|
println "txt 生成"
|
||||||
exportCsv("/home/goojia/东莞重估.csv", datas)
|
exportCsv("D:\\数据\\东莞银行\\东莞重估.csv", datas)
|
||||||
println "csv 生成"
|
println "csv 生成"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def readTxt(def fileName){
|
||||||
|
File file = new File(fileName);
|
||||||
|
def rows = [];
|
||||||
|
|
||||||
|
file.eachLine{
|
||||||
|
def row = it.split("#",-1)
|
||||||
|
if(row && row.size()>=2){
|
||||||
|
def s = row[1]
|
||||||
|
JSONObject result = JSONObject.fromObject(s);
|
||||||
|
rows << result
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
def evaluation(rets){
|
def evaluation(rets){
|
||||||
def datas = new CopyOnWriteArrayList()
|
def datas = new CopyOnWriteArrayList()
|
||||||
def totalCount = new AtomicLong()
|
def totalCount = new AtomicLong()
|
||||||
@@ -45,7 +64,7 @@ class DongGuangApiTest {
|
|||||||
void run() {
|
void run() {
|
||||||
try{
|
try{
|
||||||
totalCount.addAndGet(1)
|
totalCount.addAndGet(1)
|
||||||
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, dist.size)
|
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, 100)
|
||||||
if(ret && ret.avgPrice){
|
if(ret && ret.avgPrice){
|
||||||
def row = []
|
def row = []
|
||||||
row << dist.city
|
row << dist.city
|
||||||
|
|||||||
+3
-3
@@ -41,10 +41,10 @@ class DataStat {
|
|||||||
|
|
||||||
def run(){
|
def run(){
|
||||||
def source = "365淘房"//
|
def source = "365淘房"//
|
||||||
def citys = findCrawlCity(source, 202210);
|
def citys = findCrawlCity(source, 202211);
|
||||||
for(def city : citys){
|
for(def city : citys){
|
||||||
def count1 = findCount(source, city, 202210);
|
def count1 = findCount(source, city, 202211);
|
||||||
def count2 = findCount(source, city, 202211);
|
def count2 = findCount(source, city, 202212);
|
||||||
println city +" "+count1+" "+count2
|
println city +" "+count1+" "+count2
|
||||||
// println count1
|
// println count1
|
||||||
// println "--- ----"
|
// println "--- ----"
|
||||||
|
|||||||
Reference in New Issue
Block a user