Merge branch 'script_gj' into 'master'

删除房源

See merge request !678
This commit is contained in:
2023-07-14 10:05:15 +08:00
6 changed files with 519 additions and 40 deletions
@@ -252,7 +252,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "燃灯小区")
// q.put("name", "鲁能大厦")
DBObject keys = new BasicDBObject();
if(fields){
@@ -739,8 +739,9 @@ class CommonDatautils {
public static def findAllDistrict(def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
// q.put("name", "中兴智汇大厦")
DBObject keys = new BasicDBObject();
if(fields){
@@ -748,7 +749,8 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
@@ -3897,5 +3899,22 @@ class CommonDatautils {
return region;
}
def static getAllFiles(def path) {
File file = new File(path);
File[] tempList = file.listFiles();
def files = [];
for (int i = 0; i < tempList.length; i++) {
if (tempList[i].isFile()) {
files << tempList[i];
}
if (tempList[i].isDirectory()) {
System.out.println("文件夹:" + tempList[i]);
}
}
return files;
}
}
@@ -236,6 +236,24 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
//},0)
def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
if(files){
for(def filepath : files){
def rets = read(filepath);
}
}
def read(path){
File file = new File(path);
def rets = []
file.eachLine{it->
rets << it
}
return rets;
}
@@ -8,6 +8,7 @@ import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -20,8 +21,10 @@ class FiledStrand {
def updateIndex = 0;
static main(args) {
def task = new FiledStrand();
task.run();
// def task = new FiledStrand();
// task.run();
// task.updateOpenDate();
}
@@ -265,13 +268,58 @@ class FiledStrand {
static def processArea(all_area){
if(all_area){
def v = getValue(all_area)
if(v>=1000 && v<=10000000){
if(v>=5000 && v<=10000000){
return v.toString().toDouble().toInteger()
}
}
return null
}
static def processStandardHeight(height){
if(height){
def v = getValue(height)
if(v>=2.4 && v<=10){
return v.toString()
}
}
return null
}
static def processElevators(elevators){
if(elevators){
def v = getValue(elevators)
if(v>0 && v<=100){
return v.toString()
}
}
return null
}
static def processStandardArea(area){
if(area){
area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","")
area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll("");
return area
}
return null
}
static def processCarSiteRent(rent){
if(rent){
def v = getNumberRange(rent)
if(v){
if(v=="0"){
return null
}else{
return v.toString()
}
}
return null
}
return null
}
static def processName(name){
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
if (name.contains('(') && name.contains(')')){
@@ -338,7 +386,7 @@ class FiledStrand {
if(v){
if([40,50,70].contains(v)){
try {
return v.toString()+"年";
return v.toString();
}catch(Exception e){
println e.getMessage()
}
@@ -378,6 +426,16 @@ class FiledStrand {
return null
}
static def processStandardAddress(def esAddress, def city){
if(esAddress){
def formatAddress = AddressUtil.formatAddress(city, esAddress)
if(formatAddress && isStandardRoad(formatAddress)){
return formatAddress
}
}
return null
}
static def processBlock(city, region, def block){
if(block){
if(block.contains("无") || block.toString().length()>10){
@@ -464,6 +522,14 @@ class FiledStrand {
return false
}
static def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
static def addBuildAreaToData(def dist, def updateFile){
@@ -666,6 +732,34 @@ class FiledStrand {
return null
}
static def processFloor(floor){
def v = getValue(floor.toString());
if(v){
if(v>=1 && v<=130){
try {
return v.toString();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processRealUsePercent(percent){
def v = getValue(percent.toString());
if(v){
if(v>=20 && v<=80){
try {
return v.toString();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processFinishBuildingDate(def finishBuildingDate){
if(finishBuildingDate){
Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}");
@@ -677,24 +771,37 @@ class FiledStrand {
Pattern pat3 = Pattern.compile("\\d{4}");
def matcher3 = pat3.matcher(finishBuildingDate);
def dateStr = null
if (matcher1.find()) {
return matcher1.group().replace("年","-").replace("月","-")
dateStr = matcher1.group().replace("年","-").replace("月","-").replace(".","")
}else if(matcher2.find()){
return matcher2.group().replace("年","-")
dateStr = matcher2.group().replace("年","-").replace(".","-")
}else if(matcher3.find()){
return matcher3.group()
dateStr = matcher3.group().replace(".","-")
}
if(dateStr){
dateStr = dateStr.substring(0, 4)
}
return dateStr
}
return null;
}
static def processCompany(val){
if(val){
if(val.length()>20){
return null
}else{
if(val.contains("无")){
if(val.contains("暂无")){
return null
}
if(val == "无"){
return null
}
if(val.isNumber()){
return null
}
def fiterChars = [",",".","?","/","数据",'-','?','&#160;','&nbsp;']
@@ -771,4 +878,31 @@ class FiledStrand {
}
return null;
}
static def patternParse(String keyword, String patternStr) {
Pattern pattern = Pattern.compile(patternStr);
Matcher matcher = pattern.matcher(keyword);
String num = "";
if (matcher.find()) {
if (StringUtils.isNotBlank(matcher.group(0))) {
num = matcher.group(1).trim();
}
}
return num;
}
static def getNumberRange(def input) {
Pattern pattern = Pattern.compile("\\d+(-\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
}
return null;
}
}
@@ -164,7 +164,7 @@ class FangjiaClientUtils {
// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
if (json.status == 0) {
return "成功"
}else if(json.status == 1){
@@ -6,9 +6,11 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
@@ -28,12 +30,24 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class OfficeFieldCleanBatch extends BaseBatch{
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area",
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address",
"live_date":"live_date", "loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段
//"build_date":"build_year", "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address",
// "live_date":"live_date", "loop_line_info":"loop_line"
def fieldsMap = ["estate_charge":"property_fee","block":"block","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area"
,"finish_building_date":"finish_building_date","property_life":"property_life","standard_area":"standard_area","total_floor":"total_floor",
"standard_height":"floor_height", "real_use_percent":"room_rate", "loop_line_info":"ring_location","car_site_count":"parking",
"parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area"]//key:seaweed表字段 value:es表字段
def sourceMap = ["房天下":"房天下","办办网":"办办网","点点租":"点点租","安居客":"安居客","楼盘网":"楼盘网"]
//"安居客":"安居客app",
def sourceMap = ["房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"]
def fieldMap = ["编号":"_id","城市":"city","区域":"region","名称":"name","别名":"alias","租金":"rent_price","板块":"block","物业公司":"estate_company","物业费":"estate_charge",
"地址":"address","地址别名":"addr_alias","经度":"lng","纬度":"lat","类型":"estate_type2","建筑面积":"build_area","等级":"bed_kind",
"开发商":"developer","占地面积":"all_area","标准层面积":"standard_area","产权年限":"property_life","总层数":"total_floor","是否涉外":"is_foreign",
"容积率":"cubage_percent","绿化率":"virescence_percent","货梯数":"goods_elevator_count","竣工时间":"finish_building_date","客梯数":"person_elevators_count",
"标准层高":"standard_height","净高":"clear_height","大堂层高":"lobby_height","得房率":"real_use_percent","是否可分割":"divisible","环线位置":"loop_line_info",
"车位描述":"car_site","车位数量":"car_site_count","地上车位":"land_car_site","地下停车位":"underground_car_site","车位租金":"parking_fee","电梯数量":"elevators_count",
"是否注册":"is_register","开间面积":"bay_area","楼层信息":"floor_info"]
static DBCollection BLOCKPOLYLINECOLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border2", "mongo.db.fangjia.com")
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
@@ -46,14 +60,183 @@ class OfficeFieldCleanBatch extends BaseBatch{
@Override
void process(Object args) {
// def str = "1200平方/层,1067㎡,1650-1900平米"
// def citys = CommonDatautils.findAllCity().reverse()
// for(def c : citys){
// def offices = findAllOffice(c)
// strandField(offices)
// }
addBlock()
// addBlock()
// addFieldValue()
// test()
updateErrorField()
// export()
// addOffice()
}
def export(){
def offices = CommonDatautils.findAllFiledDistrict("office", "上海")
def fields = fieldMap.keySet().asList()
def rets = []
rets << fields
for(def i=0;i<offices.size();i++){
def data = offices[i]
if(data.region == "浦东"){
def row = []
fieldMap.each{k,v->
def val = data.get(v)
if(k == "编号"){
row << val.toString()
}else if(v=="lat" || v=="lng"){
row << val.toString().toDouble()/10000000000.0
}else if(v == "divisible"){
if(val){
if(val.toString()=="1"){
row << "是"
}else if(val.toString()=="0"){
row << "否"
}
}else{
row << ""
}
}else if(v.contains("alias")){
if(val){
row << val.join(" ")
}else{
row << ''
}
}else{
row << val
}
}
rets << row
}
}
ExcelUtils.write("D:\\数据\\写字楼\\中远(原始字段).xlsx",rets, "数据")
}
def test(){
def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格.xlsx",0 , 1)
def map = [:]
for(def d : datas){
map.put(d['A']+"#"+d['B']+'#'+d['C'], d)
}
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\中远海运写字楼.xlsx",{d->
def row = map.get(d['B']+"#"+d['C']+'#'+d['D'])
d['H'] = row['F']
d['I'] = row['G']
})
}
def updateErrorField(){
// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def offices = CommonDatautils.findAllDistricts("上海","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
for(def i=0;i<offices.size();i++){
final def o = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
updateDistField(o)
println totalCount.get() +" "+o.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def updateDistField(o){
def addr_alias = o.addr_alias
def udata = new BasicDBObject()
if(o.property_life){
def newVal = FiledStrand.processPropertyLife(o.property_life)
udata.put("property_life", newVal)
}
if(o.total_floor){
def newVal = FiledStrand.processFloor(o.total_floor)
udata.put("total_floor", newVal.toString())
}
if(o.finish_building_date){
def newVal = FiledStrand.processFinishBuildingDate(o.finish_building_date)
udata.put("finish_building_date", newVal.toString())
}
if(o.real_use_percent){
def newVal = FiledStrand.processRealUsePercent(o.real_use_percent)
udata.put("real_use_percent", newVal.toString())
}
if(o.underground_car_site){
def newVal = FiledStrand.processCarSite(o.underground_car_site)
udata.put("underground_car_site", newVal.toString())
}
if(o.land_car_site){
def newVal = FiledStrand.processCarSite(o.land_car_site)
udata.put("land_car_site", newVal.toString())
}
if(o.parking_fee){
def newVal = FiledStrand.processCarSiteRent(o.parking_fee)
udata.put("parking_fee", newVal)
}
if(addr_alias){
def addrs = []
for(def arr : addr_alias){
def addr = FiledStrand.processStandardAddress(arr, o.city)
if(addr){
addrs << addr
}
}
if(addrs!=addr_alias){
if(addrs){
udata.put("addr_alias", addrs)
}else{
udata.put("addr_alias", null)
}
}
}
// if(o.estate_charge && !o.estate_charge.isNumber()){
//// def val = o.estate_charge.toString().toDouble()
//// if(val>40 || val < 0.01){
//// println val
// udata.put("estate_charge", null)
//// }
// }
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',o._id),new BasicDBObject('$set',udata))
}
}
def addFieldValue(){
def citys = ["上海"]
def analysRets = []
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("office", c)
if(totalCount){
analysRets = addFieldFromMatch(c, totalCount, analysRets)
}
}
}
@@ -88,7 +271,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
def findEsDatas(c, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
ElasticUtils.executeSqlQuery("select * from office where city='${c}' and source='${esWebSource}'", {r->
list << r
})
return list
@@ -203,6 +386,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource)
// esRets = esRets[0..100]
def matchRets1 = matchApi(esRets)
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
@@ -313,7 +497,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
newVal = FiledStrand.processArea(esVal)
newVal = FiledStrand.processArea(esVal.toString())
if(newVal){
newVal = newVal.toString()
}
@@ -360,6 +544,33 @@ class OfficeFieldCleanBatch extends BaseBatch{
newVal = FiledStrand.processBuildStructure(esVal)
}
if(seaweedfield == "total_floor"){
newVal = FiledStrand.processFloor(esVal)
}
if(seaweedfield == "real_use_percent"){
newVal = FiledStrand.processRealUsePercent(esVal)
}
if(seaweedfield == "parking_fee"){
newVal = FiledStrand.processCarSiteRent(esVal)
}
if(seaweedfield == "elevators_count"){
newVal = FiledStrand.processElevators(esVal)
}
if(seaweedfield == "standard_area"){
newVal = FiledStrand.processStandardArea(esVal)
}
if(seaweedfield == "standard_height"){
newVal = FiledStrand.processStandardHeight(esVal)
}
if(seaweedfield == "bay_area"){
newVal = esVal
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
@@ -369,38 +580,68 @@ class OfficeFieldCleanBatch extends BaseBatch{
return newValues;
}
def updateDB(c, updateRets){
for(def rm : updateRets) {
def id = rm.key
def valMap = rm.value
BasicDBObject udata = new BasicDBObject()
valMap.each {k,v->
udata.put(k,v)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
}
}
def addBlock(){
def rets = findAllBlockRange()
def offices = CommonDatautils.findAllDistricts("上海", "office", ["city","region","name","block","lat2", "lng2"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
for(def i=0;i<=offices.size();i++){
final def office = offices[i]
if(office && !office.block){
fixedThreadPool.execute(new Runnable() {
void run() {
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def lastObj = null
for (def r : rets) {
def flag = InsidePolygon.isInArea(office.lng2 / 10000000000.0, office.lat2 / 10000000000.0, r.points)
if (flag) {
lastObj = r;
break;
def lastObj = null
for (def r : rets) {
def flag = InsidePolygon.isInArea(office.lng2 / 10000000000.0, office.lat2 / 10000000000.0, r.points)
if (flag && r.name) {
lastObj = r;
break;
}
}
}
if (lastObj) {
def udata = [:]
udata.put("block", lastObj.name)
udata.put("status", "0")
udata.put("cat", "office")
udata.put("username", "高娟")
udata.put("id", office._id.toString())
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println s
}
}
if (lastObj) {
def udata = [:]
udata.put("block", lastObj.name)
udata.put("status", "0")
udata.put("cat", "office")
udata.put("username", "高娟")
udata.put("id", office._id.toString())
udata.put("loopLineInfo", lastObj.loop)
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println s
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}
latch.await()
}
def findAllBlockRange(){
@@ -430,6 +671,28 @@ class OfficeFieldCleanBatch extends BaseBatch{
OPERATION_LOG_COLL.save(obj)
}
def addOffice(){
def city = "上海"
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\bd_file_insert(待入库).xlsx",{d->
def udata = [:]
def formatAddress = AddressUtil.formatAddress(city, d['B'])
udata.put("city", city)
udata.put("region", d['A'])
udata.put("name", d['D'])
udata.put("address", formatAddress)
long lng = (d['L'].toString().toDouble()*10000000000).longValue()
long lat = (d['K'].toString().toDouble()*10000000000).longValue()
udata.put("lat", lat)
udata.put("lng", lng)
udata.put("status", "0")
udata.put("cat", "office")
udata.put("username", "高娟")
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println d['D'] + s
})
}
@@ -0,0 +1,45 @@
package com.fangjia.script.gj.house
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class RemoveHouse extends BaseBatch{
DBCollection SEABIRD_COLL = MongoUtils.getCollectionByName("fangjia", "seabird", "mongo.db.fangjia.com")
static main(args) {
runBatch(new RemoveHouse(), args)
}
@Override
void process(Object args) {
def start = new Date()
def endDate = DateUtils.parseFromTightYYYYMMDD("2023-02-01")
def q = new BasicDBObject()
q.put("c_date", new BasicDBObject('$lte', endDate))
def dbcursor = SEABIRD_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << d._id
if(rets.size()==500){
def delquery = new BasicDBObject()
delquery.put("_id", new BasicDBObject('$in', rets))
SEABIRD_COLL.remove(delquery)
rets = []
print "."
}
}
def end = new Date()
println "¿ªÊ¼Ê±¼ä£º"+start
println "½áÊøÊ±¼ä£º"+end
}
}