删除房源
This commit is contained in:
+22
-3
@@ -252,7 +252,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "燃灯小区")
|
||||
// q.put("name", "鲁能大厦")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -739,8 +739,9 @@ class CommonDatautils {
|
||||
public static def findAllDistrict(def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
// q.put("name", "中兴智汇大厦")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -748,7 +749,8 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
@@ -3897,5 +3899,22 @@ class CommonDatautils {
|
||||
return region;
|
||||
}
|
||||
|
||||
def static getAllFiles(def path) {
|
||||
|
||||
File file = new File(path);
|
||||
File[] tempList = file.listFiles();
|
||||
|
||||
def files = [];
|
||||
for (int i = 0; i < tempList.length; i++) {
|
||||
if (tempList[i].isFile()) {
|
||||
files << tempList[i];
|
||||
}
|
||||
if (tempList[i].isDirectory()) {
|
||||
System.out.println("文件夹:" + tempList[i]);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -236,6 +236,24 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
|
||||
//},0)
|
||||
|
||||
|
||||
def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
|
||||
if(files){
|
||||
for(def filepath : files){
|
||||
def rets = read(filepath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def read(path){
|
||||
File file = new File(path);
|
||||
def rets = []
|
||||
file.eachLine{it->
|
||||
rets << it
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
@@ -20,8 +21,10 @@ class FiledStrand {
|
||||
def updateIndex = 0;
|
||||
static main(args) {
|
||||
|
||||
def task = new FiledStrand();
|
||||
task.run();
|
||||
|
||||
|
||||
// def task = new FiledStrand();
|
||||
// task.run();
|
||||
// task.updateOpenDate();
|
||||
}
|
||||
|
||||
@@ -265,13 +268,58 @@ class FiledStrand {
|
||||
static def processArea(all_area){
|
||||
if(all_area){
|
||||
def v = getValue(all_area)
|
||||
if(v>=1000 && v<=10000000){
|
||||
if(v>=5000 && v<=10000000){
|
||||
return v.toString().toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processStandardHeight(height){
|
||||
if(height){
|
||||
def v = getValue(height)
|
||||
if(v>=2.4 && v<=10){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processElevators(elevators){
|
||||
if(elevators){
|
||||
def v = getValue(elevators)
|
||||
if(v>0 && v<=100){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processStandardArea(area){
|
||||
if(area){
|
||||
area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","")
|
||||
area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll("");
|
||||
return area
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processCarSiteRent(rent){
|
||||
if(rent){
|
||||
def v = getNumberRange(rent)
|
||||
if(v){
|
||||
if(v=="0"){
|
||||
return null
|
||||
}else{
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def processName(name){
|
||||
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
|
||||
if (name.contains('(') && name.contains(')')){
|
||||
@@ -338,7 +386,7 @@ class FiledStrand {
|
||||
if(v){
|
||||
if([40,50,70].contains(v)){
|
||||
try {
|
||||
return v.toString()+"年";
|
||||
return v.toString();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
@@ -378,6 +426,16 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processStandardAddress(def esAddress, def city){
|
||||
if(esAddress){
|
||||
def formatAddress = AddressUtil.formatAddress(city, esAddress)
|
||||
if(formatAddress && isStandardRoad(formatAddress)){
|
||||
return formatAddress
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processBlock(city, region, def block){
|
||||
if(block){
|
||||
if(block.contains("无") || block.toString().length()>10){
|
||||
@@ -464,6 +522,14 @@ class FiledStrand {
|
||||
return false
|
||||
}
|
||||
|
||||
static def isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
static def addBuildAreaToData(def dist, def updateFile){
|
||||
@@ -666,6 +732,34 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processFloor(floor){
|
||||
def v = getValue(floor.toString());
|
||||
if(v){
|
||||
if(v>=1 && v<=130){
|
||||
try {
|
||||
return v.toString();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processRealUsePercent(percent){
|
||||
def v = getValue(percent.toString());
|
||||
if(v){
|
||||
if(v>=20 && v<=80){
|
||||
try {
|
||||
return v.toString();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processFinishBuildingDate(def finishBuildingDate){
|
||||
if(finishBuildingDate){
|
||||
Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}");
|
||||
@@ -677,24 +771,37 @@ class FiledStrand {
|
||||
Pattern pat3 = Pattern.compile("\\d{4}");
|
||||
def matcher3 = pat3.matcher(finishBuildingDate);
|
||||
|
||||
def dateStr = null
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group().replace("年","-").replace("月","-")
|
||||
dateStr = matcher1.group().replace("年","-").replace("月","-").replace(".","")
|
||||
}else if(matcher2.find()){
|
||||
return matcher2.group().replace("年","-")
|
||||
dateStr = matcher2.group().replace("年","-").replace(".","-")
|
||||
}else if(matcher3.find()){
|
||||
return matcher3.group()
|
||||
dateStr = matcher3.group().replace(".","-")
|
||||
}
|
||||
if(dateStr){
|
||||
dateStr = dateStr.substring(0, 4)
|
||||
}
|
||||
return dateStr
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def processCompany(val){
|
||||
if(val){
|
||||
if(val.length()>20){
|
||||
return null
|
||||
}else{
|
||||
if(val.contains("无")){
|
||||
if(val.contains("暂无")){
|
||||
return null
|
||||
}
|
||||
if(val == "无"){
|
||||
return null
|
||||
}
|
||||
if(val.isNumber()){
|
||||
return null
|
||||
}
|
||||
def fiterChars = [",",".","?","/","数据",'-','?',' ',' ']
|
||||
@@ -771,4 +878,31 @@ class FiledStrand {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def patternParse(String keyword, String patternStr) {
|
||||
Pattern pattern = Pattern.compile(patternStr);
|
||||
Matcher matcher = pattern.matcher(keyword);
|
||||
String num = "";
|
||||
if (matcher.find()) {
|
||||
if (StringUtils.isNotBlank(matcher.group(0))) {
|
||||
num = matcher.group(1).trim();
|
||||
}
|
||||
}
|
||||
return num;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def getNumberRange(def input) {
|
||||
Pattern pattern = Pattern.compile("\\d+(-\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -164,7 +164,7 @@ class FangjiaClientUtils {
|
||||
// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
|
||||
// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
|
||||
|
||||
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
|
||||
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
|
||||
if (json.status == 0) {
|
||||
return "成功"
|
||||
}else if(json.status == 1){
|
||||
|
||||
+291
-28
@@ -6,9 +6,11 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -28,12 +30,24 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
|
||||
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area",
|
||||
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address",
|
||||
"live_date":"live_date", "loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段
|
||||
//"build_date":"build_year", "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address",
|
||||
// "live_date":"live_date", "loop_line_info":"loop_line"
|
||||
def fieldsMap = ["estate_charge":"property_fee","block":"block","estate_company":"property_company","developer":"developer",
|
||||
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area"
|
||||
,"finish_building_date":"finish_building_date","property_life":"property_life","standard_area":"standard_area","total_floor":"total_floor",
|
||||
"standard_height":"floor_height", "real_use_percent":"room_rate", "loop_line_info":"ring_location","car_site_count":"parking",
|
||||
"parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area"]//key:seaweed表字段 value:es表字段
|
||||
|
||||
def sourceMap = ["房天下":"房天下","办办网":"办办网","点点租":"点点租","安居客":"安居客","楼盘网":"楼盘网"]
|
||||
//"安居客":"安居客app",
|
||||
def sourceMap = ["房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"]
|
||||
|
||||
def fieldMap = ["编号":"_id","城市":"city","区域":"region","名称":"name","别名":"alias","租金":"rent_price","板块":"block","物业公司":"estate_company","物业费":"estate_charge",
|
||||
"地址":"address","地址别名":"addr_alias","经度":"lng","纬度":"lat","类型":"estate_type2","建筑面积":"build_area","等级":"bed_kind",
|
||||
"开发商":"developer","占地面积":"all_area","标准层面积":"standard_area","产权年限":"property_life","总层数":"total_floor","是否涉外":"is_foreign",
|
||||
"容积率":"cubage_percent","绿化率":"virescence_percent","货梯数":"goods_elevator_count","竣工时间":"finish_building_date","客梯数":"person_elevators_count",
|
||||
"标准层高":"standard_height","净高":"clear_height","大堂层高":"lobby_height","得房率":"real_use_percent","是否可分割":"divisible","环线位置":"loop_line_info",
|
||||
"车位描述":"car_site","车位数量":"car_site_count","地上车位":"land_car_site","地下停车位":"underground_car_site","车位租金":"parking_fee","电梯数量":"elevators_count",
|
||||
"是否注册":"is_register","开间面积":"bay_area","楼层信息":"floor_info"]
|
||||
|
||||
static DBCollection BLOCKPOLYLINECOLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border2", "mongo.db.fangjia.com")
|
||||
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
@@ -46,14 +60,183 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// def str = "1200平方/层,1067㎡,1650-1900平米"
|
||||
|
||||
|
||||
// def citys = CommonDatautils.findAllCity().reverse()
|
||||
// for(def c : citys){
|
||||
// def offices = findAllOffice(c)
|
||||
// strandField(offices)
|
||||
// }
|
||||
|
||||
addBlock()
|
||||
// addBlock()
|
||||
|
||||
// addFieldValue()
|
||||
// test()
|
||||
updateErrorField()
|
||||
// export()
|
||||
// addOffice()
|
||||
}
|
||||
|
||||
def export(){
|
||||
def offices = CommonDatautils.findAllFiledDistrict("office", "上海")
|
||||
def fields = fieldMap.keySet().asList()
|
||||
def rets = []
|
||||
rets << fields
|
||||
for(def i=0;i<offices.size();i++){
|
||||
def data = offices[i]
|
||||
if(data.region == "浦东"){
|
||||
def row = []
|
||||
fieldMap.each{k,v->
|
||||
def val = data.get(v)
|
||||
if(k == "编号"){
|
||||
row << val.toString()
|
||||
}else if(v=="lat" || v=="lng"){
|
||||
row << val.toString().toDouble()/10000000000.0
|
||||
}else if(v == "divisible"){
|
||||
if(val){
|
||||
if(val.toString()=="1"){
|
||||
row << "是"
|
||||
}else if(val.toString()=="0"){
|
||||
row << "否"
|
||||
}
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}else if(v.contains("alias")){
|
||||
if(val){
|
||||
row << val.join(" ")
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
}else{
|
||||
row << val
|
||||
}
|
||||
}
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\中远(原始字段).xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
def test(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格.xlsx",0 , 1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
map.put(d['A']+"#"+d['B']+'#'+d['C'], d)
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\中远海运写字楼.xlsx",{d->
|
||||
def row = map.get(d['B']+"#"+d['C']+'#'+d['D'])
|
||||
d['H'] = row['F']
|
||||
d['I'] = row['G']
|
||||
})
|
||||
}
|
||||
|
||||
def updateErrorField(){
|
||||
// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
|
||||
def offices = CommonDatautils.findAllDistricts("上海","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
|
||||
for(def i=0;i<offices.size();i++){
|
||||
final def o = offices[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
updateDistField(o)
|
||||
println totalCount.get() +" "+o.city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def updateDistField(o){
|
||||
def addr_alias = o.addr_alias
|
||||
def udata = new BasicDBObject()
|
||||
if(o.property_life){
|
||||
def newVal = FiledStrand.processPropertyLife(o.property_life)
|
||||
udata.put("property_life", newVal)
|
||||
}
|
||||
|
||||
if(o.total_floor){
|
||||
def newVal = FiledStrand.processFloor(o.total_floor)
|
||||
udata.put("total_floor", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.finish_building_date){
|
||||
def newVal = FiledStrand.processFinishBuildingDate(o.finish_building_date)
|
||||
udata.put("finish_building_date", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.real_use_percent){
|
||||
def newVal = FiledStrand.processRealUsePercent(o.real_use_percent)
|
||||
udata.put("real_use_percent", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.underground_car_site){
|
||||
def newVal = FiledStrand.processCarSite(o.underground_car_site)
|
||||
udata.put("underground_car_site", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.land_car_site){
|
||||
def newVal = FiledStrand.processCarSite(o.land_car_site)
|
||||
udata.put("land_car_site", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.parking_fee){
|
||||
def newVal = FiledStrand.processCarSiteRent(o.parking_fee)
|
||||
udata.put("parking_fee", newVal)
|
||||
}
|
||||
|
||||
if(addr_alias){
|
||||
def addrs = []
|
||||
for(def arr : addr_alias){
|
||||
def addr = FiledStrand.processStandardAddress(arr, o.city)
|
||||
if(addr){
|
||||
addrs << addr
|
||||
}
|
||||
}
|
||||
if(addrs!=addr_alias){
|
||||
if(addrs){
|
||||
udata.put("addr_alias", addrs)
|
||||
}else{
|
||||
udata.put("addr_alias", null)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// if(o.estate_charge && !o.estate_charge.isNumber()){
|
||||
//// def val = o.estate_charge.toString().toDouble()
|
||||
//// if(val>40 || val < 0.01){
|
||||
//// println val
|
||||
// udata.put("estate_charge", null)
|
||||
//// }
|
||||
// }
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',o._id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def addFieldValue(){
|
||||
def citys = ["上海"]
|
||||
def analysRets = []
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("office", c)
|
||||
if(totalCount){
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -88,7 +271,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
def findEsDatas(c, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
|
||||
ElasticUtils.executeSqlQuery("select * from office where city='${c}' and source='${esWebSource}'", {r->
|
||||
list << r
|
||||
})
|
||||
return list
|
||||
@@ -203,6 +386,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findEsDatas(c, esWebSource)
|
||||
// esRets = esRets[0..100]
|
||||
def matchRets1 = matchApi(esRets)
|
||||
def matchRets2 = selectRestByCredit(matchRets1)
|
||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||
@@ -313,7 +497,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
|
||||
newVal = FiledStrand.processArea(esVal)
|
||||
newVal = FiledStrand.processArea(esVal.toString())
|
||||
if(newVal){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
@@ -360,6 +544,33 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
newVal = FiledStrand.processBuildStructure(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "total_floor"){
|
||||
newVal = FiledStrand.processFloor(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "real_use_percent"){
|
||||
newVal = FiledStrand.processRealUsePercent(esVal)
|
||||
}
|
||||
if(seaweedfield == "parking_fee"){
|
||||
newVal = FiledStrand.processCarSiteRent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "elevators_count"){
|
||||
newVal = FiledStrand.processElevators(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "standard_area"){
|
||||
newVal = FiledStrand.processStandardArea(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "standard_height"){
|
||||
newVal = FiledStrand.processStandardHeight(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "bay_area"){
|
||||
newVal = esVal
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
@@ -369,38 +580,68 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
return newValues;
|
||||
}
|
||||
|
||||
def updateDB(c, updateRets){
|
||||
|
||||
for(def rm : updateRets) {
|
||||
def id = rm.key
|
||||
def valMap = rm.value
|
||||
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
valMap.each {k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
def addBlock(){
|
||||
def rets = findAllBlockRange()
|
||||
|
||||
def offices = CommonDatautils.findAllDistricts("上海", "office", ["city","region","name","block","lat2", "lng2"])
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
|
||||
|
||||
for(def i=0;i<=offices.size();i++){
|
||||
final def office = offices[i]
|
||||
if(office && !office.block){
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def lastObj = null
|
||||
for (def r : rets) {
|
||||
def flag = InsidePolygon.isInArea(office.lng2 / 10000000000.0, office.lat2 / 10000000000.0, r.points)
|
||||
if (flag) {
|
||||
lastObj = r;
|
||||
break;
|
||||
def lastObj = null
|
||||
for (def r : rets) {
|
||||
def flag = InsidePolygon.isInArea(office.lng2 / 10000000000.0, office.lat2 / 10000000000.0, r.points)
|
||||
if (flag && r.name) {
|
||||
lastObj = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (lastObj) {
|
||||
def udata = [:]
|
||||
udata.put("block", lastObj.name)
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "office")
|
||||
udata.put("username", "高娟")
|
||||
udata.put("id", office._id.toString())
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println s
|
||||
}
|
||||
}
|
||||
if (lastObj) {
|
||||
def udata = [:]
|
||||
udata.put("block", lastObj.name)
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "office")
|
||||
udata.put("username", "高娟")
|
||||
udata.put("id", office._id.toString())
|
||||
udata.put("loopLineInfo", lastObj.loop)
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println s
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def findAllBlockRange(){
|
||||
@@ -430,6 +671,28 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
def addOffice(){
|
||||
def city = "上海"
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\bd_file_insert(待入库).xlsx",{d->
|
||||
def udata = [:]
|
||||
def formatAddress = AddressUtil.formatAddress(city, d['B'])
|
||||
udata.put("city", city)
|
||||
udata.put("region", d['A'])
|
||||
udata.put("name", d['D'])
|
||||
udata.put("address", formatAddress)
|
||||
long lng = (d['L'].toString().toDouble()*10000000000).longValue()
|
||||
long lat = (d['K'].toString().toDouble()*10000000000).longValue()
|
||||
|
||||
udata.put("lat", lat)
|
||||
udata.put("lng", lng)
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "office")
|
||||
udata.put("username", "高娟")
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println d['D'] + s
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.fangjia.script.gj.house
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class RemoveHouse extends BaseBatch{
|
||||
|
||||
DBCollection SEABIRD_COLL = MongoUtils.getCollectionByName("fangjia", "seabird", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new RemoveHouse(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def start = new Date()
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2023-02-01")
|
||||
def q = new BasicDBObject()
|
||||
q.put("c_date", new BasicDBObject('$lte', endDate))
|
||||
def dbcursor = SEABIRD_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d._id
|
||||
if(rets.size()==500){
|
||||
def delquery = new BasicDBObject()
|
||||
delquery.put("_id", new BasicDBObject('$in', rets))
|
||||
SEABIRD_COLL.remove(delquery)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
}
|
||||
def end = new Date()
|
||||
|
||||
println "¿ªÊ¼Ê±¼ä£º"+start
|
||||
println "½áÊøÊ±¼ä£º"+end
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user