@@ -239,7 +239,10 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
|
|||||||
def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
|
def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
|
||||||
if(files){
|
if(files){
|
||||||
for(def filepath : files){
|
for(def filepath : files){
|
||||||
def rets = read(filepath);
|
def filename = filepath.getPath()
|
||||||
|
def rets = read(filename);
|
||||||
|
def date = filename.substring(filename.toString().lastIndexOf("_")+1, filename.toString().indexOf("."))
|
||||||
|
println date +" "+rets.size()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -73,7 +73,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
|||||||
|
|
||||||
// addFieldValue()
|
// addFieldValue()
|
||||||
|
|
||||||
updateErrorField()
|
// updateErrorField()
|
||||||
export()
|
export()
|
||||||
test()
|
test()
|
||||||
// addOffice()
|
// addOffice()
|
||||||
@@ -120,7 +120,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
def test(){
|
def test(){
|
||||||
def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格租金.xlsx",0 , 1)
|
def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格 (2).xlsx",0 , 1)
|
||||||
def map = [:]
|
def map = [:]
|
||||||
for(def d : datas){
|
for(def d : datas){
|
||||||
map.put(d['A']+"#"+d['B']+'#'+d['C'], d)
|
map.put(d['A']+"#"+d['B']+'#'+d['C'], d)
|
||||||
|
|||||||
+1
-1
@@ -29,7 +29,7 @@ class RemoveHouse extends BaseBatch{
|
|||||||
def rets = []
|
def rets = []
|
||||||
for(def d : dbcursor){
|
for(def d : dbcursor){
|
||||||
rets << d._id
|
rets << d._id
|
||||||
if(rets.size()==100){
|
if(rets.size()==50){
|
||||||
def delquery = new BasicDBObject()
|
def delquery = new BasicDBObject()
|
||||||
delquery.put("_id", new BasicDBObject('$in', rets))
|
delquery.put("_id", new BasicDBObject('$in', rets))
|
||||||
SEABIRD_COLL.remove(delquery)
|
SEABIRD_COLL.remove(delquery)
|
||||||
|
|||||||
+2
-2
@@ -465,9 +465,9 @@ public class BeiKeCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles){
|
for(def e : cityEles){
|
||||||
def href = "https:"+e.attr("href")
|
def href = "https:"+e.attr("href")
|
||||||
if (citys.contains(e.text())){
|
// if (citys.contains(e.text())){
|
||||||
list << [e.text(), href]
|
list << [e.text(), href]
|
||||||
}
|
// }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return list
|
return list
|
||||||
|
|||||||
+1
-1
@@ -570,7 +570,7 @@ public class LianjiaCrawlByChrome {
|
|||||||
if(cityEles){
|
if(cityEles){
|
||||||
for(def e : cityEles) {
|
for(def e : cityEles) {
|
||||||
def href = e.attr("href")
|
def href = e.attr("href")
|
||||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
if (!href.contains("fang.lianjia")){
|
||||||
cityList << [e.text(), href]
|
cityList << [e.text(), href]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user