Merge branch 'script_gj' into 'master'

100-50

See merge request !683
This commit is contained in:
2023-07-14 22:24:07 +08:00
5 changed files with 10 additions and 7 deletions
@@ -239,7 +239,10 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据") def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
if(files){ if(files){
for(def filepath : files){ for(def filepath : files){
def rets = read(filepath); def filename = filepath.getPath()
def rets = read(filename);
def date = filename.substring(filename.toString().lastIndexOf("_")+1, filename.toString().indexOf("."))
println date +" "+rets.size()
} }
} }
@@ -73,7 +73,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
// addFieldValue() // addFieldValue()
updateErrorField() // updateErrorField()
export() export()
test() test()
// addOffice() // addOffice()
@@ -120,7 +120,7 @@ class OfficeFieldCleanBatch extends BaseBatch{
} }
def test(){ def test(){
def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格租金.xlsx",0 , 1) def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格 (2).xlsx",0 , 1)
def map = [:] def map = [:]
for(def d : datas){ for(def d : datas){
map.put(d['A']+"#"+d['B']+'#'+d['C'], d) map.put(d['A']+"#"+d['B']+'#'+d['C'], d)
@@ -29,7 +29,7 @@ class RemoveHouse extends BaseBatch{
def rets = [] def rets = []
for(def d : dbcursor){ for(def d : dbcursor){
rets << d._id rets << d._id
if(rets.size()==100){ if(rets.size()==50){
def delquery = new BasicDBObject() def delquery = new BasicDBObject()
delquery.put("_id", new BasicDBObject('$in', rets)) delquery.put("_id", new BasicDBObject('$in', rets))
SEABIRD_COLL.remove(delquery) SEABIRD_COLL.remove(delquery)
@@ -465,9 +465,9 @@ public class BeiKeCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles){ for(def e : cityEles){
def href = "https:"+e.attr("href") def href = "https:"+e.attr("href")
if (citys.contains(e.text())){ // if (citys.contains(e.text())){
list << [e.text(), href] list << [e.text(), href]
} // }
} }
} }
return list return list
@@ -570,7 +570,7 @@ public class LianjiaCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles) { for(def e : cityEles) {
def href = e.attr("href") def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){ if (!href.contains("fang.lianjia")){
cityList << [e.text(), href] cityList << [e.text(), href]
} }
} }