Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+3
-1
@@ -42,7 +42,9 @@ public enum NewResourceRecommendEnum {
|
||||
GOV("1034", "政府机构"),
|
||||
BOND("1033", "证券"),
|
||||
COST("1035", "消费金融"),
|
||||
ZIXUN_APP("1036", "业务咨询app");
|
||||
ZIXUN_APP("1036", "业务咨询app"),
|
||||
ARTICLE_APP("1037", "公众号推文"),
|
||||
MENU_APP("1038", "公众号菜单");
|
||||
|
||||
NewResourceRecommendEnum(String id, String value) {
|
||||
this.id = id;
|
||||
|
||||
+3
-1
@@ -38,7 +38,9 @@ public enum SalesLeadEnum {
|
||||
GOV("1034", "政府机构"),
|
||||
BOND("1033", "证券"),
|
||||
COST("1035", "消费金融"),
|
||||
ZIXUN_APP("1036", "业务咨询app");
|
||||
ZIXUN_APP("1036", "业务咨询app"),
|
||||
ARTICLE_APP("1037", "公众号推文"),
|
||||
MENU_APP("1038", "公众号菜单");
|
||||
|
||||
SalesLeadEnum(String id, String value) {
|
||||
this.id = id;
|
||||
|
||||
+25
-25
@@ -36,35 +36,35 @@ class ExportDistrictRongHeTest extends BaseBatch{
|
||||
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd"), new_dists)
|
||||
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
|
||||
|
||||
// if(curDate.toString().endsWith("10")){
|
||||
if(curDate.toString().endsWith("10")){
|
||||
|
||||
// exportEvaluateRate()
|
||||
// exportBigdataBuildprojFjw()
|
||||
// exportBigdataCommlayoutpicFjw()
|
||||
//
|
||||
//
|
||||
// exportBigdataCommunityGJRH()/
|
||||
// exportBigdataBuildingFJW()//
|
||||
// exportBigdataCommfeeGj()//
|
||||
//
|
||||
// exportBigdataCommbuildstruGJ()//
|
||||
// exportBigdataCommhousetypeGJ()//
|
||||
// exportBigdataCommbuildyearGj()//
|
||||
// exportBigdataCommhouseuseGJ()//
|
||||
// exportBigdataPositionGJ()//
|
||||
// exportBigdataDistrictFJW()//
|
||||
// exportBigdataAliasFJW()//
|
||||
//
|
||||
//
|
||||
// //本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
|
||||
// def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
// exportBigdataCityPrice(queryDate)
|
||||
// exportBigdataDistPrice(queryDate)
|
||||
// exportBigdataCommPrice(queryDate)
|
||||
exportEvaluateRate()
|
||||
exportBigdataBuildprojFjw()
|
||||
exportBigdataCommlayoutpicFjw()
|
||||
|
||||
|
||||
exportBigdataCommunityGJRH()/
|
||||
exportBigdataBuildingFJW()//
|
||||
exportBigdataCommfeeGj()//
|
||||
|
||||
exportBigdataCommbuildstruGJ()//
|
||||
exportBigdataCommhousetypeGJ()//
|
||||
exportBigdataCommbuildyearGj()//
|
||||
exportBigdataCommhouseuseGJ()//
|
||||
exportBigdataPositionGJ()//
|
||||
exportBigdataDistrictFJW()//
|
||||
exportBigdataAliasFJW()//
|
||||
|
||||
|
||||
//本12.10上传10,11月所有数据,23年1月需改为所有城市按日期同步一期数据
|
||||
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
|
||||
exportBigdataCityPrice(queryDate)
|
||||
exportBigdataDistPrice(queryDate)
|
||||
exportBigdataCommPrice(queryDate)
|
||||
|
||||
exportInOk()
|
||||
|
||||
// }
|
||||
}
|
||||
|
||||
SSHHelperTest jschCommand = new SSHHelperTest();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb_ronghe/output/rh_package.sh ${curDate}"));
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202303.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
@@ -29,16 +29,16 @@ class EsHouseStat {
|
||||
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
|
||||
// DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020)
|
||||
|
||||
// public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
// public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
// static {
|
||||
// dataSource.setUrl(address)
|
||||
// Properties connectionProperties = new Properties()
|
||||
// connectionProperties.put("user", "developer")
|
||||
// connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
// dataSource.setProperties(connectionProperties)
|
||||
// }
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -285,7 +285,7 @@ class FiledStrand {
|
||||
}
|
||||
|
||||
static def processCarSiteDesc(car_site){
|
||||
if(car_site && car_site.size()<=20){
|
||||
if(car_site && car_site.size()<=20 && car_site.size()>=2 && !car_site.contains("待定") && !car_site.contains("暂无")){
|
||||
return car_site.toString()
|
||||
}
|
||||
return null
|
||||
@@ -337,6 +337,82 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processBlock(def block){
|
||||
if(block){
|
||||
if(block.contains("无") || block.toString().length()>10){
|
||||
return null
|
||||
}
|
||||
return block
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processBuildType(def buildType, def estateType){
|
||||
if(buildType){
|
||||
def types = []
|
||||
if(estateType == "别墅"){
|
||||
def arrs = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院"]
|
||||
for(def s : arrs){
|
||||
if(buildType.contains(s)){
|
||||
types << s
|
||||
}
|
||||
}
|
||||
}else if(estateType == "商住"){
|
||||
def arrs = ["独栋","双拼","联排","叠加","老式别墅洋房"]
|
||||
for(def s : arrs){
|
||||
if(buildType.contains(s)){
|
||||
types << s
|
||||
}
|
||||
}
|
||||
}else{
|
||||
def arrs = ["多层","小高层","高层","超高层","SOHO","soho","LOFT","loft","酒店式公寓"]
|
||||
for(def s : arrs){
|
||||
if(buildType.contains(s)){
|
||||
types << s
|
||||
}
|
||||
}
|
||||
}
|
||||
if(types){
|
||||
return types.join(",").toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processlandProperty(val){
|
||||
if(val){
|
||||
def arrs = ["经适房","限价房","廉租房","宅基地","保障房","动迁房","配套房","商品房"]
|
||||
def property = null
|
||||
for(def s : arrs){
|
||||
if(val.contains(s)){
|
||||
property = s
|
||||
break;
|
||||
}
|
||||
}
|
||||
return property
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processBuildStructure(val) {
|
||||
if (val) {
|
||||
def buildStructure= null
|
||||
if(val.indexOf("砖木")>-1){
|
||||
buildStructure = "砖木结构"
|
||||
}else if(val.indexOf("钢结构")>-1){
|
||||
buildStructure = "钢结构";
|
||||
}else if(val.indexOf("钢、钢筋混凝土")>-1){
|
||||
buildStructure = "钢、钢筋混凝土结构";
|
||||
}else if(val.indexOf("钢筋混凝土")>-1){
|
||||
buildStructure = "钢筋混凝土结构";
|
||||
}else if(val.indexOf("混合")>-1){
|
||||
buildStructure = "混合结构";
|
||||
}
|
||||
return buildStructure
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def isRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]");
|
||||
def matcher = pattern.matcher(s)
|
||||
|
||||
+229
-98
@@ -7,9 +7,11 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -17,6 +19,7 @@ import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -31,15 +34,32 @@ import java.util.regex.Pattern
|
||||
class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
|
||||
DBCollection SEAWEEDMAP_COLL = MongoUtils.getCollectionByName("fangjia", "seaweedmap", "192.168.0.235", 27018)
|
||||
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
|
||||
|
||||
//,"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"
|
||||
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
//
|
||||
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
|
||||
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
||||
|
||||
//"alias":"alias",
|
||||
def fieldsMap = ["estate_charge":"property_fee","estate_type2":"estate_type2","build_date":"build_year","estate_company":"property_company","developer":"developer","virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households","car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address"]//key:seaweed表字段 value:es表字段
|
||||
//ownership_type:权属类型 为商品房、经适房、动迁房等
|
||||
//house_property 产权性质:国有产权 "house_property":"网站没有",
|
||||
def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer",
|
||||
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households",
|
||||
"car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure",
|
||||
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
|
||||
"loop_line_info":"loop_line"]//key:seaweed表字段 value:es表字段
|
||||
|
||||
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
|
||||
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
|
||||
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
|
||||
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
|
||||
|
||||
|
||||
final static def EMAILS = [
|
||||
"gaojuan@fangjia.com"
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictFieldAddBatch(), args)
|
||||
@@ -47,21 +67,39 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = ["漳州","福州","常州","龙岩","厦门","扬州","徐州","镇江","南通","泰州","连云港","盐城","淮安","宿迁","南京","苏州","合肥","无锡","上海","北京","昆明","琼海","呼和浩特","南平","青岛","常德","黄石","黄冈","汉中","安康","榆林","南充","泸州","资阳","克拉玛依","荆门","铜川","内江","达州","巴中","鄂州","十堰","遂宁","随州","广安","荆州","咸宁","阿克苏","喀什","昌吉","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","恩施","咸阳","宜宾","自贡","攀枝花","延安","德阳","西安","重庆","成都","绵阳","乌鲁木齐","宜昌","武汉","乐山","眉山","宝鸡","襄阳","孝感","拉萨","深圳","三门峡","杭州","怀化","聊城","图木舒克","渭南","金华","宁德","三明","泉州","莆田","营口","牡丹江","盘锦","本溪","铁岭","齐齐哈尔","四平","辽源","锦州","佳木斯","白山","绥化","鸡西","七台河","白城","朝阳","双鸭山","通化","鹤岗","黑河","阜新","抚顺","葫芦岛","伊春","松原","大兴安岭","延边","吉林","沈阳","大连","长春","哈尔滨","鞍山","大庆","丹东"] //"广州", "成都","杭州", "深圳", "重庆","苏州","上海","北京"
|
||||
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
|
||||
def analysRets = []
|
||||
|
||||
def rh_analysRets = []
|
||||
def fields = fieldsMap.keySet() as List
|
||||
def headers = ["来源", "城市"]
|
||||
fields.each{it->
|
||||
headers << fieldsNameMap.get(it)
|
||||
}
|
||||
|
||||
headers << "抓取量"
|
||||
headers << "匹配量"
|
||||
headers << "0.9分以上量"
|
||||
headers << "匹配率"
|
||||
headers << "0.9以上匹配率"
|
||||
// analysRets << headers
|
||||
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
||||
// addFieldFromRonghe(c, totalCount)
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)
|
||||
if(totalCount){
|
||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)
|
||||
}
|
||||
}
|
||||
println analysRets.toString()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
|
||||
sendStatEmail(headers, analysRets, rh_analysRets)
|
||||
// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
|
||||
}
|
||||
|
||||
def addFieldFromMatch(c, totalCount, analysRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findEsDatas(c, esWebSource)
|
||||
def matchRets1 = matchApi(esRets)
|
||||
@@ -80,7 +118,18 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
|
||||
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
|
||||
}
|
||||
analysRets << [seaweedmapSource, c, precentMap.get("property_fee"), esRets.size(), matchRets1.size(), matchRets2.size(), pre1, pre2]
|
||||
def analyRet = []
|
||||
analyRet << seaweedmapSource
|
||||
analyRet << c
|
||||
fields.each{key->
|
||||
analyRet << precentMap.get(key)
|
||||
}
|
||||
analyRet << esRets.size()
|
||||
analyRet << matchRets1.size()
|
||||
analyRet << matchRets2.size()
|
||||
analyRet << pre1
|
||||
analyRet << pre2
|
||||
analysRets << analyRet
|
||||
}
|
||||
return analysRets;
|
||||
}
|
||||
@@ -88,7 +137,8 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
|
||||
|
||||
|
||||
def addFieldFromRonghe(c, totalCount){
|
||||
def addFieldFromRonghe(c, totalCount, rh_analysRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
sourceMap.each{seaweedmapSource,esWebSource->
|
||||
def rhRets = findRongheDists(c, seaweedmapSource)
|
||||
def esRets = findEsDistrict(c, esWebSource)
|
||||
@@ -99,13 +149,35 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "----------------------------------------------------------------------------------"
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
if(esRets){
|
||||
pre1 = DataHandleUtils.getDoubleRound(2, rhRets.size()*100/esRets.size())
|
||||
pre2 = DataHandleUtils.getDoubleRound(2, esRets.size()*100/esRets.size())
|
||||
}
|
||||
def analyRet = []
|
||||
analyRet << seaweedmapSource
|
||||
analyRet << c
|
||||
fields.each{key->
|
||||
analyRet << precentMap.get(key)
|
||||
}
|
||||
analyRet << esRets.size()
|
||||
analyRet << rhRets.size()
|
||||
analyRet << esRets.size()
|
||||
analyRet << pre1
|
||||
analyRet << pre2
|
||||
rh_analysRets << analyRet
|
||||
}
|
||||
return rh_analysRets;
|
||||
}
|
||||
|
||||
def processFieldsFromRh(c, rhRets, esRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
def fieldValueMap = [:]
|
||||
for(def rhDist : rhRets){
|
||||
if(rhDist.commid && rhDist.commid!="null"){
|
||||
continue
|
||||
}
|
||||
def dist = CommonDatautils.findDistrictFieldById(rhDist.commid, fields)
|
||||
if(!dist){
|
||||
println c+" "+rhDist.commid+" 不存在"
|
||||
@@ -116,26 +188,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
println c+" "+rhDist.id+" 不存在[ES]"
|
||||
continue
|
||||
}
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = dist.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = esDbo.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield == "build_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def newValues = strandField(c, dist, esDbo)
|
||||
if(newValues){
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
@@ -149,77 +202,15 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def processFieldsFromMatch(c,matchRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
fields << "city"
|
||||
fields << "estate_type2"
|
||||
def fieldValueMap = [:]
|
||||
for(def matchDist : matchRets){
|
||||
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
println c+" "+matchDist.match_id+" 不存在"
|
||||
continue
|
||||
}
|
||||
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = dist.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = matchDist.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield == "build_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
if(seaweedfield =="virescence_percent"){
|
||||
newVal = FiledStrand.processVirescencePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="cubage_percent"){
|
||||
newVal = FiledStrand.processCubagePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
|
||||
newVal = FiledStrand.processArea(esVal)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield =="household_count"){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
if(seaweedfield =="loudong_count"){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="car_site_count"){
|
||||
newVal = FiledStrand.processCarSite(esVal)
|
||||
}
|
||||
if(seaweedfield =="car_site"){
|
||||
newVal = FiledStrand.processCarSiteDesc(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="property_life"){
|
||||
newVal = FiledStrand.processPropertyLife(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="alias"){
|
||||
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
|
||||
}
|
||||
|
||||
if(seaweedfield =="address"){
|
||||
newVal = FiledStrand.processAddress(esVal, dist.city)
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def newValues = strandField(c, dist, matchDist)
|
||||
if(newValues){
|
||||
println dist._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
@@ -229,6 +220,89 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
return fieldValueMap;
|
||||
}
|
||||
|
||||
def strandField(c,dist, matchDist){
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = dist.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = matchDist.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
if(seaweedfield =="virescence_percent"){
|
||||
newVal = FiledStrand.processVirescencePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="cubage_percent"){
|
||||
newVal = FiledStrand.processCubagePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
|
||||
newVal = FiledStrand.processArea(esVal)
|
||||
if(newVal){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(seaweedfield =="household_count"){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
if(seaweedfield =="loudong_count"){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="car_site_count"){
|
||||
newVal = FiledStrand.processCarSite(esVal)
|
||||
}
|
||||
if(seaweedfield =="car_site"){
|
||||
newVal = FiledStrand.processCarSiteDesc(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="property_life"){
|
||||
newVal = FiledStrand.processPropertyLife(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="alias"){
|
||||
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
|
||||
}
|
||||
|
||||
if(seaweedfield =="address"){
|
||||
newVal = FiledStrand.processAddress(esVal, dist.city)
|
||||
}
|
||||
if(seaweedfield =="block"){
|
||||
newVal = FiledStrand.processBlock(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_type"){
|
||||
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
if(seaweedfield == "land_property"){
|
||||
newVal = FiledStrand.processlandProperty(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_structure"){
|
||||
newVal = FiledStrand.processBuildStructure(esVal)
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return newValues;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findRongheDists(def city, def source){
|
||||
@@ -285,9 +359,9 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
def precentMap = [:]
|
||||
fieldsMap.each() {key,skey->
|
||||
if(countMap.containsKey(key)) {
|
||||
precentMap.put(skey, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
|
||||
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
|
||||
}else {
|
||||
precentMap.put(skey, 0)
|
||||
precentMap.put(key, 0)
|
||||
}
|
||||
}
|
||||
return precentMap;
|
||||
@@ -414,4 +488,61 @@ class DistrictFieldAddBatch extends BaseBatch{
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
def sendStatEmail(headers, analysRets, rh_nalysRets){
|
||||
Date dateTime = DateUtils.mins(1);
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("yyyy.MM.dd");
|
||||
StringBuffer sb = new StringBuffer();
|
||||
if(analysRets){
|
||||
sb.append("<table border='1' style='border-collapse:collapse; border-spacing:0'>")
|
||||
sb.append("<tr><font style='size:15px;color:grey;'>字段补充情况分析</tr>")
|
||||
sb.append("<tr style='color:black;font-size:18px;cellspacing:1px;'>")
|
||||
headers.each{it->
|
||||
if(it=="来源" || it=="城市"){
|
||||
sb.append("<th style='width:80px;'>"+ it+"</th>")
|
||||
}else{
|
||||
sb.append("<th >"+ it+"</th>")
|
||||
}
|
||||
}
|
||||
sb.append("</tr>")
|
||||
|
||||
for(def ret : analysRets){
|
||||
sb.append("<tr style='text-align:center;font-color:gray;cellspacing:1px;'>");
|
||||
ret.each{it->
|
||||
sb.append("<td>" + it + "</td>")
|
||||
}
|
||||
sb.append("</tr>");
|
||||
}
|
||||
sb.append("</table><br/>");
|
||||
}
|
||||
|
||||
|
||||
StringBuffer rh_sb = new StringBuffer();
|
||||
if(rh_nalysRets){
|
||||
rh_sb.append("<table border='1' style='border-collapse:collapse; border-spacing:0'>")
|
||||
rh_sb.append("<tr><font style='size:15px;color:grey;'>字段补充情况分析</tr>")
|
||||
rh_sb.append("<tr style='color:black;font-size:18px;cellspacing:1px;'>")
|
||||
headers.each{it->
|
||||
if(it=="来源" || it=="城市"){
|
||||
rh_sb.append("<th style='width:80px;'>"+ it+"</th>")
|
||||
}else{
|
||||
rh_sb.append("<th >"+ it+"</th>")
|
||||
}
|
||||
}
|
||||
rh_sb.append("</tr>")
|
||||
|
||||
for(def ret : rh_nalysRets){
|
||||
rh_sb.append("<tr style='text-align:center;font-color:gray;cellspacing:1px;'>");
|
||||
ret.each{it->
|
||||
rh_sb.append("<td>" + it + "</td>")
|
||||
}
|
||||
rh_sb.append("</tr>");
|
||||
}
|
||||
rh_sb.append("</table><br/>");
|
||||
}
|
||||
|
||||
for (def email : EMAILS) {
|
||||
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 字段提升率统计
|
||||
*/
|
||||
DBCollection CITY_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", MongoUtils.MongoDB_Domain_mongo);
|
||||
|
||||
def fieldsMap = ["district_count":"小区数量","name":"名称" ,"region":"区域" ,"block":"板块" ,"address":"地址" ,"estateType2":"物业类型" ,"householdCount":"户数" ,"openDate":"开盘时间" ,"liveDate":"入住时间" ,"buildDate":"建筑年代" ,"allArea":"占地面积" ,"buildArea":"建筑面积" ,"buildType":"建筑类型" ,"developer":"开发商" ,"estateCompany":"物业公司" ,"cubagePercent":"容积率" ,"virescencePercent":"绿化率" ,"carSite":"停车位" ,"estateCharge":"物业费" ,"floorplanPics":"户型图" ,"outPics":"外景图" ,"finishBuildingDate":"竣工时间" ,"businessQuarter":"商圈" ,"startBuildingDate":"开建时间" ,"loopLineInfo":"环线" ,"propertyLife":"产权年限" ,"houseProperty":"产权性质" ,"licence":"预售许可证" ,"buildStructure":"建筑结构" ,"lat":"百度纬度" ,"pics":"图片" ,"streetCommunity":"街道" ,"schools":"学校" ,"houseStructure":"房屋结构" ,"alias":"别名" ,"addrAlias":"地址别名" ,"mainTypeArea":"主力面积" ,"loudongCount":"楼栋数" ,"latLng" :"高德经度" ,"rentPrice":"租金" ]
|
||||
def weeknd = DateUtils.getFirstDayOfWeek(new Date())
|
||||
def lastWeeknd = DateUtils.getFirstDayOfWeek(new Date()-7)
|
||||
|
||||
|
||||
def weekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", weeknd))
|
||||
def lastWeekendDatas = CITY_INFO_CONN.find(new BasicDBObject("weekend", lastWeeknd))
|
||||
|
||||
def weekendMapDatas = findDataByCity(weekendDatas)
|
||||
def lastWeekendMapDatas = findDataByCity(lastWeekendDatas)
|
||||
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
|
||||
def fields = fieldsMap.keySet() as List
|
||||
def rets = []
|
||||
def headers = []
|
||||
headers << "城市"
|
||||
fields.each{k->
|
||||
headers << fieldsMap.get(k)
|
||||
}
|
||||
rets << headers
|
||||
for(def c : citys){
|
||||
def weekendDetailMap = weekendMapDatas.get(c)
|
||||
def lastWeekendDetailMap = lastWeekendMapDatas.get(c)
|
||||
def ret = []
|
||||
ret << c
|
||||
fields.each{key->
|
||||
if(weekendDetailMap){
|
||||
def v1 = weekendDetailMap.get(key)
|
||||
def v2 = lastWeekendDetailMap.get(key)
|
||||
if(v1 && v2){
|
||||
ret << DataHandleUtils.getDoubleRound(2, v1-v2)
|
||||
}else{
|
||||
ret << "-"
|
||||
}
|
||||
}else{
|
||||
ret << "-"
|
||||
}
|
||||
}
|
||||
rets << ret
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\小区\\字段提升率"+DateUtils.formatDateByPattern(weeknd,"yyyy-MM-dd")+".xlsx", rets, "字段提升率统计")
|
||||
|
||||
|
||||
|
||||
def findDataByCity(weekendDatas){
|
||||
def map = [:]
|
||||
for(def d : weekendDatas){
|
||||
d.district_detail.district_count = d.existing_district_count
|
||||
map.put(d.city, d.district_detail)
|
||||
}
|
||||
return map
|
||||
}
|
||||
+37
-2
@@ -1,7 +1,9 @@
|
||||
package com.fangjia.script.gj.es
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -19,11 +21,44 @@ class EsMonthHouseStat {
|
||||
def task = new EsMonthHouseStat()
|
||||
// task.run()
|
||||
// task.tradeRun()
|
||||
task.houseRun()
|
||||
// task.houseRun()
|
||||
|
||||
// task.statHouseUpdateCount()
|
||||
|
||||
// task.statDistHouse()
|
||||
task.statRent()
|
||||
}
|
||||
|
||||
def statRent(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\租金.xlsx", { d ->
|
||||
index++
|
||||
def region = RegionData.formatRegion(d['A'], d['B'])
|
||||
if(region){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
if(!dist){
|
||||
dist = OpenApiUtils.matchDistrictByAddr(d['A'], d['C'], region)
|
||||
if(dist){
|
||||
dist = CommonDatautils.findDistrictById(dist.id)
|
||||
}
|
||||
}
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
def distCount = findHouseCount(dist.city, dist.region, alias, 20220501, 20230501)
|
||||
d['G'] = distCount
|
||||
d['E'] = dist.region
|
||||
d['F'] = dist.name
|
||||
println index+" "+ d['C'] +" "+distCount
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def statDistHouse(){
|
||||
@@ -172,7 +207,7 @@ class EsMonthHouseStat {
|
||||
def total = 0
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
String sql = "select count(*) as c from latest_seawater where cat='rent' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and root='lianjia.com'"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
|
||||
+36
-8
@@ -69,7 +69,7 @@ public class BeiKeCrawlByChrome {
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
getLogin(driver)
|
||||
driver = getLogin()
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
@@ -108,15 +108,32 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && s.contains("无法访问此网站")){
|
||||
driver = getLogin()
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
}
|
||||
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
for(def m : citys){
|
||||
def city = m[0]
|
||||
def url = m[1]
|
||||
driver.get(url+"/chengjiao/")
|
||||
def newUrl = url+"/chengjiao/"
|
||||
driver.get(newUrl)
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -220,6 +237,9 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if(roomStr && !roomStr.contains("室")){
|
||||
roomStr = arrs[2]
|
||||
}
|
||||
if (roomStr) {
|
||||
if (roomStr.indexOf("室") > 0) {
|
||||
roomStr = roomStr.substring(0, roomStr.indexOf("室"))
|
||||
@@ -235,7 +255,13 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
def areaStr = arrs[2]
|
||||
if(areaStr && !areaStr.contains("平米")){
|
||||
if(arrs.size()>=4){
|
||||
areaStr = arrs[3]
|
||||
}
|
||||
}
|
||||
areaStr = areaStr.replace("平米","")
|
||||
if (areaStr && areaStr.isNumber()) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
@@ -254,7 +280,7 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
|
||||
if(detail.total_price){
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
detail.url = detailUrl
|
||||
def dbo = findTrade(detail)
|
||||
@@ -404,15 +430,16 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = "https:"+e.attr("href")
|
||||
if (citys.contains(e.text())){
|
||||
// if (citys.contains(e.text())){
|
||||
list << [e.text(), href]
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
def getLogin(){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
@@ -432,6 +459,7 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(10000)
|
||||
return driver
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+28
-9
@@ -122,7 +122,7 @@ public class LianjiaCrawlByChrome {
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
// cityMap = cityMap.reverse()
|
||||
getLogin(driver)
|
||||
driver = getLogin()
|
||||
crawlData(driver, cityMap)
|
||||
|
||||
|
||||
@@ -191,10 +191,12 @@ public class LianjiaCrawlByChrome {
|
||||
for(def ret : cityList){
|
||||
def city = ret[0]
|
||||
def cityUrl = ret[1]
|
||||
def newUrl = cityUrl+"chengjiao/"
|
||||
driver.get(newUrl)
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
driver.get(cityUrl+"chengjiao/")
|
||||
Thread.sleep(10000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def doc = checkDocument(driver, newUrl);
|
||||
|
||||
def regions = JQueryUtils.select(".position dl:contains(位置)+dl a", doc)
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
@@ -240,6 +242,21 @@ public class LianjiaCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def checkDocument(driver, url){
|
||||
def s = driver.getPageSource()
|
||||
def flag = true
|
||||
while(flag){
|
||||
if(s && s.contains("无法访问此网站")){
|
||||
driver = getLogin()
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
s = driver.getPageSource()
|
||||
}else{
|
||||
flag = false
|
||||
}
|
||||
}
|
||||
return Jsoup.parse(s);
|
||||
}
|
||||
|
||||
def getPageDeail(driver, city, cityUrl, regionText){
|
||||
def stop = false
|
||||
@@ -278,7 +295,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
def parse(city,e, regionText){
|
||||
try {
|
||||
def title = e.select(".title a").text()
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
def arrs = title.split(" ")
|
||||
@@ -318,7 +335,7 @@ public class LianjiaCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if (price && price.isNumber()) {
|
||||
detail.total_price = (int)(price.toDouble() * 10000)
|
||||
}
|
||||
@@ -333,7 +350,7 @@ public class LianjiaCrawlByChrome {
|
||||
detail.source = "链家在线"
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
|
||||
if(detail.total_price){
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def dbo = findTrade(detail)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
@@ -482,15 +499,16 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityList << [e.text(), href]
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return cityList
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
def getLogin(){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
Thread.sleep(10000)
|
||||
@@ -510,6 +528,7 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(20000)
|
||||
return driver
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-9
@@ -27,19 +27,11 @@ class OpenUriUserIdCountDay extends BaseBatch {
|
||||
runBatch(new OpenUriUserIdCountDay(), args)
|
||||
}
|
||||
final static def EMAILS = [
|
||||
"joe@fangjia.com",
|
||||
"wangwenyan@fangjia.com",
|
||||
"xiaohai@fangjia.com",
|
||||
"caoyuyan@fangjia.com",
|
||||
"jiangxiaochun@fangjia.com",
|
||||
"gaojuan@fangjia.com",
|
||||
'shijing@fangjia.com',
|
||||
'zhongjingyi@fangjia.com',
|
||||
"duweiliang@fangjia.com",
|
||||
"sunjian@fangjia.com",
|
||||
"yaojingmeng@fangjia.com",
|
||||
"xuqi@fangjia.com",
|
||||
"wangpeili@fangjia.com"
|
||||
"gaojuan@fangjia.com"
|
||||
]
|
||||
|
||||
@Override
|
||||
|
||||
Reference in New Issue
Block a user