diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 20e49d87c..e499c32ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,21 +25,21 @@ class ExportOracleData { def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //不会变 - exportEvaluationRateData() - exportBigdataPoiData() - exportBigdataDistrictData() - - exportGuidedPrice()//随人工改变 - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 - -// //处理小区情况 - exportBigdataCommunity() - exportCommunityStaging() - exportCommunityBuileType() - - exportThirdPartyPrice2(lastMonth) +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() +// +// exportGuidedPrice()//随人工改变 +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 +// +//// //处理小区情况 +// exportBigdataCommunity() +// exportCommunityStaging() +// exportCommunityBuileType() +// +// exportThirdPartyPrice2(lastMonth) exportBigdataForsale("2022-01-19") - exportBigdataDeal("2022-01-01") +// exportBigdataDeal("2022-01-01") //计算结果相关 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 146c1f07b..0b63403ce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01") -def testMap = findCaluateCityPPE(sqlInstance236, 202202) -ExcelUtils.modify(fileName,{d-> - if(d['C']){ - def oldcount = dealMap.get(d['C']) - d['V'] = oldcount - def casecount = testMap.get(d['C']) - d['W'] = casecount - println d - } -},0) +//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01") +//def testMap = findCaluateCityPPE(sqlInstance236, 202202) +//ExcelUtils.modify(fileName,{d-> +// if(d['C']){ +// def oldcount = dealMap.get(d['C']) +// d['V'] = oldcount +// def casecount = testMap.get(d['C']) +// d['W'] = casecount +// println d +// } +//},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -782,6 +782,23 @@ def getStreetInfo(def city, def glocation){ //} //ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") +def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) +def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) + +def keys = [] +for(def d : datas2){ + keys << d['A']+"##"+d['C']+"##"+d['B'] +} + +def index = 0 +for(def d : datas1){ + def k = d['A']+"##"+d['C']+"##"+d['B'] + if(!keys.contains(k)){ + index ++ + println index +" "+k + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestEmail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestEmail.groovy index a0921b0c4..4e66c4284 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestEmail.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestEmail.groovy @@ -7,11 +7,11 @@ email.setSSL(true); email.setHostName("smtp.exmail.qq.com"); email.setSslSmtpPort("465"); email.setCharset("GBK") -email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码 +email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码 email.addTo("gaojuan@fangjia.com"); //接收方 email.setFrom("service@fangjia.com"); //发送方 email.setSubject("test"); //标题 -email.setContent("你?", "text/html"); +email.setContent("测试", "text/html"); //email.setHostName("smtp.exmail.qq.com"); //email.setSSL(true); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index 71142969a..0470fa957 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -37,7 +37,10 @@ class CrawlDistrictIdSelect { } def run(){ - def datas = findCrawlData() + def source = "58同城" + def root = "58.com" + def shortCityMap = findCitySeeds(root) + def datas = findCrawlData(source, shortCityMap, root) def list = [] datas.each {k,v-> if(v){ @@ -58,15 +61,32 @@ class CrawlDistrictIdSelect { } } - def findCrawlData(){ + def findCitySeeds(root){ def q = new BasicDBObject() - q.put("city","北京") - q.put("source", "搜房") + q.put("root", root) + q.put("city", "福州") + + def dbcursor = districtseedsColl.find(q) + def cityMap = [:] + for(def d : dbcursor){ + if(!cityMap.keySet().contains(d.city)){ + cityMap.put(d.city, d.city_short) + } + } + return cityMap + + } + + def findCrawlData(def source, shortCityMap, def root){ + def q = new BasicDBObject() + q.put("city","福州") + q.put("source", source) q.put("s_date",202112) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = "esf.fang.com/loupan/" + def prefix = "xiaoqu/" def keyList =new HashSet() + def index = 0 for(def db : dbcursor){ def url = null if(!db.url){ @@ -76,36 +96,42 @@ class CrawlDistrictIdSelect { }else{ url = db.url } - if(url && url.contains(prefix) ){ - def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".htm")) + + if(url){ + def district_id = "" + if(url.contains(prefix)){ + district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1) + }else{ + district_id = url; + } if(district_id=="0"){ continue } - def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix)) - if(!shortCity){ - shortCity = "bj" - } - def key = shortCity +"#"+district_id - def ids = retMap.get(key) + def ids = retMap.get(db.city) if(!ids){ ids = [] - retMap.put(key, ids) + retMap.put(db.city, ids) } - - if(!keyList.contains(key)){ - def map = new BasicDBObject() - map.root = "fang.com" - map.city_short= shortCity - map.city = db.city - map.region = db.region - map.district_id = district_id - map.source_url = url - ids << map - - keyList << key + if(!keyList.contains(db.city+"#"+district_id)){ + def count = findIsExists(db.city, district_id, root) + if(!count){ + def map = new BasicDBObject() + map.root = "58.com" + map.city_short= shortCityMap.get(db.city) + map.city = db.city + map.region = db.region + map.district_id = district_id + map.source_url = url + map.name = db.name + ids << map + }else{ + index++ + println "[${index}]已存在:"+db.city +" "+db.name + } + keyList << db.city+"#"+district_id }else{ - println "重复:"+key + println "重复:"+db.city+"#"+db.name } }else{ println url +" 链接不合要求" @@ -113,4 +139,13 @@ class CrawlDistrictIdSelect { } return retMap } + + def findIsExists(city, district_id, root){ + def q = new BasicDBObject() + q.put("city",city) + q.put("district_id", district_id) + q.put("root",root) + def count = districtseedsColl.count(q) + return count + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy index 21f508609..f43bda14e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy @@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "G00jia123456"); + email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com");//发送方 email.setSubject(mail.subject); //标题 @@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "G00jia123456"); + email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com"); email.setSubject(mail.subject); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index bc691499b..5fb2fef63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -181,7 +181,7 @@ class Esf58Crawl extends BaseBatch{ // "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", // "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" def cityList = [ - "重庆":"cq|37","保定":"bd|424","信阳":"xy|8694","南京":"nj|172","南宁":"nn|845","厦门":"xm|606","青岛":"qd|122","哈尔滨":"hrb|202","天津":"tj|18","太原":"ty|740","宁德":"nd|7951","宜昌":"yc|858","成都":"cd|102","惠州":"huizhou|722","扬州":"yz|637","昆明":"km|541","杭州":"hz|79","武汉":"wh|158","泰州":"taizhou|693","洛阳":"luoyang|556","深圳":"sz|4","石家庄":"sjz|241","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", + "福州":"fz|304" ] static main(args) {