Merge branch 'script_gj' into 'master'

update

See merge request !266
This commit is contained in:
2022-02-28 10:13:25 +08:00
6 changed files with 110 additions and 58 deletions
@@ -25,21 +25,21 @@ class ExportOracleData {
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
//不会变
exportEvaluationRateData()
exportBigdataPoiData()
exportBigdataDistrictData()
exportGuidedPrice()//随人工改变
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
// //处理小区情况
exportBigdataCommunity()
exportCommunityStaging()
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
//
// exportGuidedPrice()//随人工改变
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
//
//// //处理小区情况
// exportBigdataCommunity()
// exportCommunityStaging()
// exportCommunityBuileType()
//
// exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-01-19")
exportBigdataDeal("2022-01-01")
// exportBigdataDeal("2022-01-01")
//计算结果相关
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
}
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
def testMap = findCaluateCityPPE(sqlInstance236, 202202)
ExcelUtils.modify(fileName,{d->
if(d['C']){
def oldcount = dealMap.get(d['C'])
d['V'] = oldcount
def casecount = testMap.get(d['C'])
d['W'] = casecount
println d
}
},0)
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['V'] = oldcount
// def casecount = testMap.get(d['C'])
// d['W'] = casecount
// println d
// }
//},0)
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -782,6 +782,23 @@ def getStreetInfo(def city, def glocation){
//}
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据")
def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1)
def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1)
def keys = []
for(def d : datas2){
keys << d['A']+"##"+d['C']+"##"+d['B']
}
def index = 0
for(def d : datas1){
def k = d['A']+"##"+d['C']+"##"+d['B']
if(!keys.contains(k)){
index ++
println index +" "+k
}
}
@@ -7,11 +7,11 @@ email.setSSL(true);
email.setHostName("smtp.exmail.qq.com");
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码
email.addTo("gaojuan@fangjia.com"); //接收方
email.setFrom("service@fangjia.com"); //发送方
email.setSubject("test"); //标题
email.setContent("你?", "text/html");
email.setContent("测试", "text/html");
//email.setHostName("smtp.exmail.qq.com");
//email.setSSL(true);
@@ -37,7 +37,10 @@ class CrawlDistrictIdSelect {
}
def run(){
def datas = findCrawlData()
def source = "58同城"
def root = "58.com"
def shortCityMap = findCitySeeds(root)
def datas = findCrawlData(source, shortCityMap, root)
def list = []
datas.each {k,v->
if(v){
@@ -58,15 +61,32 @@ class CrawlDistrictIdSelect {
}
}
def findCrawlData(){
def findCitySeeds(root){
def q = new BasicDBObject()
q.put("city","北京")
q.put("source", "搜房")
q.put("root", root)
q.put("city", "福州")
def dbcursor = districtseedsColl.find(q)
def cityMap = [:]
for(def d : dbcursor){
if(!cityMap.keySet().contains(d.city)){
cityMap.put(d.city, d.city_short)
}
}
return cityMap
}
def findCrawlData(def source, shortCityMap, def root){
def q = new BasicDBObject()
q.put("city","福州")
q.put("source", source)
q.put("s_date",202112)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = "esf.fang.com/loupan/"
def prefix = "xiaoqu/"
def keyList =new HashSet()
def index = 0
for(def db : dbcursor){
def url = null
if(!db.url){
@@ -76,36 +96,42 @@ class CrawlDistrictIdSelect {
}else{
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".htm"))
if(url){
def district_id = ""
if(url.contains(prefix)){
district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1)
}else{
district_id = url;
}
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
if(!shortCity){
shortCity = "bj"
}
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
def ids = retMap.get(db.city)
if(!ids){
ids = []
retMap.put(key, ids)
retMap.put(db.city, ids)
}
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "fang.com"
map.city_short= shortCity
map.city = db.city
map.region = db.region
map.district_id = district_id
map.source_url = url
ids << map
keyList << key
if(!keyList.contains(db.city+"#"+district_id)){
def count = findIsExists(db.city, district_id, root)
if(!count){
def map = new BasicDBObject()
map.root = "58.com"
map.city_short= shortCityMap.get(db.city)
map.city = db.city
map.region = db.region
map.district_id = district_id
map.source_url = url
map.name = db.name
ids << map
}else{
index++
println "[${index}]已存在:"+db.city +" "+db.name
}
keyList << db.city+"#"+district_id
}else{
println "重复:"+key
println "重复:"+db.city+"#"+db.name
}
}else{
println url +" 链接不合要求"
@@ -113,4 +139,13 @@ class CrawlDistrictIdSelect {
}
return retMap
}
def findIsExists(city, district_id, root){
def q = new BasicDBObject()
q.put("city",city)
q.put("district_id", district_id)
q.put("root",root)
def count = districtseedsColl.count(q)
return count
}
}
@@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch {
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "G00jia123456");
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
email.addTo(mail.receiverEmail);
email.setFrom("service@fangjia.com");//发送方
email.setSubject(mail.subject); //标题
@@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch {
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "G00jia123456");
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
email.addTo(mail.receiverEmail);
email.setFrom("service@fangjia.com");
email.setSubject(mail.subject);
@@ -181,7 +181,7 @@ class Esf58Crawl extends BaseBatch{
// "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361",
// "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575"
def cityList = [
"重庆":"cq|37","保定":"bd|424","信阳":"xy|8694","南京":"nj|172","南宁":"nn|845","厦门":"xm|606","青岛":"qd|122","哈尔滨":"hrb|202","天津":"tj|18","太原":"ty|740","宁德":"nd|7951","宜昌":"yc|858","成都":"cd|102","惠州":"huizhou|722","扬州":"yz|637","昆明":"km|541","杭州":"hz|79","武汉":"wh|158","泰州":"taizhou|693","洛阳":"luoyang|556","深圳":"sz|4","石家庄":"sjz|241","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572",
"福州":"fz|304"
]
static main(args) {