+14
-14
@@ -25,21 +25,21 @@ class ExportOracleData {
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
//不会变
|
||||
exportEvaluationRateData()
|
||||
exportBigdataPoiData()
|
||||
exportBigdataDistrictData()
|
||||
|
||||
exportGuidedPrice()//随人工改变
|
||||
exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
|
||||
// //处理小区情况
|
||||
exportBigdataCommunity()
|
||||
exportCommunityStaging()
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
// exportEvaluationRateData()
|
||||
// exportBigdataPoiData()
|
||||
// exportBigdataDistrictData()
|
||||
//
|
||||
// exportGuidedPrice()//随人工改变
|
||||
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策
|
||||
//
|
||||
//// //处理小区情况
|
||||
// exportBigdataCommunity()
|
||||
// exportCommunityStaging()
|
||||
// exportCommunityBuileType()
|
||||
//
|
||||
// exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-01-19")
|
||||
exportBigdataDeal("2022-01-01")
|
||||
// exportBigdataDeal("2022-01-01")
|
||||
|
||||
|
||||
//计算结果相关
|
||||
|
||||
+29
-12
@@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
}
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
|
||||
def testMap = findCaluateCityPPE(sqlInstance236, 202202)
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['C']){
|
||||
def oldcount = dealMap.get(d['C'])
|
||||
d['V'] = oldcount
|
||||
def casecount = testMap.get(d['C'])
|
||||
d['W'] = casecount
|
||||
println d
|
||||
}
|
||||
},0)
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
|
||||
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
|
||||
//ExcelUtils.modify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// def oldcount = dealMap.get(d['C'])
|
||||
// d['V'] = oldcount
|
||||
// def casecount = testMap.get(d['C'])
|
||||
// d['W'] = casecount
|
||||
// println d
|
||||
// }
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
|
||||
// if(d['I'] && d['K']){
|
||||
@@ -782,6 +782,23 @@ def getStreetInfo(def city, def glocation){
|
||||
//}
|
||||
//ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据")
|
||||
|
||||
def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1)
|
||||
def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1)
|
||||
|
||||
def keys = []
|
||||
for(def d : datas2){
|
||||
keys << d['A']+"##"+d['C']+"##"+d['B']
|
||||
}
|
||||
|
||||
def index = 0
|
||||
for(def d : datas1){
|
||||
def k = d['A']+"##"+d['C']+"##"+d['B']
|
||||
if(!keys.contains(k)){
|
||||
index ++
|
||||
println index +" "+k
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -7,11 +7,11 @@ email.setSSL(true);
|
||||
email.setHostName("smtp.exmail.qq.com");
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码
|
||||
email.addTo("gaojuan@fangjia.com"); //接收方
|
||||
email.setFrom("service@fangjia.com"); //发送方
|
||||
email.setSubject("test"); //标题
|
||||
email.setContent("你?", "text/html");
|
||||
email.setContent("测试", "text/html");
|
||||
|
||||
//email.setHostName("smtp.exmail.qq.com");
|
||||
//email.setSSL(true);
|
||||
|
||||
+62
-27
@@ -37,7 +37,10 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findCrawlData()
|
||||
def source = "58同城"
|
||||
def root = "58.com"
|
||||
def shortCityMap = findCitySeeds(root)
|
||||
def datas = findCrawlData(source, shortCityMap, root)
|
||||
def list = []
|
||||
datas.each {k,v->
|
||||
if(v){
|
||||
@@ -58,15 +61,32 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
}
|
||||
|
||||
def findCrawlData(){
|
||||
def findCitySeeds(root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","北京")
|
||||
q.put("source", "搜房")
|
||||
q.put("root", root)
|
||||
q.put("city", "福州")
|
||||
|
||||
def dbcursor = districtseedsColl.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(!cityMap.keySet().contains(d.city)){
|
||||
cityMap.put(d.city, d.city_short)
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
}
|
||||
|
||||
def findCrawlData(def source, shortCityMap, def root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
q.put("source", source)
|
||||
q.put("s_date",202112)
|
||||
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def retMap = [:]
|
||||
def prefix = "esf.fang.com/loupan/"
|
||||
def prefix = "xiaoqu/"
|
||||
def keyList =new HashSet()
|
||||
def index = 0
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
@@ -76,36 +96,42 @@ class CrawlDistrictIdSelect {
|
||||
}else{
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".htm"))
|
||||
|
||||
if(url){
|
||||
def district_id = ""
|
||||
if(url.contains(prefix)){
|
||||
district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1)
|
||||
}else{
|
||||
district_id = url;
|
||||
}
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
|
||||
if(!shortCity){
|
||||
shortCity = "bj"
|
||||
}
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
def ids = retMap.get(db.city)
|
||||
if(!ids){
|
||||
ids = []
|
||||
retMap.put(key, ids)
|
||||
retMap.put(db.city, ids)
|
||||
}
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "fang.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
if(!keyList.contains(db.city+"#"+district_id)){
|
||||
def count = findIsExists(db.city, district_id, root)
|
||||
if(!count){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "58.com"
|
||||
map.city_short= shortCityMap.get(db.city)
|
||||
map.city = db.city
|
||||
map.region = db.region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
map.name = db.name
|
||||
ids << map
|
||||
}else{
|
||||
index++
|
||||
println "[${index}]已存在:"+db.city +" "+db.name
|
||||
}
|
||||
keyList << db.city+"#"+district_id
|
||||
}else{
|
||||
println "重复:"+key
|
||||
println "重复:"+db.city+"#"+db.name
|
||||
}
|
||||
}else{
|
||||
println url +" 链接不合要求"
|
||||
@@ -113,4 +139,13 @@ class CrawlDistrictIdSelect {
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findIsExists(city, district_id, root){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city)
|
||||
q.put("district_id", district_id)
|
||||
q.put("root",root)
|
||||
def count = districtseedsColl.count(q)
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch {
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "G00jia123456");
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
|
||||
email.addTo(mail.receiverEmail);
|
||||
email.setFrom("service@fangjia.com");//发送方
|
||||
email.setSubject(mail.subject); //标题
|
||||
@@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch {
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "G00jia123456");
|
||||
email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi");
|
||||
email.addTo(mail.receiverEmail);
|
||||
email.setFrom("service@fangjia.com");
|
||||
email.setSubject(mail.subject);
|
||||
|
||||
+1
-1
@@ -181,7 +181,7 @@ class Esf58Crawl extends BaseBatch{
|
||||
// "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361",
|
||||
// "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575"
|
||||
def cityList = [
|
||||
"重庆":"cq|37","保定":"bd|424","信阳":"xy|8694","南京":"nj|172","南宁":"nn|845","厦门":"xm|606","青岛":"qd|122","哈尔滨":"hrb|202","天津":"tj|18","太原":"ty|740","宁德":"nd|7951","宜昌":"yc|858","成都":"cd|102","惠州":"huizhou|722","扬州":"yz|637","昆明":"km|541","杭州":"hz|79","武汉":"wh|158","泰州":"taizhou|693","洛阳":"luoyang|556","深圳":"sz|4","石家庄":"sjz|241","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572",
|
||||
"福州":"fz|304"
|
||||
]
|
||||
|
||||
static main(args) {
|
||||
|
||||
Reference in New Issue
Block a user