Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
@@ -96,7 +96,7 @@ abstract class BaseBatch {
|
||||
exception = true
|
||||
e.printStackTrace()
|
||||
__jms.monitor(batchName, '', extraData, e)
|
||||
def msg = "脚本发生异常 BatchName:${batchName}\t"
|
||||
def msg = "脚本发生异常错误 BatchName:${batchName}\t"
|
||||
def batchOwner = getBatchOwner(batch)
|
||||
if(batchOwner && batchOwner.size() > 0){
|
||||
def owner = null
|
||||
@@ -110,19 +110,20 @@ abstract class BaseBatch {
|
||||
if(e){
|
||||
msg += "\tException:" + e.getClass().name
|
||||
}
|
||||
def phoneNum = getBatchPhones(batch)
|
||||
if(SystemUtils.IS_OS_LINUX){
|
||||
sendDingDingMsg("2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c",msg)
|
||||
sendDingDingMsg("ae14f50045e9375c354940a2afb86f14e453bd1f08f78da0ba6e788d35bccec4",msg, phoneNum)
|
||||
|
||||
try {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("phone", "13764369683");
|
||||
params.put("type", "taskFail");
|
||||
params.put("msg", batchName.substring(batchName.length() - 20, batchName.length()));
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
httpRequest.get("http://didata-cloud-server.fangjia.com/sms/send", params, "utf-8");
|
||||
} catch (Exception ex) {
|
||||
|
||||
}
|
||||
// try {
|
||||
// Map<String, String> params = new HashMap<String, String>();
|
||||
// params.put("phone", "13764369683");
|
||||
// params.put("type", "taskFail");
|
||||
// params.put("msg", batchName.substring(batchName.length() - 20, batchName.length()));
|
||||
// HttpRequest httpRequest = new HttpRequest();
|
||||
// httpRequest.get("http://didata-cloud-server.fangjia.com/sms/send", params, "utf-8");
|
||||
// } catch (Exception ex) {
|
||||
//
|
||||
// }
|
||||
|
||||
task.exception = msg.toString()
|
||||
}
|
||||
@@ -174,6 +175,27 @@ abstract class BaseBatch {
|
||||
return ret
|
||||
}
|
||||
|
||||
static def getBatchPhones(BaseBatch bb) {
|
||||
def phones = []
|
||||
try {
|
||||
BatchOwner annotation = AnnotationUtils.findAnnotation(bb.getClass(), BatchOwner.class)
|
||||
if (annotation) {
|
||||
annotation.value().each {
|
||||
def map = it.toMap()
|
||||
if (map?.mobile) {
|
||||
phones << map.mobile
|
||||
}
|
||||
}
|
||||
} else {
|
||||
bb.logWarn('!!!请使用' + BatchOwner.class.getName() + '注解标明本Batch的拥有者!!!')
|
||||
}
|
||||
} catch (Exception e) {
|
||||
bb.logWarn('!!!解析BatchOwner失败!!!', e)
|
||||
}
|
||||
return phones[0] ? phones[0] : ""
|
||||
}
|
||||
|
||||
|
||||
static def getBatchDescribe(BaseBatch bb){
|
||||
try {
|
||||
BatchOwner annotation = AnnotationUtils.findAnnotation(bb.getClass(), BatchOwner.class)
|
||||
@@ -217,8 +239,8 @@ abstract class BaseBatch {
|
||||
__jms.sendDingMsg(mobile, msg)
|
||||
}
|
||||
|
||||
def static sendDingDingMsg(String token, String msg){
|
||||
DingDingMessageUtil.sendTextMessage(token, msg);
|
||||
def static sendDingDingMsg(String token, String msg, String phoneNum){
|
||||
DingDingMessageUtil.sendTextMessageAndAt(token, msg, phoneNum);
|
||||
}
|
||||
|
||||
// def static getProp(String key){
|
||||
|
||||
+28
-85
@@ -1,24 +1,20 @@
|
||||
package com.fangjia.script.averager
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.core.util.DingDingMessageUtil
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.cover.EmailTemplate
|
||||
import com.fangjia.script.jms.QueueRedirectConsumer
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ServerAreaMap
|
||||
import com.google.common.collect.MapDifference
|
||||
import com.google.common.collect.Maps
|
||||
import com.mongodb.*
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 通知计算二手房均价
|
||||
* @author duxiaoyu
|
||||
@@ -28,13 +24,11 @@ import org.bson.types.ObjectId
|
||||
class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
|
||||
|
||||
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public final static DBCollection REGION_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia').getCollection('region_block')
|
||||
public final static DBCollection COMMAND_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
|
||||
public final static DBCollection REGION_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia').getCollection('region_stat')
|
||||
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
|
||||
public final static DBCollection DISTRICT_STAT_206_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.206",27020)
|
||||
|
||||
public static final DBCollection SEAWEED_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'seaweed')
|
||||
public final static DBCollection COMMAND_COLL = MongoUtils.getDbCollection(Constants.LOG_MONGO_URI, 'fangjia_log', 'calc_command')
|
||||
public final static DBCollection REGION_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_stat')
|
||||
public final static DBCollection DISTRICT_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia_stat', 'district_stat')
|
||||
public final static DBCollection REGION_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_block')
|
||||
public final static def WORKER_MACHINE_IPS = ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1","fangjia-priceanalysis-2"]
|
||||
public final static def TOKEN="35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"
|
||||
|
||||
@@ -63,22 +57,14 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
final Date startDate = new Date();
|
||||
|
||||
for (String city : cities) {
|
||||
List<String> idList = getAllDistrictIds(city)
|
||||
logInfo(city + ":" + idList.size())
|
||||
List<DBObject> allDistrict = getAllDistrict(city)
|
||||
logInfo(city + ":" + allDistrict.size())
|
||||
//生成计算小区队列
|
||||
|
||||
for (String id : idList) {
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("name", 1)
|
||||
fields.put("estate_type2", 1)
|
||||
DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields)
|
||||
|
||||
if (district != null && !districtHasPrice(district.get('city'), district.get('region'), weekend, district.get('name'))) {
|
||||
String region = district.get('region')
|
||||
String name = district.get('name')
|
||||
String estateType = district.get('estate_type2')
|
||||
for (DBObject district : allDistrict) {
|
||||
if (!districtHasPrice((String)district.get('city'), (String)district.get('region'), weekend, (String)district.get('name'))) {
|
||||
String region = (String)district.get('region')
|
||||
String name = (String)district.get('name')
|
||||
String estateType = (String)district.get('estate_type2')
|
||||
districtCount++
|
||||
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
|
||||
saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
|
||||
@@ -92,8 +78,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
Thread.sleep(1000 * 5)
|
||||
println("${new Date().format("yyyyMMdd HH:mm:ss")} 等待均价计算完成")
|
||||
}
|
||||
|
||||
|
||||
for (String city : cities) {
|
||||
//计算该城市的城市、区域、板块均价
|
||||
List<DBObject> regions = getRegions(city)
|
||||
@@ -176,7 +160,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
appliesCount++;
|
||||
}
|
||||
}
|
||||
DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
|
||||
def statMap = [:]
|
||||
def preStatMap = [:]
|
||||
DISTRICT_STAT_COLL.find(new BasicDBObject("weekend", weekend)).batchSize(1000).each { BasicDBObject doc->
|
||||
@@ -222,7 +205,7 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
blockCount = REGION_STAT_COLL.count(query);
|
||||
def msg = "本周小区均价计算量:" + districtCount + ";本周城市均价计算量:" + cityCount + ";本周区域均价计算量:" + regionCount + ";本周板块均价计算量:" + blockCount + ";本周小区均价涨跌幅超过4%小区数量:" + appliesCount + ";计算均价超过10%小区量:" + bigMarkupCount+ ";运行时间:" + secondToTime(time) + ".";
|
||||
//发送钉钉提示
|
||||
sendDingDingMsg(TOKEN, msg);
|
||||
DingDingMessageUtil.sendTextMessage(TOKEN, msg)
|
||||
EmailTemplate sd = new EmailTemplate("mixedZone.ftl")
|
||||
Map map = new HashMap()
|
||||
map.put('applies', applies)
|
||||
@@ -286,35 +269,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
boolean isCalcFinish(){
|
||||
DBObject q = new BasicDBObject("status",0)
|
||||
return COMMAND_COLL.count(q) == 0
|
||||
}
|
||||
void DisSync206To43(Date startDate){
|
||||
//数据迁移
|
||||
def query = new BasicDBObject()
|
||||
query.put("m_date", new BasicDBObject('$gte', startDate))
|
||||
DBCursor cursor = DISTRICT_STAT_206_COLL.find(query).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(10000)
|
||||
long count = 0
|
||||
long total = cursor.count()
|
||||
List<BasicDBObject> bs = new ArrayList<BasicDBObject>()
|
||||
for (BasicDBObject log : cursor) {
|
||||
bs.add(log)
|
||||
count++
|
||||
if (bs.size() > 1000) {
|
||||
try {
|
||||
DISTRICT_STAT_COLL.insert(bs)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
bs.clear();
|
||||
println("数据迁移:${DateUtil.formatByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")} ${total}/${count} 进度:${DataUtil.roundDouble((count * 100l) / total, 2)}")
|
||||
}
|
||||
}
|
||||
try {
|
||||
DISTRICT_STAT_COLL.insert(bs)
|
||||
} catch (Exception ex) {
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -339,9 +293,9 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
* @param weekend
|
||||
* @return
|
||||
*/
|
||||
boolean districtHasPrice(String city, String region, Date weekend,String district_name){
|
||||
String ischeckDistrictHasPrice = System.getProperty("ischeckDistrictHasPrice")
|
||||
if(ischeckDistrictHasPrice!=null && ischeckDistrictHasPrice.trim().equals("true")){
|
||||
boolean districtHasPrice(String city, String region, Date weekend, String district_name){
|
||||
String isCheckDistrictHasPrice = System.getProperty("isCheckDistrictHasPrice")
|
||||
if(isCheckDistrictHasPrice!=null && isCheckDistrictHasPrice.trim().equalsIgnoreCase("true")){
|
||||
def query = new BasicDBObject()
|
||||
query.append("city", city)
|
||||
query.append("region", region)
|
||||
@@ -398,31 +352,20 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
|
||||
private List<String> getAllDistrictIds(String city) {
|
||||
private List<DBObject> getAllDistrict(String city) {
|
||||
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("cat", "district")
|
||||
q.put("visible", 0)
|
||||
q.put("status", 0)
|
||||
|
||||
int trytimes = 5
|
||||
Set<String> uniqueIdList = new LinkedHashSet<String>()
|
||||
DBObject fields = new BasicDBObject()
|
||||
fields.put("city", 1)
|
||||
fields.put("region", 1)
|
||||
fields.put("name", 1)
|
||||
fields.put("estate_type2", 1)
|
||||
|
||||
for(int i=0; i<trytimes; i++) {
|
||||
DSCursor cur = SEAWEED_COLL.find(q, new BasicDBObject("_id", 1)).batchSize(2000)
|
||||
int totalCount = cur.count()
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
while(iter.hasNext()) {
|
||||
DBObject dbo = iter.next()
|
||||
String id = dbo.get('_id')
|
||||
uniqueIdList.add(id)
|
||||
}
|
||||
|
||||
if(uniqueIdList.size() == totalCount) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return new ArrayList<String>(uniqueIdList)
|
||||
return SEAWEED_COLL.find(q, fields).batchSize(2000).toArray()
|
||||
}
|
||||
|
||||
private List<DBObject> getRegions(String city) {
|
||||
|
||||
+3
-3
@@ -32,8 +32,8 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
|
||||
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
|
||||
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
|
||||
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017)
|
||||
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
|
||||
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
|
||||
|
||||
public final static def WORKER_MACHINE_IPS = ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1","fangjia-priceanalysis-2"]
|
||||
public final static def TOKEN="35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"
|
||||
@@ -185,7 +185,6 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
q.put("category", CALC_CATEGORY)
|
||||
q.put('status', new BasicDBObject('$in', [0,1,102])) // 0-初始状态 1-处理中 102-计算完成 2-完成 3-处理错误 4-处理异常
|
||||
while(true) {
|
||||
sleep(5 * 60 * 1000)
|
||||
def c = COMMAND_COLL.count(q)
|
||||
logInfo("calc command: " + c)
|
||||
if(c <= 0) {
|
||||
@@ -208,6 +207,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
sleep(5 * 60 * 1000)
|
||||
}
|
||||
|
||||
|
||||
|
||||
+14
-8
@@ -25,7 +25,7 @@ import com.mongodb.DBObject
|
||||
class ApiInvokeStatistics extends BaseBatch{
|
||||
final static def mongoDomain = "mongo.db.fangjia.com"
|
||||
public static final DBCollection OPEN_URI_LOG = com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI,"business", "open_uri_log")
|
||||
public static final DBCollection OPEN_URI_LOG_COUNT = MongoUtils.getCollectionByName("business", "api_invoke_statistical", "mongo.db.fangjia.com", 27017)
|
||||
public static final DBCollection OPEN_URI_LOG_COUNT = MongoUtils.getCollectionByName("business", "api_invoke_statistical", "mongo.db.fangjia.com", 28017)
|
||||
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
static main(args) {
|
||||
def s = System.currentTimeMillis()
|
||||
@@ -43,8 +43,13 @@ class ApiInvokeStatistics extends BaseBatch{
|
||||
def getYestodayLog(Date today,Date yestoday){
|
||||
//Date today = getToday();
|
||||
//Date yestoday = getNextDay(today)
|
||||
BasicDBObject query= new BasicDBObject("c_date", new BasicDBObject('$gt',yestoday).append('$lt', today))
|
||||
DBCursor dbc=OPEN_URI_LOG.find(query).batchSize(10000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def deleteQuery = new BasicDBObject("statistical_date", yestoday)
|
||||
OPEN_URI_LOG_COUNT.remove(deleteQuery)
|
||||
println "已删除 ${yestoday.format('yyyy-MM-dd')} 的旧统计数据"
|
||||
BasicDBObject query= new BasicDBObject("c_date", new BasicDBObject('$gte',yestoday).append('$lt', today))
|
||||
// DBCursor dbc=OPEN_URI_LOG.find(query).batchSize(10000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
DBCursor dbc = OPEN_URI_LOG.find(query, new BasicDBObject("res_map", 0)).batchSize(10000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
|
||||
def results = [:]
|
||||
int c = 0
|
||||
dbc.each {BasicDBObject log->
|
||||
@@ -103,11 +108,12 @@ class ApiInvokeStatistics extends BaseBatch{
|
||||
obj.append("total_time",total_time)
|
||||
obj.append("statistical_date",yestoday)
|
||||
obj.append("user_name",companyName)
|
||||
DBObject qu = new BasicDBObject();
|
||||
qu.append("user_id", userIDStr);
|
||||
qu.append("request_uri", uri);
|
||||
qu.append("statistical_date", yestoday)
|
||||
OPEN_URI_LOG_COUNT.update(qu, new BasicDBObject('$set', obj), true, true);
|
||||
// DBObject qu = new BasicDBObject();
|
||||
// qu.append("user_id", userIDStr);
|
||||
// qu.append("request_uri", uri);
|
||||
// qu.append("statistical_date", yestoday)
|
||||
// OPEN_URI_LOG_COUNT.update(qu, new BasicDBObject('$set', obj), true, true);
|
||||
OPEN_URI_LOG_COUNT.insert(obj)
|
||||
}
|
||||
|
||||
|
||||
|
||||
-2
@@ -64,8 +64,6 @@ class AutocompleteDataUpdaterBatch extends BaseBatch {
|
||||
DSCursor cur = SEAWEED_COLL.find(q, fields).batchSize(2000)
|
||||
Iterator<DBObject> iter = cur.iterator()
|
||||
int i = 0
|
||||
|
||||
Map<String, String> keywordMap = new HashMap<String, String>()
|
||||
while(iter.hasNext()) {
|
||||
i++
|
||||
if(i % 100 == 0) {
|
||||
|
||||
@@ -245,7 +245,7 @@ public class CommonUtils {
|
||||
data.put("remark", 'week')
|
||||
data.put("c_date", new Date())
|
||||
if(isTest){
|
||||
data.put("ip", "192.168.0.231")
|
||||
data.put("ip", "192.168.10.29")
|
||||
COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false)
|
||||
}else{
|
||||
data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)])
|
||||
|
||||
+2
-1
@@ -4,6 +4,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
|
||||
/**
|
||||
* Created by dxy on 2015/9/14.
|
||||
@@ -43,7 +44,7 @@ public class ServerAreaMap {
|
||||
}
|
||||
|
||||
private static def findServerAreas(host, port, q, sq) {
|
||||
DBCollection collection = MongoUtils.getCollectionByName("fangjia", "server_area", host, port);
|
||||
DBCollection collection = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, "fangjia", "server_area");
|
||||
DBCursor cur = null;
|
||||
if(q != null) {
|
||||
cur = collection.find(q)
|
||||
|
||||
+16
-11
@@ -1,14 +1,19 @@
|
||||
package com.fangjia.script.wenyan
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
|
||||
def citys = ["北京", "唐山", "天津", "秦皇岛", "石家庄", "包头", "太原", "丹东", "呼和浩特", "锦州", "沈阳", "吉林", "大连", "牡丹江", "长春", "无锡", "哈尔滨", "扬州", "上海", "徐州", "南京", "温州", "杭州", "金华", "宁波", "蚌埠", "合肥", "安庆", "福州", "泉州", "厦门", "九江", "南昌", "赣州", "济南", "烟台", "青岛", "济宁", "郑州", "洛阳", "武汉", "平顶山", "长沙", "宜昌", "广州", "襄阳", "深圳", "岳阳", "南宁", "常德", "海口", "惠州", "重庆", "湛江", "成都", "韶关", "贵阳", "桂林", "昆明", "北海", "西安", "三亚", "兰州", "泸州", "西宁", "南充", "银川", "遵义", "乌鲁木齐", "大理", "北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州", "合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台", "徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁", "南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华", "昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城", "芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州", "连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口", "西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"]
|
||||
citys.unique()
|
||||
citys.each { def city ->
|
||||
if (RegionData.isCity(city)) {
|
||||
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date > 20210407 and city='${city}'", {
|
||||
println("$city\t${it.count}")
|
||||
})
|
||||
}
|
||||
}
|
||||
import org.apache.commons.mail.SimpleEmail
|
||||
|
||||
import javax.mail.internet.MimeUtility
|
||||
|
||||
EMAIL_SERVER = "smtp.exmail.qq.com"
|
||||
SimpleEmail email = new SimpleEmail();
|
||||
email.setSSL(true);
|
||||
email.setHostName(EMAIL_SERVER);
|
||||
email.setSslSmtpPort("465");
|
||||
email.setCharset("GBK")
|
||||
email.setAuthentication("service@fangjia.com", "iXWzdL69CnzxksaE");
|
||||
email.addTo("wangwenyan@fangjia.com");
|
||||
email.setFrom("service@fangjia.com", MimeUtility.encodeText("房价网大数据中心"), "utf-8");//发送方
|
||||
email.setSubject("标题");
|
||||
email.setContent("你好", "text/html");
|
||||
email.send();
|
||||
@@ -1,62 +1,11 @@
|
||||
package com.fangjia.script.wenyan
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.http.NameValuePair
|
||||
import org.apache.http.client.config.CookieSpecs
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.apache.http.client.utils.URLEncodedUtils
|
||||
import org.apache.http.impl.client.CloseableHttpClient
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
import org.bson.Document
|
||||
|
||||
//com.mongodb.client.MongoCollection<Document> GUIDING_PRICE = Mongo3Utils.getDbCollection("fangjia", "guiding_price")
|
||||
//
|
||||
//Mongo3Utils.getDbCollection("fangjia", "confirm_price_data").find(new BasicDBObject("isGuidancePrice", 0)).each { Document doc->
|
||||
// Document item = new Document()
|
||||
// item.put("guiding_type", 0)
|
||||
// item.put("city", doc.city)
|
||||
// item.put("region", doc.region)
|
||||
// item.put("name", doc.district_name)
|
||||
// item.put("guiding_person", doc.user_name)
|
||||
// item.put("category", "district")
|
||||
// item.put("guiding_value", doc.price)
|
||||
// item.put("guiding_date", doc.c_date)
|
||||
// item.put("house_type", 0)
|
||||
// item.put("estate_type", DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(doc.city, doc.region, doc.district_name))?.estate_type2)
|
||||
// item.put("c_date", new Date())
|
||||
// GUIDING_PRICE.insertOne(item)
|
||||
// println(item)
|
||||
//}
|
||||
def keys = []
|
||||
ExcelUtils.read("D:\\Users\\wenyan\\Desktop\\ͶƱ½á¹ûÃ÷ϸ(1).xlsx").each {List item->
|
||||
keys << "${item['A']}_${item['B']}_${item["C"]}".toString()
|
||||
println(item)
|
||||
}
|
||||
MongoCollection<Document> FANGJIA_STAT = Mongo3Utils.getDbCollection("fangjia_stat", "district_stat")
|
||||
def creditMap = [:]
|
||||
FANGJIA_STAT.find(new Document("weekend", DateUtil.parseDate("20200322", "yyyyMMdd"))).each {Document d->
|
||||
def key ="${d.city}_${d.region}_${d.district_name}".toString()
|
||||
if(!keys.contains(key)){
|
||||
Double credit = (int)(d.credit * 10)
|
||||
creditMap[credit] = creditMap.get(credit, 0) + 1
|
||||
}
|
||||
}
|
||||
creditMap.each {def key, value->
|
||||
println("${key}\t${value}")
|
||||
ExcelUtils.read("C:\\Users\\goojia\\Downloads\\꼿제팀섬꿎桿친겼.xlsx",1).each {
|
||||
def one = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(it['B'], it['C'], it['D']))
|
||||
println("${one.lng/10000000000},${one.lat/10000000000}")
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -2,49 +2,10 @@ package com.fangjia.script.wenyan
|
||||
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.action.get.GetRequest
|
||||
import org.elasticsearch.action.get.GetResponse
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
////def data = []
|
||||
////data << ["ID","城市","区域","名称","地址", "财力评级"]
|
||||
//RestHighLevelClient client = ElasticUtils.getClient89("192.168.0.89")
|
||||
//district_analy_data_client = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data")
|
||||
//level_map = ["1": "S", "2": "A", "3": "B", "4": "C", "5": "D"]
|
||||
//level_maps = new HashMap<>()
|
||||
//for(def key in level_map.keySet()){
|
||||
// level_maps.put(level_map[key], key)
|
||||
//}
|
||||
//ExcelUtils.modify("C:\\Users\\goojia\\Desktop\\财力评级-厦门.xlsx",{List item->
|
||||
//// GetResponse response = client.get(new GetRequest("community_summary", item["A"] as String), RequestOptions.DEFAULT)
|
||||
//// def query = new Document("city", item['B'].toString().replace("市", ""))
|
||||
//// query.append("region", item['C'].replace("区", ""))
|
||||
//// query.append("district_name", item["D"])
|
||||
//// def stat = district_analy_data_client.find(query).first()
|
||||
//// println(query)
|
||||
//// item["F"] = response.getSourceAsMap()?.consumption_rating
|
||||
//// item["G"] = stat?.community_rating
|
||||
// item["G"] = level_maps.get(item["G"], null)
|
||||
//// println(item)
|
||||
//})
|
||||
//System.exit(0)
|
||||
////for (def region in ["思明", "翔安", "湖里"]) {
|
||||
//// DbUtils.seaweed.find(new BasicDBObject("city", "厦门").append("region", region).append("cat", "district").append("status", 0)).each { BasicDBObject dbo ->
|
||||
//// println("${dbo._id.toString()}\t${dbo.city}市\t${dbo.region}区\t${dbo.name}\t${dbo.address}")
|
||||
//// }
|
||||
////}
|
||||
|
||||
ExcelUtils.read("C:\\Users\\goojia\\Desktop\\海南省城市均价.xlsx",1).each {List item->
|
||||
def yearMonth = DateUtil.formatByPattern(item[0], "yyyyMM").toInteger()
|
||||
|
||||
println(item)
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\goojia\\Desktop\\区域楼盘评级.xlsx",{List item->
|
||||
item["E"] = ProvinceData.getDisplay(item[0])
|
||||
print(item)
|
||||
})
|
||||
+22
@@ -30,6 +30,28 @@ class DingDingMessageUtil {
|
||||
}
|
||||
}
|
||||
|
||||
static void sendTextMessageAndAt(String access_token, String msg, String atMobile) throws Exception {
|
||||
HttpRequest request = new HttpRequest();
|
||||
HttpPost httppost = new HttpPost("https://oapi.dingtalk.com/robot/send?access_token=" + access_token);
|
||||
httppost.addHeader("Content-Type", "application/json; charset=utf-8");
|
||||
|
||||
// 创建消息体,只包含一个手机号
|
||||
String textMsg = "{ " +
|
||||
"\"msgtype\": \"text\", " +
|
||||
"\"text\": {\"content\": \"" + msg + "\"}, " +
|
||||
"\"at\": {\"atMobiles\": [\"" + atMobile + "\"]}" + // 只 @ 一个手机号
|
||||
"}";
|
||||
|
||||
StringEntity se = new StringEntity(textMsg, "utf-8");
|
||||
httppost.setEntity(se);
|
||||
HttpResponse response = request.getHttpClient().execute(httppost);
|
||||
|
||||
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
|
||||
String result = EntityUtils.toString(response.getEntity(), "utf-8");
|
||||
System.out.println("消息发送结果:" + result);
|
||||
}
|
||||
}
|
||||
|
||||
static void sendMarkdownMessage(String access_token, String title, String msg){
|
||||
HttpRequest request = new HttpRequest()
|
||||
JSONObject requestBody = new JSONObject()
|
||||
|
||||
Reference in New Issue
Block a user