Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2025-10-20 09:56:44 +08:00
121 changed files with 6654 additions and 1422 deletions
@@ -96,7 +96,7 @@ abstract class BaseBatch {
exception = true
e.printStackTrace()
__jms.monitor(batchName, '', extraData, e)
def msg = "脚本发生异常 BatchName:${batchName}\t"
def msg = "脚本发生异常错误 BatchName:${batchName}\t"
def batchOwner = getBatchOwner(batch)
if(batchOwner && batchOwner.size() > 0){
def owner = null
@@ -110,19 +110,20 @@ abstract class BaseBatch {
if(e){
msg += "\tException:" + e.getClass().name
}
def phoneNum = getBatchPhones(batch)
if(SystemUtils.IS_OS_LINUX){
sendDingDingMsg("2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c",msg)
sendDingDingMsg("ae14f50045e9375c354940a2afb86f14e453bd1f08f78da0ba6e788d35bccec4",msg, phoneNum)
try {
Map<String, String> params = new HashMap<String, String>();
params.put("phone", "13764369683");
params.put("type", "taskFail");
params.put("msg", batchName.substring(batchName.length() - 20, batchName.length()));
HttpRequest httpRequest = new HttpRequest();
httpRequest.get("http://didata-cloud-server.fangjia.com/sms/send", params, "utf-8");
} catch (Exception ex) {
}
// try {
// Map<String, String> params = new HashMap<String, String>();
// params.put("phone", "13764369683");
// params.put("type", "taskFail");
// params.put("msg", batchName.substring(batchName.length() - 20, batchName.length()));
// HttpRequest httpRequest = new HttpRequest();
// httpRequest.get("http://didata-cloud-server.fangjia.com/sms/send", params, "utf-8");
// } catch (Exception ex) {
//
// }
task.exception = msg.toString()
}
@@ -174,6 +175,27 @@ abstract class BaseBatch {
return ret
}
static def getBatchPhones(BaseBatch bb) {
def phones = []
try {
BatchOwner annotation = AnnotationUtils.findAnnotation(bb.getClass(), BatchOwner.class)
if (annotation) {
annotation.value().each {
def map = it.toMap()
if (map?.mobile) {
phones << map.mobile
}
}
} else {
bb.logWarn('!!!请使用' + BatchOwner.class.getName() + '注解标明本Batch的拥有者!!!')
}
} catch (Exception e) {
bb.logWarn('!!!解析BatchOwner失败!!!', e)
}
return phones[0] ? phones[0] : ""
}
static def getBatchDescribe(BaseBatch bb){
try {
BatchOwner annotation = AnnotationUtils.findAnnotation(bb.getClass(), BatchOwner.class)
@@ -217,8 +239,8 @@ abstract class BaseBatch {
__jms.sendDingMsg(mobile, msg)
}
def static sendDingDingMsg(String token, String msg){
DingDingMessageUtil.sendTextMessage(token, msg);
def static sendDingDingMsg(String token, String msg, String phoneNum){
DingDingMessageUtil.sendTextMessageAndAt(token, msg, phoneNum);
}
// def static getProp(String key){
@@ -1,24 +1,20 @@
package com.fangjia.script.averager
import com.fangjia.core.util.DataUtil
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.core.util.DingDingMessageUtil
import com.fangjia.core.util.MongoUtils
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.cover.EmailTemplate
import com.fangjia.script.jms.QueueRedirectConsumer
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ServerAreaMap
import com.google.common.collect.MapDifference
import com.google.common.collect.Maps
import com.mongodb.*
import org.bson.Document
import org.bson.types.ObjectId
/**
* 通知计算二手房均价
* @author duxiaoyu
@@ -28,13 +24,11 @@ import org.bson.types.ObjectId
class InsertMonitorCalcCommandBatch extends BaseBatch {
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public final static DBCollection REGION_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia').getCollection('region_block')
public final static DBCollection COMMAND_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public final static DBCollection REGION_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia').getCollection('region_stat')
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
public final static DBCollection DISTRICT_STAT_206_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.206",27020)
public static final DBCollection SEAWEED_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'seaweed')
public final static DBCollection COMMAND_COLL = MongoUtils.getDbCollection(Constants.LOG_MONGO_URI, 'fangjia_log', 'calc_command')
public final static DBCollection REGION_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_stat')
public final static DBCollection DISTRICT_STAT_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia_stat', 'district_stat')
public final static DBCollection REGION_COLL = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, 'fangjia', 'region_block')
public final static def WORKER_MACHINE_IPS = ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1","fangjia-priceanalysis-2"]
public final static def TOKEN="35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"
@@ -63,22 +57,14 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
final Date startDate = new Date();
for (String city : cities) {
List<String> idList = getAllDistrictIds(city)
logInfo(city + ":" + idList.size())
List<DBObject> allDistrict = getAllDistrict(city)
logInfo(city + ":" + allDistrict.size())
//生成计算小区队列
for (String id : idList) {
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
fields.put("estate_type2", 1)
DBObject district = SEAWEED_COLL.findOne(new BasicDBObject("_id", new ObjectId(id)), fields)
if (district != null && !districtHasPrice(district.get('city'), district.get('region'), weekend, district.get('name'))) {
String region = district.get('region')
String name = district.get('name')
String estateType = district.get('estate_type2')
for (DBObject district : allDistrict) {
if (!districtHasPrice((String)district.get('city'), (String)district.get('region'), weekend, (String)district.get('name'))) {
String region = (String)district.get('region')
String name = (String)district.get('name')
String estateType = (String)district.get('estate_type2')
districtCount++
String ip = WORKER_MACHINE_IPS[districtCount % WORKER_MACHINE_IPS.size()]
saveDistrictCalcCommand(city, region, name, estateType, weekend, ip)
@@ -92,8 +78,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
Thread.sleep(1000 * 5)
println("${new Date().format("yyyyMMdd HH:mm:ss")} 等待均价计算完成")
}
for (String city : cities) {
//计算该城市的城市、区域、板块均价
List<DBObject> regions = getRegions(city)
@@ -176,7 +160,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
appliesCount++;
}
}
DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
def statMap = [:]
def preStatMap = [:]
DISTRICT_STAT_COLL.find(new BasicDBObject("weekend", weekend)).batchSize(1000).each { BasicDBObject doc->
@@ -222,7 +205,7 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
blockCount = REGION_STAT_COLL.count(query);
def msg = "本周小区均价计算量:" + districtCount + ";本周城市均价计算量:" + cityCount + ";本周区域均价计算量:" + regionCount + ";本周板块均价计算量:" + blockCount + ";本周小区均价涨跌幅超过4%小区数量:" + appliesCount + ";计算均价超过10%小区量:" + bigMarkupCount+ ";运行时间:" + secondToTime(time) + ".";
//发送钉钉提示
sendDingDingMsg(TOKEN, msg);
DingDingMessageUtil.sendTextMessage(TOKEN, msg)
EmailTemplate sd = new EmailTemplate("mixedZone.ftl")
Map map = new HashMap()
map.put('applies', applies)
@@ -286,35 +269,6 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
boolean isCalcFinish(){
DBObject q = new BasicDBObject("status",0)
return COMMAND_COLL.count(q) == 0
}
void DisSync206To43(Date startDate){
//数据迁移
def query = new BasicDBObject()
query.put("m_date", new BasicDBObject('$gte', startDate))
DBCursor cursor = DISTRICT_STAT_206_COLL.find(query).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(10000)
long count = 0
long total = cursor.count()
List<BasicDBObject> bs = new ArrayList<BasicDBObject>()
for (BasicDBObject log : cursor) {
bs.add(log)
count++
if (bs.size() > 1000) {
try {
DISTRICT_STAT_COLL.insert(bs)
} catch (Exception e) {
e.printStackTrace()
}
bs.clear();
println("数据迁移:${DateUtil.formatByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")} ${total}/${count} 进度:${DataUtil.roundDouble((count * 100l) / total, 2)}")
}
}
try {
DISTRICT_STAT_COLL.insert(bs)
} catch (Exception ex) {
}
}
/**
@@ -339,9 +293,9 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
* @param weekend
* @return
*/
boolean districtHasPrice(String city, String region, Date weekend,String district_name){
String ischeckDistrictHasPrice = System.getProperty("ischeckDistrictHasPrice")
if(ischeckDistrictHasPrice!=null && ischeckDistrictHasPrice.trim().equals("true")){
boolean districtHasPrice(String city, String region, Date weekend, String district_name){
String isCheckDistrictHasPrice = System.getProperty("isCheckDistrictHasPrice")
if(isCheckDistrictHasPrice!=null && isCheckDistrictHasPrice.trim().equalsIgnoreCase("true")){
def query = new BasicDBObject()
query.append("city", city)
query.append("region", region)
@@ -398,31 +352,20 @@ class InsertMonitorCalcCommandBatch extends BaseBatch {
}
private List<String> getAllDistrictIds(String city) {
private List<DBObject> getAllDistrict(String city) {
DBObject q = new BasicDBObject()
q.put("city", city)
q.put("cat", "district")
q.put("visible", 0)
q.put("status", 0)
int trytimes = 5
Set<String> uniqueIdList = new LinkedHashSet<String>()
DBObject fields = new BasicDBObject()
fields.put("city", 1)
fields.put("region", 1)
fields.put("name", 1)
fields.put("estate_type2", 1)
for(int i=0; i<trytimes; i++) {
DSCursor cur = SEAWEED_COLL.find(q, new BasicDBObject("_id", 1)).batchSize(2000)
int totalCount = cur.count()
Iterator<DBObject> iter = cur.iterator()
while(iter.hasNext()) {
DBObject dbo = iter.next()
String id = dbo.get('_id')
uniqueIdList.add(id)
}
if(uniqueIdList.size() == totalCount) {
break
}
}
return new ArrayList<String>(uniqueIdList)
return SEAWEED_COLL.find(q, fields).batchSize(2000).toArray()
}
private List<DBObject> getRegions(String city) {
@@ -32,8 +32,8 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017)
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
public final static def WORKER_MACHINE_IPS = ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1","fangjia-priceanalysis-2"]
public final static def TOKEN="35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"
@@ -185,7 +185,6 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
q.put("category", CALC_CATEGORY)
q.put('status', new BasicDBObject('$in', [0,1,102])) // 0-初始状态 1-处理中 102-计算完成 2-完成 3-处理错误 4-处理异常
while(true) {
sleep(5 * 60 * 1000)
def c = COMMAND_COLL.count(q)
logInfo("calc command: " + c)
if(c <= 0) {
@@ -208,6 +207,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
break;
}
}
sleep(5 * 60 * 1000)
}
@@ -25,7 +25,7 @@ import com.mongodb.DBObject
class ApiInvokeStatistics extends BaseBatch{
final static def mongoDomain = "mongo.db.fangjia.com"
public static final DBCollection OPEN_URI_LOG = com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI,"business", "open_uri_log")
public static final DBCollection OPEN_URI_LOG_COUNT = MongoUtils.getCollectionByName("business", "api_invoke_statistical", "mongo.db.fangjia.com", 27017)
public static final DBCollection OPEN_URI_LOG_COUNT = MongoUtils.getCollectionByName("business", "api_invoke_statistical", "mongo.db.fangjia.com", 28017)
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
static main(args) {
def s = System.currentTimeMillis()
@@ -43,8 +43,13 @@ class ApiInvokeStatistics extends BaseBatch{
def getYestodayLog(Date today,Date yestoday){
//Date today = getToday();
//Date yestoday = getNextDay(today)
BasicDBObject query= new BasicDBObject("c_date", new BasicDBObject('$gt',yestoday).append('$lt', today))
DBCursor dbc=OPEN_URI_LOG.find(query).batchSize(10000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def deleteQuery = new BasicDBObject("statistical_date", yestoday)
OPEN_URI_LOG_COUNT.remove(deleteQuery)
println "已删除 ${yestoday.format('yyyy-MM-dd')} 的旧统计数据"
BasicDBObject query= new BasicDBObject("c_date", new BasicDBObject('$gte',yestoday).append('$lt', today))
// DBCursor dbc=OPEN_URI_LOG.find(query).batchSize(10000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
DBCursor dbc = OPEN_URI_LOG.find(query, new BasicDBObject("res_map", 0)).batchSize(10000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def results = [:]
int c = 0
dbc.each {BasicDBObject log->
@@ -103,11 +108,12 @@ class ApiInvokeStatistics extends BaseBatch{
obj.append("total_time",total_time)
obj.append("statistical_date",yestoday)
obj.append("user_name",companyName)
DBObject qu = new BasicDBObject();
qu.append("user_id", userIDStr);
qu.append("request_uri", uri);
qu.append("statistical_date", yestoday)
OPEN_URI_LOG_COUNT.update(qu, new BasicDBObject('$set', obj), true, true);
// DBObject qu = new BasicDBObject();
// qu.append("user_id", userIDStr);
// qu.append("request_uri", uri);
// qu.append("statistical_date", yestoday)
// OPEN_URI_LOG_COUNT.update(qu, new BasicDBObject('$set', obj), true, true);
OPEN_URI_LOG_COUNT.insert(obj)
}
@@ -64,8 +64,6 @@ class AutocompleteDataUpdaterBatch extends BaseBatch {
DSCursor cur = SEAWEED_COLL.find(q, fields).batchSize(2000)
Iterator<DBObject> iter = cur.iterator()
int i = 0
Map<String, String> keywordMap = new HashMap<String, String>()
while(iter.hasNext()) {
i++
if(i % 100 == 0) {
@@ -245,7 +245,7 @@ public class CommonUtils {
data.put("remark", 'week')
data.put("c_date", new Date())
if(isTest){
data.put("ip", "192.168.0.231")
data.put("ip", "192.168.10.29")
COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false)
}else{
data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)])
@@ -4,6 +4,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.fangjia.core.util.MongoUtils
/**
* Created by dxy on 2015/9/14.
@@ -43,7 +44,7 @@ public class ServerAreaMap {
}
private static def findServerAreas(host, port, q, sq) {
DBCollection collection = MongoUtils.getCollectionByName("fangjia", "server_area", host, port);
DBCollection collection = MongoUtils.getDbCollection(com.fangjia.core.Constants.FANGJIA_MONGO_URI, "fangjia", "server_area");
DBCursor cur = null;
if(q != null) {
cur = collection.find(q)
@@ -1,14 +1,19 @@
package com.fangjia.script.wenyan
import com.fangjia.core.RegionData
import com.fangjia.script.util.ElasticUtils
def citys = ["北京", "唐山", "天津", "秦皇岛", "石家庄", "包头", "太原", "丹东", "呼和浩特", "锦州", "沈阳", "吉林", "大连", "牡丹江", "长春", "无锡", "哈尔滨", "扬州", "上海", "徐州", "南京", "温州", "杭州", "金华", "宁波", "蚌埠", "合肥", "安庆", "福州", "泉州", "厦门", "九江", "南昌", "赣州", "济南", "烟台", "青岛", "济宁", "郑州", "洛阳", "武汉", "平顶山", "长沙", "宜昌", "广州", "襄阳", "深圳", "岳阳", "南宁", "常德", "海口", "惠州", "重庆", "湛江", "成都", "韶关", "贵阳", "桂林", "昆明", "北海", "西安", "三亚", "兰州", "泸州", "西宁", "南充", "银川", "遵义", "乌鲁木齐", "大理", "北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州", "合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台", "徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁", "南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华", "昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城", "芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州", "连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口", "西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"]
citys.unique()
citys.each { def city ->
if (RegionData.isCity(city)) {
ElasticUtils.executeSqlQuery("select count(*) as count from seawater where p_date > 20210407 and city='${city}'", {
println("$city\t${it.count}")
})
}
}
import org.apache.commons.mail.SimpleEmail
import javax.mail.internet.MimeUtility
EMAIL_SERVER = "smtp.exmail.qq.com"
SimpleEmail email = new SimpleEmail();
email.setSSL(true);
email.setHostName(EMAIL_SERVER);
email.setSslSmtpPort("465");
email.setCharset("GBK")
email.setAuthentication("service@fangjia.com", "iXWzdL69CnzxksaE");
email.addTo("wangwenyan@fangjia.com");
email.setFrom("service@fangjia.com", MimeUtility.encodeText("房价网大数据中心"), "utf-8");//发送方
email.setSubject("标题");
email.setContent("你好", "text/html");
email.send();
@@ -1,62 +1,11 @@
package com.fangjia.script.wenyan
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.apache.http.NameValuePair
import org.apache.http.client.config.CookieSpecs
import org.apache.http.client.config.RequestConfig
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.utils.URLEncodedUtils
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import org.bson.Document
//com.mongodb.client.MongoCollection<Document> GUIDING_PRICE = Mongo3Utils.getDbCollection("fangjia", "guiding_price")
//
//Mongo3Utils.getDbCollection("fangjia", "confirm_price_data").find(new BasicDBObject("isGuidancePrice", 0)).each { Document doc->
// Document item = new Document()
// item.put("guiding_type", 0)
// item.put("city", doc.city)
// item.put("region", doc.region)
// item.put("name", doc.district_name)
// item.put("guiding_person", doc.user_name)
// item.put("category", "district")
// item.put("guiding_value", doc.price)
// item.put("guiding_date", doc.c_date)
// item.put("house_type", 0)
// item.put("estate_type", DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(doc.city, doc.region, doc.district_name))?.estate_type2)
// item.put("c_date", new Date())
// GUIDING_PRICE.insertOne(item)
// println(item)
//}
def keys = []
ExcelUtils.read("D:\\Users\\wenyan\\Desktop\\ͶƱ½á¹ûÃ÷ϸ(1).xlsx").each {List item->
keys << "${item['A']}_${item['B']}_${item["C"]}".toString()
println(item)
}
MongoCollection<Document> FANGJIA_STAT = Mongo3Utils.getDbCollection("fangjia_stat", "district_stat")
def creditMap = [:]
FANGJIA_STAT.find(new Document("weekend", DateUtil.parseDate("20200322", "yyyyMMdd"))).each {Document d->
def key ="${d.city}_${d.region}_${d.district_name}".toString()
if(!keys.contains(key)){
Double credit = (int)(d.credit * 10)
creditMap[credit] = creditMap.get(credit, 0) + 1
}
}
creditMap.each {def key, value->
println("${key}\t${value}")
ExcelUtils.read("C:\\Users\\goojia\\Downloads\\꼿제팀섬꿎桿친겼.xlsx",1).each {
def one = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(it['B'], it['C'], it['D']))
println("${one.lng/10000000000},${one.lat/10000000000}")
}
@@ -2,49 +2,10 @@ package com.fangjia.script.wenyan
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.Document
import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.get.GetResponse
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
fullCityMap = CommonDatautils.queryFullNameCity()
////def data = []
////data << ["ID","城市","区域","名称","地址", "财力评级"]
//RestHighLevelClient client = ElasticUtils.getClient89("192.168.0.89")
//district_analy_data_client = Mongo3Utils.getDbCollection("fangjia_stat", "district_analy_data")
//level_map = ["1": "S", "2": "A", "3": "B", "4": "C", "5": "D"]
//level_maps = new HashMap<>()
//for(def key in level_map.keySet()){
// level_maps.put(level_map[key], key)
//}
//ExcelUtils.modify("C:\\Users\\goojia\\Desktop\\财力评级-厦门.xlsx",{List item->
//// GetResponse response = client.get(new GetRequest("community_summary", item["A"] as String), RequestOptions.DEFAULT)
//// def query = new Document("city", item['B'].toString().replace("市", ""))
//// query.append("region", item['C'].replace("区", ""))
//// query.append("district_name", item["D"])
//// def stat = district_analy_data_client.find(query).first()
//// println(query)
//// item["F"] = response.getSourceAsMap()?.consumption_rating
//// item["G"] = stat?.community_rating
// item["G"] = level_maps.get(item["G"], null)
//// println(item)
//})
//System.exit(0)
////for (def region in ["思明", "翔安", "湖里"]) {
//// DbUtils.seaweed.find(new BasicDBObject("city", "厦门").append("region", region).append("cat", "district").append("status", 0)).each { BasicDBObject dbo ->
//// println("${dbo._id.toString()}\t${dbo.city}市\t${dbo.region}区\t${dbo.name}\t${dbo.address}")
//// }
////}
ExcelUtils.read("C:\\Users\\goojia\\Desktop\\海南省城市均价.xlsx",1).each {List item->
def yearMonth = DateUtil.formatByPattern(item[0], "yyyyMM").toInteger()
println(item)
}
ExcelUtils.multiThreadingModify("C:\\Users\\goojia\\Desktop\\区域楼盘评级.xlsx",{List item->
item["E"] = ProvinceData.getDisplay(item[0])
print(item)
})
@@ -30,6 +30,28 @@ class DingDingMessageUtil {
}
}
static void sendTextMessageAndAt(String access_token, String msg, String atMobile) throws Exception {
HttpRequest request = new HttpRequest();
HttpPost httppost = new HttpPost("https://oapi.dingtalk.com/robot/send?access_token=" + access_token);
httppost.addHeader("Content-Type", "application/json; charset=utf-8");
// 创建消息体,只包含一个手机号
String textMsg = "{ " +
"\"msgtype\": \"text\", " +
"\"text\": {\"content\": \"" + msg + "\"}, " +
"\"at\": {\"atMobiles\": [\"" + atMobile + "\"]}" + // 只 @ 一个手机号
"}";
StringEntity se = new StringEntity(textMsg, "utf-8");
httppost.setEntity(se);
HttpResponse response = request.getHttpClient().execute(httppost);
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
String result = EntityUtils.toString(response.getEntity(), "utf-8");
System.out.println("消息发送结果:" + result);
}
}
static void sendMarkdownMessage(String access_token, String title, String msg){
HttpRequest request = new HttpRequest()
JSONObject requestBody = new JSONObject()