Merge branch 'FSHJOB1.2' into 'master'

房生活用户统计任务优化

See merge request !15
This commit is contained in:
虞继恩
2018-03-06 15:52:43 +08:00
3 changed files with 87 additions and 14 deletions
@@ -0,0 +1,61 @@
package com.fangjia.job.common.core;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* 自定义线程池<br>
* 默认的newFixedThreadPool里的LinkedBlockingQueue是一个无边界队列,如果不断的往里加任务,最终会导致内存的不可控<br>
* 增加了有边界的队列,使用了CallerRunsPolicy拒绝策略
* @author yinjihuan
*
*/
public class FangjiaThreadPoolExecutor {
private static ExecutorService executorService = newFixedThreadPool(50);
private static ExecutorService newFixedThreadPool(int nThreads) {
return new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<Runnable>(10000), new DefaultThreadFactory(), new CallerRunsPolicy());
}
public static void execute(Runnable command) {
executorService.execute(command);
}
public static void shutdown() {
executorService.shutdown();
}
static class DefaultThreadFactory implements ThreadFactory {
private static final AtomicInteger poolNumber = new AtomicInteger(1);
private final ThreadGroup group;
private final AtomicInteger threadNumber = new AtomicInteger(1);
private final String namePrefix;
DefaultThreadFactory() {
SecurityManager s = System.getSecurityManager();
group = (s != null) ? s.getThreadGroup() :
Thread.currentThread().getThreadGroup();
namePrefix = "FSH-pool-" +
poolNumber.getAndIncrement() +
"-thread-";
}
public Thread newThread(Runnable r) {
Thread t = new Thread(group, r,
namePrefix + threadNumber.getAndIncrement(),
0);
if (t.isDaemon())
t.setDaemon(false);
if (t.getPriority() != Thread.NORM_PRIORITY)
t.setPriority(Thread.NORM_PRIORITY);
return t;
}
}
}
@@ -10,6 +10,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import com.cxytiandi.jdbc.util.PageBean;
import com.dangdang.ddframe.job.api.ShardingContext;
import com.dangdang.ddframe.job.api.simple.SimpleJob;
import com.fangjia.job.common.core.FangjiaThreadPoolExecutor;
import com.fangjia.job.common.util.DataUtil;
import com.fangjia.job.common.util.DingDingMessageUtil;
import com.fangjia.job.fsh.document.UserStat;
@@ -64,29 +65,40 @@ public class UserStatJob implements SimpleJob {
@Override
public void execute(ShardingContext shardingContext) {
logger.info("开始执行UserStatJob");
logger.info("开始执行UserStatJob" + shardingContext.getShardingParameter());
// 理财潜客过滤条件
PotentialCustomersFilterConfig financiaFilterConfig = potentialCustomersFilterConfigService.get(1);
final PotentialCustomersFilterConfig financiaFilterConfig = potentialCustomersFilterConfigService.get(1);
// 贷款潜客过滤条件
PotentialCustomersFilterConfig loanFilterConfig = potentialCustomersFilterConfigService.get(2);
final PotentialCustomersFilterConfig loanFilterConfig = potentialCustomersFilterConfigService.get(2);
// 理财潜客分析条件
List<FinancialCustomersConfig> financialCustomersList = financialCustomersConfigService.queryAll();
final List<FinancialCustomersConfig> financialCustomersList = financialCustomersConfigService.queryAll();
// 贷款潜客分析条件
List<LoanCustomersConfig> LoanCustomersConfigList = loanCustomersConfigService.queryAll();
final List<LoanCustomersConfig> LoanCustomersConfigList = loanCustomersConfigService.queryAll();
long total = enterpriseProductUserService.queryCount();
int pages = PageBean.calcPages(total, 1000);
for (int i = 1; i <= pages; i++) {
List<EnterpriseProductUser> users = enterpriseProductUserService.queryByPage(i, 1000);
for (EnterpriseProductUser user : users) {
try {
processStat(user);
// 理财和贷款潜客统计
customerStatService.customerStat(user, financiaFilterConfig, loanFilterConfig, financialCustomersList, LoanCustomersConfigList);
} catch (Exception e) {
logger.error("用户维度统计任务异常", e);
DingDingMessageUtil.sendTextMessage("用户维度统计任务异常:" + e.getMessage());
final EnterpriseProductUser u = user;
if (u.getEid() %2 == Integer.parseInt(shardingContext.getShardingParameter())) {
FangjiaThreadPoolExecutor.execute(new Runnable() {
@Override
public void run() {
try {
processStat(u);
// 理财和贷款潜客统计
customerStatService.customerStat(u, financiaFilterConfig, loanFilterConfig, financialCustomersList, LoanCustomersConfigList);
} catch (Exception e) {
logger.error("用户维度统计任务异常", e);
DingDingMessageUtil.sendTextMessage("用户维度统计任务异常:" + e.getMessage());
}
}
});
}
}
}
logger.info("UserStatJob执行结束");
@@ -55,7 +55,7 @@
<!-- 用户统计任务 每天1点10分执行 -->
<job:simple id="userStatJob" class="com.fangjia.job.fsh.job.UserStatJob" registry-center-ref="regCenter"
sharding-total-count="1" cron="0 10 1 * * ?" sharding-item-parameters=""
sharding-total-count="2" cron="0 10 1 * * ?" sharding-item-parameters="0=0,1=1"
failover="true" description="【房生活】用户维度统计任务,统计出用户的房产,置换,贷款等信息 UserStatJob"
overwrite="true" event-trace-rdb-data-source="elasticJobLog" job-exception-handler="com.fangjia.job.fsh.handler.CustomJobExceptionHandler">