Skip to content

G
gic-platform-enterprise
Commit 5af1db57 by 陶光胜
Options

报表中心

parent 0df30567
Showing with 60 additions and 10 deletions
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/dao/mapper/TabDownloadReportMapper.java
......@@ -3,6 +3,9 @@ package com.gic.enterprise.dao.mapper;
import com.gic.enterprise.entity.TabDownloadReport;
import com.gic.enterprise.qo.DownloadReportQO;
import com.github.pagehelper.Page;
import java.util.List;
/**
*
* @ClassName: TabDownloadReportMapper
......@@ -72,4 +75,6 @@ public interface TabDownloadReportMapper {
* @throws
*/
Page<TabDownloadReport> listDownloadReport(DownloadReportQO downloadReportQO);
List<TabDownloadReport> listUnfinishedTask();
}
\ No newline at end of file
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/DownloadReportService.java
......@@ -6,6 +6,8 @@ import com.gic.enterprise.entity.TabDownloadReport;
import com.gic.enterprise.qo.DownloadReportQO;
import com.github.pagehelper.Page;
import java.util.List;
/**
*
* @Description:
......@@ -111,4 +113,6 @@ public interface DownloadReportService {
* @throws
*/
void updateDataCount(Integer reportId, Integer count);
List<TabDownloadReport> listUnfinishedTask();
}
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/impl/DownloadReportServiceImpl.java
......@@ -16,6 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.Date;
import java.util.List;
/**
*
......@@ -127,6 +128,11 @@ public class DownloadReportServiceImpl implements DownloadReportService {
tabDownloadReportMapper.updateByPrimaryKeySelective(downloadReport);
}
@Override
public List<TabDownloadReport> listUnfinishedTask() {
return tabDownloadReportMapper.listUnfinishedTask();
}
private String getReportId() {
String reportId = CreateRandomUtils.generateNumberStr(8);
TabDownloadReport downloadReport = new TabDownloadReport();
......
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/outer/impl/DownloadReportApiServiceImpl.java
......@@ -75,6 +75,7 @@ public class DownloadReportApiServiceImpl implements DownloadReportApiService {
public void run() {
try {
RMap<Object, Object> map = RedisUtil.getRedisClient().getMap(reportKey);
log.info("监测心跳数据:{}", JSON.toJSONString(map));
if(MapUtils.isNotEmpty(map)){
Iterator<Object> iterator = map.keySet().iterator();
while (iterator.hasNext()){
......@@ -251,6 +252,7 @@ public class DownloadReportApiServiceImpl implements DownloadReportApiService {
@Override
public ServiceResponse<Void> receiveHeart(Integer reportId) {
log.info("接受心跳包：{}", reportId);
RMap<Object, Object> map = RedisUtil.getRedisClient().getMap(reportKey);
map.put(reportId, System.currentTimeMillis());
return ServiceResponse.success();
......
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/outer/impl/HiveDataDownloadApiServiceImpl.java
package com.gic.enterprise.service.outer.impl;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.fastjson.JSON;
import com.gic.api.base.commons.ServiceResponse;
import com.gic.commons.util.EntityUtil;
import com.gic.enterprise.constant.*;
......@@ -15,12 +16,14 @@ import com.gic.enterprise.service.DownloadReportService;
import com.gic.enterprise.service.HiveDataDownloadApiService;
import com.gic.enterprise.utils.FileUtils;
import com.gic.enterprise.utils.HiveConnUtils;
import com.gic.redis.data.util.RedisUtil;
import com.gic.thirdparty.BucketNameEnum;
import com.gic.thirdparty.FileOperateUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.redisson.api.RSet;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
......@@ -58,6 +61,8 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
runDownloadTask(3);
//启动获取任务审核状态定时任务
runApplyTask(5);
//初始化任务
initTask();
}
/***
......@@ -75,9 +80,11 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
DownloadReportDTO downloadReportDTO = EntityUtil.changeEntityByJSON(DownloadReportDTO.class, hiveDataDownloadDTO);
downloadReportDTO.setDataCount(0);
String sql = "";
DownloadTaskPOJO downloadTaskPOJO = null;
try{
sql = getSql(hiveDataDownloadDTO);
downloadReportDTO.setHiveSql(sql);
downloadTaskPOJO = buildTask(hiveDataDownloadDTO, sql);
downloadReportDTO.setHiveSql(JSON.toJSONString(downloadTaskPOJO));
}catch (Exception e){
log.warn(e);
return ServiceResponse.failure(ErrorCode.SYSTEM_ERROR.getCode(), e.getMessage());
......@@ -88,21 +95,28 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
if(!response.isSuccess()){
return response;
}
buildTask(response.getResult(), hiveDataDownloadDTO, sql);
downloadTaskPOJO.setReportId(response.getResult());
taskList.add(downloadTaskPOJO);
return response;
}
private void buildTask(Integer reportId, HiveDataDownloadDTO hiveDataDownloadDTO, String sql){
private DownloadTaskPOJO buildTask(HiveDataDownloadDTO hiveDataDownloadDTO, String sql){
DownloadTaskPOJO downloadTaskPOJO = new DownloadTaskPOJO();
downloadTaskPOJO.setReportId(reportId);
downloadTaskPOJO.setHiveSql(sql);
downloadTaskPOJO.setCanDownload(!DownloadReportDataTypeEnum.COMPLETE_DATA.getCode().equals(hiveDataDownloadDTO.getDataType()));
downloadTaskPOJO.setDesenField(hiveDataDownloadDTO.getDesenField());
downloadTaskPOJO.setExcelExtension(hiveDataDownloadDTO.getExcelExtension());
downloadTaskPOJO.setTableName(hiveDataDownloadDTO.getTableName());
downloadTaskPOJO.setFileName(hiveDataDownloadDTO.getFileName());
downloadTaskPOJO.setExcelExtension(hiveDataDownloadDTO.getExcelExtension());
taskList.add(downloadTaskPOJO);
return downloadTaskPOJO;
}
private void initTask(){
List<TabDownloadReport> list = downloadReportService.listUnfinishedTask();
for(TabDownloadReport report : list){
DownloadTaskPOJO downloadTaskPOJO = JSON.parseObject(report.getHiveSql(), DownloadTaskPOJO.class);
taskList.add(downloadTaskPOJO);
}
}
......@@ -115,6 +129,10 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
log.info("定时执行任务下载：{}", taskList.size());
while (iterator.hasNext()){
DownloadTaskPOJO task = iterator.next();
boolean exist = RedisUtil.getRedisClient().getBucket("report:" + task.getReportId()).trySet(task.getReportId());
if(exist){
continue;
}
if(task.isCanDownload()){
log.info("开始下载任务:{}", task.getReportId());
iterator.remove();
......
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/utils/FileUtils.java
......@@ -9,9 +9,11 @@ import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelper;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import java.io.*;
......@@ -34,7 +36,7 @@ public class FileUtils {
String filePath = getFilePath(fileName, fileExt);
File tmpFile = createTmpFile(filePath);
if (fileExt.equals(ExcelExtensionEnum.CSV.getCode())) {
createCsvFile(rs, dataType, desenField, tmpFile, fileName, fileExt);
createCsvFile(rs, dataType, desenField, tmpFile, fileName);
} else {
createXls(rs, dataType, desenField, tmpFile, fileName, fileExt);
}
......@@ -46,7 +48,12 @@ public class FileUtils {
private static void createXls(ResultSet rs, Integer dataType, List<String> desenField, File tempFile, String fileName, Integer fileExt){
log.info("开始生成文件 " + fileName + ".xlsx");
try{
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Workbook wb = null;
if(fileExt.equals(ExcelExtensionEnum.EXCEL_2003.getCode())){
wb = new SXSSFWorkbook(100); // 内存中保留 100 行
} else {
wb = new HSSFWorkbook();
}
Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0);
Cell cell;
......@@ -76,14 +83,13 @@ public class FileUtils {
FileOutputStream fileOut = new FileOutputStream(tempFile);
wb.write(fileOut);
fileOut.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
log.info("临时文件已生成， " + fileName + ".xlsx");
}catch (Exception e){
log.warn(e);
}
}
private static void createCsvFile(ResultSet rs, Integer dataType, List<String> desenField, File tempFile, String fileName, Integer fileExt){
private static void createCsvFile(ResultSet rs, Integer dataType, List<String> desenField, File tempFile, String fileName){
try {
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(tempFile), Charset.forName("GBK"));
ResultSetHelper helper = new CsvResultSetHelper(dataType, desenField);
......
gic-platform-enterprise-service/src/main/resources/mapper/TabDownloadReportMapper.xml
......@@ -406,4 +406,11 @@
</if>
order by create_time desc
</select>
<select id="listUnfinishedTask" resultMap="BaseResultMap">
select
<include refid="Base_Column_List" />
from tab_download_report
where status in(5,6) or audit_result = 0
order by create_time desc
</select>
</mapper>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment