Skip to content

G
gic-platform-enterprise
Commit 7b8447fc by guojuxing
Options

Merge remote-tracking branch 'origin/developer' into developer

parents 65495ead 1948ce29
Showing with 107 additions and 63 deletions
gic-platform-enterprise-api/src/main/java/com/gic/enterprise/constant/DownloadReportStatusEnum.java
......@@ -39,8 +39,9 @@ public enum DownloadReportStatusEnum {
*/
GENERATED(5, "已生成"),
ERROR(6, "生成异常")
;
ERROR(6, "生成异常"),
WAITLINE(7, "排队中");
private Integer code;
private String msg;
......
gic-platform-enterprise-api/src/main/java/com/gic/enterprise/dto/HiveDataDownloadDTO.java
......@@ -2,27 +2,40 @@ package com.gic.enterprise.dto;
import com.gic.enterprise.constant.ExcelExtensionEnum;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class HiveDataDownloadDTO implements Serializable {
public interface hiveData{
}
@NotNull(message = "企业id不能为空", groups = hiveData.class)
private Integer enterpriseId;
/**表名**/
@NotNull(message = "表名不能为空", groups = hiveData.class)
private String tableName;
/**查询条件**/
@NotEmpty(message = "查询字段不能为空", groups = hiveData.class)
private List<HiveQueryCondition> queryConditionList;
/**申请人用户id*/
@NotNull(message = "申请人不能为空", groups = hiveData.class)
private Integer applyUserId;
/**数据源（频道页），用 "-" 隔开上级和本级**/
private String dataContent;
/**报告类型 1：完整数据， 2：脱敏数据， 3：二维码***/
@NotNull(message = "报告类型不能为空", groups = hiveData.class)
private Integer dataType;
/**可以尝试用request.getHeader("Referer")获取dataUrl**/
private String dataUrl;
/**下载文件名*/
@NotNull(message = "文件名称不能为空", groups = hiveData.class)
private String fileName;
/**文件扩展 1.xlsx 2.xls 3.csv**/
@NotNull(message = "文件类型不能为空", groups = hiveData.class)
private Integer excelExtension;
/**待下载数据的开始时间**/
private String reportStartTime;
......
gic-platform-enterprise-api/src/main/java/com/gic/enterprise/qo/DownloadReportQO.java
......@@ -24,7 +24,7 @@ public class DownloadReportQO extends PageQO {
@Max(value = 3, message = "不存在该类型", groups = SelectValidView.class)
private Integer dataType;
@Min(value = 0, message = "不存在该状态", groups = SelectValidView.class)
@Max(value = 5, message = "不存在该状态", groups = SelectValidView.class)
@Max(value = 7, message = "不存在该状态", groups = SelectValidView.class)
private Integer downloadReportStatus;
private String searchValue;
......
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/dao/mapper/TabDownloadReportMapper.java
......@@ -3,6 +3,9 @@ package com.gic.enterprise.dao.mapper;
import com.gic.enterprise.entity.TabDownloadReport;
import com.gic.enterprise.qo.DownloadReportQO;
import com.github.pagehelper.Page;
import java.util.List;
/**
*
* @ClassName: TabDownloadReportMapper
......@@ -72,4 +75,6 @@ public interface TabDownloadReportMapper {
* @throws
*/
Page<TabDownloadReport> listDownloadReport(DownloadReportQO downloadReportQO);
List<TabDownloadReport> listUnfinishedTask();
}
\ No newline at end of file
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/DownloadReportService.java
......@@ -6,6 +6,8 @@ import com.gic.enterprise.entity.TabDownloadReport;
import com.gic.enterprise.qo.DownloadReportQO;
import com.github.pagehelper.Page;
import java.util.List;
/**
*
* @Description:
......@@ -111,4 +113,6 @@ public interface DownloadReportService {
* @throws
*/
void updateDataCount(Integer reportId, Integer count);
List<TabDownloadReport> listUnfinishedTask();
}
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/impl/DownloadReportServiceImpl.java
......@@ -16,6 +16,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.Date;
import java.util.List;
/**
*
......@@ -50,6 +51,7 @@ public class DownloadReportServiceImpl implements DownloadReportService {
TabDownloadReport downloadReport = new TabDownloadReport();
downloadReport.setDownloadReportId(downloadReportId);
downloadReport.setDownloadUrl(downloadUrl);
downloadReport.setStatus(DownloadReportStatusEnum.GENERATED.getCode());
if(count != null){
downloadReport.setDataCount(count);
}
......@@ -60,7 +62,9 @@ public class DownloadReportServiceImpl implements DownloadReportService {
public Page<TabDownloadReport> listDownloadReport(DownloadReportQO downloadReportQO) {
PageHelper.startPage(downloadReportQO.getCurrentPage(), downloadReportQO.getPageSize());
if (downloadReportQO.getDownloadReportStatus() != null) {
switch (downloadReportQO.getDownloadReportStatus()) {
Integer downloadReportStatus = downloadReportQO.getDownloadReportStatus();
downloadReportQO.setDownloadReportStatus(null);
switch (downloadReportStatus) {
case 0:
downloadReportQO.setAuditResult(DataSecurityAuditEnum.WAIT.getCode());
downloadReportQO.setLtAuditExpireTime(new Date());
......@@ -76,13 +80,16 @@ public class DownloadReportServiceImpl implements DownloadReportService {
downloadReportQO.setHasDownloadUrl(null);
break;
case 4:
downloadReportQO.setAuditResult(DataSecurityAuditEnum.PASS.getCode());
downloadReportQO.setHasDownloadUrl(false);
downloadReportQO.setDownloadReportStatus(4);
break;
case 5:
downloadReportQO.setAuditResult(DataSecurityAuditEnum.PASS.getCode());
downloadReportQO.setHasDownloadUrl(true);
downloadReportQO.setDownloadReportStatus(5);
break;
case 6:
downloadReportQO.setDownloadReportStatus(6);
break;
case 7:
downloadReportQO.setDownloadReportStatus(7);
default:
break;
}
......@@ -127,6 +134,11 @@ public class DownloadReportServiceImpl implements DownloadReportService {
tabDownloadReportMapper.updateByPrimaryKeySelective(downloadReport);
}
@Override
public List<TabDownloadReport> listUnfinishedTask() {
return tabDownloadReportMapper.listUnfinishedTask();
}
private String getReportId() {
String reportId = CreateRandomUtils.generateNumberStr(8);
TabDownloadReport downloadReport = new TabDownloadReport();
......
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/outer/impl/DownloadReportApiServiceImpl.java
......@@ -75,15 +75,17 @@ public class DownloadReportApiServiceImpl implements DownloadReportApiService {
public void run() {
try {
RMap<Object, Object> map = RedisUtil.getRedisClient().getMap(reportKey);
log.info("监测心跳数据:{}", JSON.toJSONString(map));
if(MapUtils.isNotEmpty(map)){
Iterator<Object> iterator = map.keySet().iterator();
while (iterator.hasNext()){
Object next = iterator.next();
long putTime = (long) map.get(next);
if((System.currentTimeMillis() - putTime) > 10*60*1000){
log.info("报告超时:{}", next);
//超过十分钟 如果报告还在生成中则设置为失败
TabDownloadReport report = downloadReportService.getById((Integer) next);
if(report != null && report.getStatus().equals(DownloadReportStatusEnum.BUILD.getCode())){
if(report != null && !report.getStatus().equals(DownloadReportStatusEnum.GENERATED.getCode()) && report.getAuditResult().equals(DownloadReportStatusEnum.PASS.getCode())){
downloadReportService.updateDownloadStatus((Integer)next, DownloadReportStatusEnum.ERROR);
}
map.remove(next);
......@@ -131,7 +133,12 @@ public class DownloadReportApiServiceImpl implements DownloadReportApiService {
DateUtil.dateToStr(downloadReportDTO.getReportEndTime(), DateUtil.FORMAT_DATE_10);
downloadReportDTO.setFileName(fileName);
}
if("hive".equals(downloadReportDTO.getDataSource())){
downloadReportDTO.setStatus(DownloadReportStatusEnum.WAITLINE.getCode());
} else {
//sdk下载没有任务队列 直接生成中状态
downloadReportDTO.setStatus(DownloadReportStatusEnum.BUILD.getCode());
}
Integer downloadReportId = downloadReportService.save(downloadReportDTO);
return EnterpriseServiceResponse.success(downloadReportId);
}
......@@ -160,11 +167,8 @@ public class DownloadReportApiServiceImpl implements DownloadReportApiService {
if (DataSecurityAuditEnum.WAIT.getCode().equals(downloadReportDTO.getAuditResult()) && DateUtil.compareDate(currDate, downloadReportDTO.getAuditExpireTime())) {
downloadReportDTO.setDownloadReportStatus(DownloadReportStatusEnum.OVER_TIME.getCode());
downloadReportDTO.setAuditReason("审核超时");
} else if (DataSecurityAuditEnum.PASS.getCode().equals(downloadReportDTO.getAuditResult())) {
// 判断生成状态
downloadReportDTO.setDownloadReportStatus(StringUtils.isNotBlank(downloadReportDTO.getDownloadUrl()) ? DownloadReportStatusEnum.GENERATED.getCode() : DownloadReportStatusEnum.BUILD.getCode());
} else {
downloadReportDTO.setDownloadReportStatus(downloadReportDTO.getAuditResult());
downloadReportDTO.setDownloadReportStatus(downloadReportDTO.getStatus());
}
downloadReportDTO.setDownloadUrl(null);
}
......@@ -251,9 +255,9 @@ public class DownloadReportApiServiceImpl implements DownloadReportApiService {
@Override
public ServiceResponse<Void> receiveHeart(Integer reportId) {
log.info("接受心跳包：{}", reportId);
RMap<Object, Object> map = RedisUtil.getRedisClient().getMap(reportKey);
map.put(reportId, System.currentTimeMillis());
return ServiceResponse.success();
}
}
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/outer/impl/HiveDataDownloadApiServiceImpl.java
package com.gic.enterprise.service.outer.impl;
import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.alibaba.fastjson.JSON;
import com.gic.api.base.commons.ServiceResponse;
import com.gic.commons.util.EntityUtil;
import com.gic.enterprise.constant.*;
......@@ -15,12 +16,15 @@ import com.gic.enterprise.service.DownloadReportService;
import com.gic.enterprise.service.HiveDataDownloadApiService;
import com.gic.enterprise.utils.FileUtils;
import com.gic.enterprise.utils.HiveConnUtils;
import com.gic.redis.data.util.RedisUtil;
import com.gic.store.utils.valid.ValidUtil;
import com.gic.thirdparty.BucketNameEnum;
import com.gic.thirdparty.FileOperateUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.redisson.api.RSet;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
......@@ -58,6 +62,8 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
runDownloadTask(3);
//启动获取任务审核状态定时任务
runApplyTask(5);
//初始化任务
initTask();
}
/***
......@@ -72,12 +78,19 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
@Override
public ServiceResponse<Integer> createDownloadTask(HiveDataDownloadDTO hiveDataDownloadDTO) {
//校验参数
ServiceResponse paramResult = ValidUtil.allCheckValidate(hiveDataDownloadDTO, HiveDataDownloadDTO.hiveData.class);
if(!paramResult.isSuccess()){
return paramResult;
}
DownloadReportDTO downloadReportDTO = EntityUtil.changeEntityByJSON(DownloadReportDTO.class, hiveDataDownloadDTO);
downloadReportDTO.setDataCount(0);
String sql = "";
DownloadTaskPOJO downloadTaskPOJO = null;
try{
sql = getSql(hiveDataDownloadDTO);
downloadReportDTO.setHiveSql(sql);
downloadTaskPOJO = buildTask(hiveDataDownloadDTO, sql);
downloadReportDTO.setHiveSql(JSON.toJSONString(downloadTaskPOJO));
}catch (Exception e){
log.warn(e);
return ServiceResponse.failure(ErrorCode.SYSTEM_ERROR.getCode(), e.getMessage());
......@@ -88,21 +101,28 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
if(!response.isSuccess()){
return response;
}
buildTask(response.getResult(), hiveDataDownloadDTO, sql);
downloadTaskPOJO.setReportId(response.getResult());
taskList.add(downloadTaskPOJO);
return response;
}
private void buildTask(Integer reportId, HiveDataDownloadDTO hiveDataDownloadDTO, String sql){
private DownloadTaskPOJO buildTask(HiveDataDownloadDTO hiveDataDownloadDTO, String sql){
DownloadTaskPOJO downloadTaskPOJO = new DownloadTaskPOJO();
downloadTaskPOJO.setReportId(reportId);
downloadTaskPOJO.setHiveSql(sql);
downloadTaskPOJO.setCanDownload(!DownloadReportDataTypeEnum.COMPLETE_DATA.getCode().equals(hiveDataDownloadDTO.getDataType()));
downloadTaskPOJO.setDesenField(hiveDataDownloadDTO.getDesenField());
downloadTaskPOJO.setExcelExtension(hiveDataDownloadDTO.getExcelExtension());
downloadTaskPOJO.setTableName(hiveDataDownloadDTO.getTableName());
downloadTaskPOJO.setFileName(hiveDataDownloadDTO.getFileName());
downloadTaskPOJO.setExcelExtension(hiveDataDownloadDTO.getExcelExtension());
taskList.add(downloadTaskPOJO);
return downloadTaskPOJO;
}
private void initTask(){
List<TabDownloadReport> list = downloadReportService.listUnfinishedTask();
for(TabDownloadReport report : list){
DownloadTaskPOJO downloadTaskPOJO = JSON.parseObject(report.getHiveSql(), DownloadTaskPOJO.class);
taskList.add(downloadTaskPOJO);
}
}
......@@ -115,6 +135,10 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
log.info("定时执行任务下载：{}", taskList.size());
while (iterator.hasNext()){
DownloadTaskPOJO task = iterator.next();
boolean exist = RedisUtil.getRedisClient().getBucket("report:" + task.getReportId()).trySet(task.getReportId());
if(exist){
continue;
}
if(task.isCanDownload()){
log.info("开始下载任务:{}", task.getReportId());
iterator.remove();
......@@ -233,38 +257,4 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
return sql.toString();
}
public static void main(String[] args){
HiveDataDownloadDTO dataDownloadDTO = new HiveDataDownloadDTO();
dataDownloadDTO.setEnterpriseId(1166);
dataDownloadDTO.setApplyUserId(13805);
dataDownloadDTO.setDataContent("用户-用户列表");
dataDownloadDTO.setDataType(DownloadReportDataTypeEnum.DESENSITIZATION_DATA.getCode());
dataDownloadDTO.setDataUrl("https://four.gicdev.com/member-tag/member-tag/tag-val-edit?id=264137629747888139&name=111111");
//dataDownloadDTO.setDesenField();
dataDownloadDTO.setExcelExtension(ExcelExtensionEnum.CSV.getCode());
dataDownloadDTO.setFileName("boomhivetest");
//dataDownloadDTO.setSql("SELECT order_id AS `订单号` FROM extract_order_information;");
dataDownloadDTO.setTableName("extract_order_information");
List<HiveQueryCondition> list = new ArrayList<>();
HiveQueryCondition queryCondition = new HiveQueryCondition();
queryCondition.setFieldName("order_id");
queryCondition.setFieldMark("订单号");
list.add(queryCondition);
HiveQueryCondition queryCondition1 = new HiveQueryCondition();
queryCondition1.setFieldName("enterprise_id");
queryCondition1.setFieldMark("企业id");
queryCondition1.setFieldFilterOptEnum(FieldFilterOptEnum.EQ);
queryCondition1.setFieldTypeEnum(FieldTypeEnum.STRING);
queryCondition1.setFilterValue(Arrays.asList("ff8080816a36326c016a53380d8b5f52"));
list.add(queryCondition1);
dataDownloadDTO.setQueryConditionList(list);
try{
String sql = new HiveDataDownloadApiServiceImpl().getSql(dataDownloadDTO);
System.out.println("sql = " + sql);
}catch (Exception e){
e.printStackTrace();
}
}
}
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/utils/FileUtils.java
......@@ -9,9 +9,11 @@ import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelper;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import java.io.*;
......@@ -34,7 +36,7 @@ public class FileUtils {
String filePath = getFilePath(fileName, fileExt);
File tmpFile = createTmpFile(filePath);
if (fileExt.equals(ExcelExtensionEnum.CSV.getCode())) {
createCsvFile(rs, dataType, desenField, tmpFile, fileName, fileExt);
createCsvFile(rs, dataType, desenField, tmpFile, fileName);
} else {
createXls(rs, dataType, desenField, tmpFile, fileName, fileExt);
}
......@@ -46,7 +48,12 @@ public class FileUtils {
private static void createXls(ResultSet rs, Integer dataType, List<String> desenField, File tempFile, String fileName, Integer fileExt){
log.info("开始生成文件 " + fileName + ".xlsx");
try{
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Workbook wb = null;
if(fileExt.equals(ExcelExtensionEnum.EXCEL_2003.getCode())){
wb = new SXSSFWorkbook(100); // 内存中保留 100 行
} else {
wb = new HSSFWorkbook();
}
Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0);
Cell cell;
......@@ -76,14 +83,13 @@ public class FileUtils {
FileOutputStream fileOut = new FileOutputStream(tempFile);
wb.write(fileOut);
fileOut.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
log.info("临时文件已生成， " + fileName + ".xlsx");
}catch (Exception e){
log.warn(e);
}
}
private static void createCsvFile(ResultSet rs, Integer dataType, List<String> desenField, File tempFile, String fileName, Integer fileExt){
private static void createCsvFile(ResultSet rs, Integer dataType, List<String> desenField, File tempFile, String fileName){
try {
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(tempFile), Charset.forName("GBK"));
ResultSetHelper helper = new CsvResultSetHelper(dataType, desenField);
......
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/utils/HiveConnUtils.java
......@@ -35,9 +35,6 @@ public class HiveConnUtils {
}
public static Connection getConnnection(String url, String user, String password){
if(!start){
//todo 启动线程检查连接状态
}
try {
String key = url + ":" + user;
Connection connection = conCache.get(key);
......
gic-platform-enterprise-service/src/main/resources/mapper/TabDownloadReportMapper.xml
......@@ -360,7 +360,7 @@
select
<include refid="Base_Column_List" />
from tab_download_report
where status = 1
where 1 = 1
<if test="enterpriseId != null ">
and enterprise_id = #{enterpriseId}
</if>
......@@ -404,6 +404,16 @@
<if test="projectCode == null or projectCode == '' ">
and project_code in ('gic', 'mall')
</if>
<if test="downloadReportStatus != null">
and status=#{downloadReportStatus}
</if>
order by create_time desc
</select>
<select id="listUnfinishedTask" resultMap="BaseResultMap">
select
<include refid="Base_Column_List" />
from tab_download_report
where (status in(5,6) or audit_result = 0) and data_source='hive'
order by create_time desc
</select>
</mapper>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment