Commit b6f06a15 by fudahua

Merge branch 'developer' into 'master'

Developer

See merge request !50
parents 27cb83ac 7a5a41ce
...@@ -114,4 +114,14 @@ public class FreeQueryTaskCondition implements Serializable { ...@@ -114,4 +114,14 @@ public class FreeQueryTaskCondition implements Serializable {
public void setDesensiType(Integer desensiType) { public void setDesensiType(Integer desensiType) {
this.desensiType = desensiType; this.desensiType = desensiType;
} }
private int amount;
public int getAmount() {
return amount;
}
public void setAmount(int amount) {
this.amount = amount;
}
} }
package com.gic.cloud.data.hook.service;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class FileUtil {
private static final int BUFFER_SIZE = 2 * 1024;
/**
* 压缩zip
* @param srcFiles
* @param out
* @throws RuntimeException
*/
public static void toZip(List<File> srcFiles, OutputStream out) throws RuntimeException {
long start = System.currentTimeMillis();
ZipOutputStream zos = null;
try {
zos = new ZipOutputStream(out);
for (File srcFile : srcFiles) {
byte[] buf = new byte[BUFFER_SIZE];
zos.putNextEntry(new ZipEntry(srcFile.getName()));
int len;
FileInputStream in = new FileInputStream(srcFile);
while ((len = in.read(buf)) != -1) {
zos.write(buf, 0, len);
}
zos.closeEntry();
in.close();
}
long end = System.currentTimeMillis();
System.out.println("压缩完成,耗时:" + (end - start) + " ms");
} catch (Exception e) {
throw new RuntimeException("zip error from ZipUtils", e);
} finally {
if (zos != null) {
try {
zos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
* 获取限制调试
* @return
*/
public static Integer getLimitSize(){
Config appConfig = ConfigService.getAppConfig();
return appConfig.getIntProperty("xls.size.limit", 1000000);
}
}
...@@ -76,6 +76,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -76,6 +76,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private static final Integer SMALL_SIZE = 2; private static final Integer SMALL_SIZE = 2;
private static final Integer XLS_SIZE = 1000000;
private static final Map<String, String> bigTaskRunningMap = new ConcurrentHashMap<>(); private static final Map<String, String> bigTaskRunningMap = new ConcurrentHashMap<>();
@Autowired @Autowired
...@@ -809,7 +811,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -809,7 +811,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
if (conn != null) { if (conn != null) {
try { try {
Statement stat = conn.createStatement(); Statement stat = conn.createStatement();
Integer limitSize = FileUtil.getLimitSize();
// stat.setQueryTimeout(60 * 1000); // stat.setQueryTimeout(60 * 1000);
stat.execute("REFRESH TABLE " + condition.getTableId()); // 强制刷新表结构 stat.execute("REFRESH TABLE " + condition.getTableId()); // 强制刷新表结构
ResultSet rs = stat.executeQuery(fullQuery); ResultSet rs = stat.executeQuery(fullQuery);
...@@ -817,6 +819,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -817,6 +819,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
ResultSetHelper helper = new CsvResultSetHelper(queryDataType, condition.getDecryptFilters(), condition.getConditions(), condition.getEnterpriseIds().get(0), decryptKeyService); ResultSetHelper helper = new CsvResultSetHelper(queryDataType, condition.getDecryptFilters(), condition.getConditions(), condition.getEnterpriseIds().get(0), decryptKeyService);
// 生成指定格式下载元文件 // 生成指定格式下载元文件
String originalFilePath = ""; String originalFilePath = "";
boolean zipFlag=false;
List<File> listFile=new ArrayList<>();
if (task.getFormat().equals(DownloadFileFormat.CSV)) { // 如果指定为 CSV 格式 if (task.getFormat().equals(DownloadFileFormat.CSV)) { // 如果指定为 CSV 格式
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标下载文件 " + condition.getTaskId() + ".csv"); logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标下载文件 " + condition.getTaskId() + ".csv");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv"; originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv";
...@@ -832,69 +836,97 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -832,69 +836,97 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
writer.close(); writer.close();
out.close();//记得关闭资源 out.close();//记得关闭资源
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".csv"); logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".csv");
listFile.add(new File(originalFilePath));
} else { // 如果指定为 XLS 格式 } else { // 如果指定为 XLS 格式
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标下载文件 " + condition.getTaskId() + ".xlsx"); logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx"; int filePos=0;
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行 if (condition.getAmount()>limitSize){
Sheet sheet = wb.createSheet(); int num = (condition.getAmount() / limitSize)+(condition.getAmount()%limitSize>0?1:0);
Row row = sheet.createRow(0); while (num-->0) {
Cell cell; originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId()+"-"+filePos + ".xlsx";
String[] columnNames = helper.getColumnNames(rs); filePos++;
for(int j =0; j<columnNames.length; j++){ saveXlsSplit(originalFilePath,helper,rs,condition);
cell = row.createCell(j); File file = new File(originalFilePath);
cell.setCellValue(columnNames[j]); listFile.add(file);
}
// 遍历输出行
int rowCount = 0;
while (rs.next()) {
rowCount++;
row = sheet.createRow(rowCount);
String[] columnValues = helper.getColumnValues(rs, true, "", "");
for(int j=0; j<columnValues.length; j++){
row.createCell(j).setCellValue(columnValues[j]);
} }
} // WHILE OVER zipFlag=true;
FileOutputStream fileOut = new FileOutputStream(originalFilePath); }else{
wb.write(fileOut); originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式 saveXlsSplit(originalFilePath,helper,rs,condition);
fileOut.close(); File file = new File(originalFilePath);
//wb.close(); listFile.add(file);
wb.dispose(); // SXSSFWorkbook 没有 close 方法 }
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx"); String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
OutputStream os = new FileOutputStream(zipFilePath);
FileUtil.toZip(listFile,os);
// originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
// SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
// Sheet sheet = wb.createSheet();
// Row row = sheet.createRow(0);
// Cell cell;
// String[] columnNames = helper.getColumnNames(rs);
// for(int j =0; j<columnNames.length; j++){
// cell = row.createCell(j);
// cell.setCellValue(columnNames[j]);
// }
// // 遍历输出行
// int rowCount = 0;
// while (rs.next()) {
// rowCount++;
// row = sheet.createRow(rowCount);
// String[] columnValues = helper.getColumnValues(rs, true, "", "");
// for(int j=0; j<columnValues.length; j++){
// row.createCell(j).setCellValue(columnValues[j]);
// }
// } // WHILE OVER
// FileOutputStream fileOut = new FileOutputStream(originalFilePath);
// wb.write(fileOut);
// //fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
// fileOut.close();
// //wb.close();
// wb.dispose(); // SXSSFWorkbook 没有 close 方法
// logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
} // IF ELSE OVER } // IF ELSE OVER
String cloudFileUrl = "https://"; String cloudFileUrl = "https://";
// 如果指定压缩,则使用之 // 如果指定压缩,则使用之
//if (task.getFormat().equals("zip")) { //if (task.getFormat().equals("zip")) {
String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx"; String taskFileExt = task.getUseCompress().equals(Global.YES)||zipFlag ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
if (task.getUseCompress().equals(Global.YES)) { if (zipFlag||task.getUseCompress().equals(Global.YES)) {
logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标压缩文件 " + condition.getTaskId() + ".zip"); logger.info("[ runDownloadTask.run ]: {}", "准备生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip"; String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
OutputStream os = new FileOutputStream(zipFilePath);
File zipFile = new File(zipFilePath); File zipFile = new File(zipFilePath);
ZipOutputStream zos = null; FileUtil.toZip(listFile,os);
byte[] buf = new byte[1024];
int length = 0; //
try { //
OutputStream os = new FileOutputStream(zipFilePath); // ZipOutputStream zos = null;
BufferedOutputStream bos = new BufferedOutputStream(os); // byte[] buf = new byte[1024];
zos = new ZipOutputStream(bos); // int length = 0;
zos.setLevel(6); // 压缩率选择 0-9 // try {
InputStream is = new FileInputStream(originalFilePath); // OutputStream os = new FileOutputStream(zipFilePath);
BufferedInputStream bis = new BufferedInputStream(is); // BufferedOutputStream bos = new BufferedOutputStream(os);
zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1))); // zos = new ZipOutputStream(bos);
while ((length = bis.read(buf)) > 0) { // zos.setLevel(6); // 压缩率选择 0-9
zos.write(buf, 0, length); // InputStream is = new FileInputStream(originalFilePath);
} // BufferedInputStream bis = new BufferedInputStream(is);
bis.close(); // zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
is.close(); // while ((length = bis.read(buf)) > 0) {
//bos.close(); // zos.write(buf, 0, length);
//os.close(); // }
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip"); // bis.close();
} catch (Exception ex2) { // is.close();
throw ex2; // //bos.close();
} finally { // //os.close();
zos.closeEntry(); // logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
zos.close(); // } catch (Exception ex2) {
} // throw ex2;
// } finally {
// zos.closeEntry();
// zos.close();
// }
logger.info("[ 开始上传压缩文件到腾讯云 ]: {}", task.getId()); logger.info("[ 开始上传压缩文件到腾讯云 ]: {}", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName()); cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
} else { } else {
...@@ -929,6 +961,43 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -929,6 +961,43 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
} // IF OVER } // IF OVER
} }
private void saveXlsSplit(String originalFilePath,ResultSetHelper helper,ResultSet rs,FlatQueryTaskCondition condition){
try {
Integer limitSize = FileUtil.getLimitSize();
// String originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0);
Cell cell;
String[] columnNames = helper.getColumnNames(rs);
for(int j =0; j<columnNames.length; j++){
cell = row.createCell(j);
cell.setCellValue(columnNames[j]);
}
// 遍历输出行
int rowCount = 0;
while (rowCount<limitSize&&rs.next()) {
rowCount++;
row = sheet.createRow(rowCount);
String[] columnValues = helper.getColumnValues(rs, true, "", "");
for(int j=0; j<columnValues.length; j++){
row.createCell(j).setCellValue(columnValues[j]);
}
} // WHILE OVER
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
}catch (Exception e) {
e.printStackTrace();
logger.info("异常:{}",e);
}
}
/** 下载申请检查计时器 */ /** 下载申请检查计时器 */
//private Timer applyTimer = new Timer(); //private Timer applyTimer = new Timer();
......
...@@ -51,6 +51,8 @@ public class FreeQueryServiceImpl implements IFreeQueryService { ...@@ -51,6 +51,8 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
/** 日志类 */ /** 日志类 */
private static LogPak log = new LogPak(FreeQueryServiceImpl.class); private static LogPak log = new LogPak(FreeQueryServiceImpl.class);
private static final Integer XLS_SIZE = 10000;
/** csv / xls 下载目录 */ /** csv / xls 下载目录 */
public static final String SAVE_FOLDER = "/usr/local/data-hook-file"; public static final String SAVE_FOLDER = "/usr/local/data-hook-file";
...@@ -308,6 +310,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService { ...@@ -308,6 +310,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
// 生成条件 // 生成条件
FreeQueryTaskCondition condition = new FreeQueryTaskCondition(); FreeQueryTaskCondition condition = new FreeQueryTaskCondition();
condition.setAmount(amount);
condition.setTaskId(curTask.getId()); condition.setTaskId(curTask.getId());
condition.setSql(sql); condition.setSql(sql);
condition.setEnterpriseId(enterpriseId); condition.setEnterpriseId(enterpriseId);
...@@ -334,6 +337,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService { ...@@ -334,6 +337,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
if(record.getDesensiType() != null){ if(record.getDesensiType() != null){
FreeQueryTaskCondition condition = new FreeQueryTaskCondition(); FreeQueryTaskCondition condition = new FreeQueryTaskCondition();
condition.setTaskId(record.getId()); condition.setTaskId(record.getId());
condition.setAmount(record.getAmount());
condition.setSql(record.getRealSql()); condition.setSql(record.getRealSql());
condition.setEnterpriseId(record.getEnterpriseId()); condition.setEnterpriseId(record.getEnterpriseId());
condition.setDesensiType(record.getDesensiType()); condition.setDesensiType(record.getDesensiType());
...@@ -394,12 +398,16 @@ public class FreeQueryServiceImpl implements IFreeQueryService { ...@@ -394,12 +398,16 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
} // IF OVER } // IF OVER
} // FOR OVER } // FOR OVER
if (condition != null) { if (condition != null) {
Integer limitSize = FileUtil.getLimitSize();
// 更新任务状态 // 更新任务状态
DownloadTask task = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId()); DownloadTask task = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId());
task.setStatus(DownloadTaskStatus.BUILDING); task.setStatus(DownloadTaskStatus.BUILDING);
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task); DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
String fullQuery = condition.getSql(); String fullQuery = condition.getSql();
boolean zipFlag=false;
List<File> listFile=new ArrayList<>();
Connection conn = MysqlHelper.getFreeQueryConnection(condition.getEnterpriseId()); Connection conn = MysqlHelper.getFreeQueryConnection(condition.getEnterpriseId());
if (conn != null) { if (conn != null) {
...@@ -432,97 +440,122 @@ public class FreeQueryServiceImpl implements IFreeQueryService { ...@@ -432,97 +440,122 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".csv"); log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".csv");
} else { } else {
log.debug("runDownloadTask.run", "准备生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx"); log.debug("runDownloadTask.run", "准备生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx"; int filePos=0;
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行 if (condition.getAmount()>limitSize){
Sheet sheet = wb.createSheet(); int num = (condition.getAmount() / limitSize)+(condition.getAmount()%limitSize>0?1:0);
Row row = sheet.createRow(0); while (num-->0) {
Cell cell; originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId()+"-"+filePos + ".xlsx";
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头 filePos++;
String colName = rs.getMetaData().getColumnLabel(j + 1); saveXlsx(condition,originalFilePath,rs,task);
cell = row.createCell(j); File file = new File(originalFilePath);
cell.setCellValue(colName); listFile.add(file);
}
zipFlag=true;
}else{
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
saveXlsx(condition,originalFilePath,rs,task);
File file = new File(originalFilePath);
listFile.add(file);
} }
// 遍历输出行
int rowCount = 0; // SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
while (rs.next()) { // Sheet sheet = wb.createSheet();
rowCount++; // Row row = sheet.createRow(0);
row = sheet.createRow(rowCount); // Cell cell;
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头
//String c = rs.getString(j + 1); // String colName = rs.getMetaData().getColumnLabel(j + 1);
//row.createCell(j).setCellValue(c); // cell = row.createCell(j);
String cName = rs.getMetaData().getColumnName(j+1); // cell.setCellValue(colName);
if (task.getQueryDataType() == QueryDataType.SAFE // }
&& (FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName) || FilterFieldUtils.FILETERS_USER_NAME.contains(cName))) { // // 遍历输出行
if(FilterFieldUtils.FILETERS_USER_NAME.contains(cName)){ // int rowCount = 0;
row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(rs.getString(j+1))); // while (rs.next()) {
} else { // rowCount++;
row.createCell(j).setCellValue("******"); // row = sheet.createRow(rowCount);
} // for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) {
} else { // //String c = rs.getString(j + 1);
int cType = rs.getMetaData().getColumnType(j + 1); // //row.createCell(j).setCellValue(c);
switch (cType) { // String cName = rs.getMetaData().getColumnName(j+1);
case Types.TIMESTAMP: // if (task.getQueryDataType() == QueryDataType.SAFE
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : ""); // && (FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName) || FilterFieldUtils.FILETERS_USER_NAME.contains(cName))) {
break; // if(FilterFieldUtils.FILETERS_USER_NAME.contains(cName)){
case Types.DATE: // row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(rs.getString(j+1)));
row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : ""); // } else {
break; // row.createCell(j).setCellValue("******");
case Types.TIME: // }
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : ""); // } else {
break; // int cType = rs.getMetaData().getColumnType(j + 1);
default: // switch (cType) {
String string = rs.getString(j + 1); // case Types.TIMESTAMP:
if(StringUtils.isNotBlank(string)){ // row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : "");
if(string.contains("E0")){ // break;
string = string + "\t"; // case Types.DATE:
} // row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : "");
} // break;
row.createCell(j).setCellValue(string); // case Types.TIME:
break; // row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : "");
} // break;
} // default:
} // FOR OVER // String string = rs.getString(j + 1);
} // WHILE OVER // if(StringUtils.isNotBlank(string)){
FileOutputStream fileOut = new FileOutputStream(originalFilePath); // if(string.contains("E0")){
wb.write(fileOut); // string = string + "\t";
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式 // }
fileOut.close(); // }
//wb.close(); // row.createCell(j).setCellValue(string);
wb.dispose(); // SXSSFWorkbook 没有 close 方法 // break;
log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx"); // }
// }
// } // FOR OVER
// } // WHILE OVER
// FileOutputStream fileOut = new FileOutputStream(originalFilePath);
// wb.write(fileOut);
// //fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
// fileOut.close();
// //wb.close();
// wb.dispose(); // SXSSFWorkbook 没有 close 方法
// log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
} }
String cloudFileUrl = "https://"; String cloudFileUrl = "https://";
String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx"; String taskFileExt = task.getUseCompress().equals(Global.YES)||zipFlag ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
if (task.getUseCompress().equals(Global.YES)) { if (task.getUseCompress().equals(Global.YES)||zipFlag) {
log.debug("runDownloadTask.run", "准备生成自定义查询压缩文件 " + condition.getTaskId() + ".zip"); log.debug("runDownloadTask.run", "准备生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
// String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip"; String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
OutputStream os = new FileOutputStream(zipFilePath);
File zipFile = new File(zipFilePath); File zipFile = new File(zipFilePath);
ZipOutputStream zos = null; FileUtil.toZip(listFile,os);
byte[] buf = new byte[1024];
int length = 0;
try { // File zipFile = new File(zipFilePath);
OutputStream os = new FileOutputStream(zipFilePath); // ZipOutputStream zos = null;
BufferedOutputStream bos = new BufferedOutputStream(os); // byte[] buf = new byte[1024];
zos = new ZipOutputStream(bos); // int length = 0;
zos.setLevel(6); // 压缩率选择 0-9 // try {
InputStream is = new FileInputStream(originalFilePath); // OutputStream os = new FileOutputStream(zipFilePath);
BufferedInputStream bis = new BufferedInputStream(is); // BufferedOutputStream bos = new BufferedOutputStream(os);
zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1))); // zos = new ZipOutputStream(bos);
while ((length = bis.read(buf)) > 0) { // zos.setLevel(6); // 压缩率选择 0-9
zos.write(buf, 0, length); // InputStream is = new FileInputStream(originalFilePath);
} // BufferedInputStream bis = new BufferedInputStream(is);
bis.close(); // zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
is.close(); // while ((length = bis.read(buf)) > 0) {
//bos.close(); // zos.write(buf, 0, length);
//os.close(); // }
log.debug("runDownloadTask.run", "已生成自定义查询压缩文件 " + condition.getTaskId() + ".zip"); // bis.close();
} catch (Exception ex2) { // is.close();
throw ex2; // //bos.close();
} finally { // //os.close();
zos.closeEntry(); // log.debug("runDownloadTask.run", "已生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
zos.close(); // } catch (Exception ex2) {
} // throw ex2;
log.debug("开始上传压缩文件到腾讯云", task.getId()); // } finally {
// zos.closeEntry();
// zos.close();
// }
// log.debug("开始上传压缩文件到腾讯云", task.getId());
cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName()); cloudFileUrl += FileUploadUtil.simpleUploadFileFromLocal(zipFile, task.getName() + "-" + task.getId()+taskFileExt, BucketNameEnum.COMPRESS_60000.getName());
} else { } else {
log.debug("开始上传文件到腾讯云", task.getId()); log.debug("开始上传文件到腾讯云", task.getId());
...@@ -559,6 +592,68 @@ public class FreeQueryServiceImpl implements IFreeQueryService { ...@@ -559,6 +592,68 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
}, interval*1000, interval*1000, TimeUnit.MILLISECONDS); }, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
} }
private void saveXlsx(FreeQueryTaskCondition condition,String originalFilePath,ResultSet rs,DownloadTask task) throws Exception{
Integer limitSize = FileUtil.getLimitSize();
// originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0);
Cell cell;
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头
String colName = rs.getMetaData().getColumnLabel(j + 1);
cell = row.createCell(j);
cell.setCellValue(colName);
}
// 遍历输出行
int rowCount = 0;
while (rowCount<limitSize&&rs.next()) {
rowCount++;
row = sheet.createRow(rowCount);
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) {
//String c = rs.getString(j + 1);
//row.createCell(j).setCellValue(c);
String cName = rs.getMetaData().getColumnName(j+1);
if (task.getQueryDataType() == QueryDataType.SAFE
&& (FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName) || FilterFieldUtils.FILETERS_USER_NAME.contains(cName))) {
if(FilterFieldUtils.FILETERS_USER_NAME.contains(cName)){
row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(rs.getString(j+1)));
} else {
row.createCell(j).setCellValue("******");
}
} else {
int cType = rs.getMetaData().getColumnType(j + 1);
switch (cType) {
case Types.TIMESTAMP:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : "");
break;
case Types.DATE:
row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : "");
break;
case Types.TIME:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : "");
break;
default:
String string = rs.getString(j + 1);
if(StringUtils.isNotBlank(string)){
if(string.contains("E0")){
string = string + "\t";
}
}
row.createCell(j).setCellValue(string);
break;
}
}
} // FOR OVER
} // WHILE OVER
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close();
//wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法
log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
}
/** 下载申请检查计时器 */ /** 下载申请检查计时器 */
//private Timer applyTimer = new Timer(); //private Timer applyTimer = new Timer();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment