Skip to content

G
gic-cloud
Commit 7bba938c by 陶光胜
Options

取数平台下载优化

parent c48cb32e
Showing with 310 additions and 300 deletions
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
...@@ -508,174 +508,179 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService { ...@@ -508,174 +508,179 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
this.downloadTaskTimer.schedule(new TimerTask() { this.downloadTaskTimer.schedule(new TimerTask() {
@Override @Override
public void run() { public void run() {
if (taskConditions != null && taskConditions.size() > 0) { try{
FlatQueryTaskCondition condition = null; if (taskConditions != null && taskConditions.size() > 0) {
//FlatQueryTaskCondition condition = taskConditions.remove(0); // 移除并获取第一个任务条件 FlatQueryTaskCondition condition = null;
for (int i=0; i<taskConditions.size(); i++ ) { //FlatQueryTaskCondition condition = taskConditions.remove(0); // 移除并获取第一个任务条件
log.debug("自助指标当前正在执行的任务为：", JSON.toJSONString(taskConditions.get(i))); for (int i=0; i<taskConditions.size(); i++ ) {
if (taskConditions.get(i).getBuildPermitted().equals(Global.YES)) { log.debug("自助指标当前正在执行的任务为：", JSON.toJSONString(taskConditions.get(i)));
condition = taskConditions.remove(i); // 移除并获取第一个任务条件 if (taskConditions.get(i).getBuildPermitted().equals(Global.YES)) {
break; condition = taskConditions.remove(i); // 移除并获取第一个任务条件
} // IF OVER break;
} // FOR OVER } // IF OVER
if (condition != null) { } // FOR OVER
// 更新任务状态 if (condition != null) {
DownloadTask task = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId()); // 更新任务状态
task.setStatus(DownloadTaskStatus.BUILDING); DownloadTask task = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId());
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task); task.setStatus(DownloadTaskStatus.BUILDING);
log.debug("runDownloadTask.run", "自助指标下载任务执行：" + task.getId()); DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
String fullQuery = buildFlatQuerySQL( log.debug("runDownloadTask.run", "自助指标下载任务执行：" + task.getId());
false, // 下载用途 String fullQuery = buildFlatQuerySQL(
condition.getTableId(), false, // 下载用途
condition.getEnterpriseIds(), condition.getTableId(),
condition.getConditions(), condition.getEnterpriseIds(),
condition.getOrderField(), condition.getConditions(),
condition.getOrderDir(), condition.getOrderField(),
condition.getExecDistinct(), condition.getOrderDir(),
0, condition.getExecDistinct(),
condition.getAuthStoreIdList()); 0,
condition.getAuthStoreIdList());
Connection conn = HiveHelper.getDownloadHiveConnection();
if (conn != null) { Connection conn = HiveHelper.getDownloadHiveConnection();
try { if (conn != null) {
Statement stat = conn.createStatement(); try {
Statement stat = conn.createStatement();
// stat.setQueryTimeout(60 * 1000); // stat.setQueryTimeout(60 * 1000);
stat.execute("REFRESH TABLE " + condition.getTableId()); // 强制刷新表结构 stat.execute("REFRESH TABLE " + condition.getTableId()); // 强制刷新表结构
ResultSet rs = stat.executeQuery(fullQuery); ResultSet rs = stat.executeQuery(fullQuery);
// 生成指定格式下载元文件 // 生成指定格式下载元文件
String originalFilePath = ""; String originalFilePath = "";
if (task.getFormat().equals(DownloadFileFormat.CSV)) { // 如果指定为 CSV 格式 if (task.getFormat().equals(DownloadFileFormat.CSV)) { // 如果指定为 CSV 格式
log.debug("runDownloadTask.run", "准备生成自助指标下载文件 " + condition.getTaskId() + ".csv"); log.debug("runDownloadTask.run", "准备生成自助指标下载文件 " + condition.getTaskId() + ".csv");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv"; originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv";
File tmp = new File(originalFilePath); File tmp = new File(originalFilePath);
if (tmp.exists()) { // 删除可能存在的文件 if (tmp.exists()) { // 删除可能存在的文件
tmp.delete(); tmp.delete();
} }
//CSVWriter csvWriter = new CSVWriter(new FileWriter(csvPath), '\t'); //CSVWriter csvWriter = new CSVWriter(new FileWriter(csvPath), '\t');
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(originalFilePath), Charset.forName("GBK")); OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(originalFilePath), Charset.forName("GBK"));
ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.DECRYPT : CsvDataFilterMode.DESENSI, condition.getDecryptFilters()); ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.DECRYPT : CsvDataFilterMode.DESENSI, condition.getDecryptFilters());
CSVWriter writer = new CSVWriter(out, ','); CSVWriter writer = new CSVWriter(out, ',');
writer.setResultService(helper); writer.setResultService(helper);
writer.writeAll(rs, true); writer.writeAll(rs, true);
writer.close(); writer.close();
out.close();//记得关闭资源 out.close();//记得关闭资源
log.debug("runDownloadTask.run", "已生成自助指标下载文件 " + condition.getTaskId() + ".csv"); log.debug("runDownloadTask.run", "已生成自助指标下载文件 " + condition.getTaskId() + ".csv");
} else { // 如果指定为 XLS 格式 } else { // 如果指定为 XLS 格式
log.debug("runDownloadTask.run", "准备生成自助指标下载文件 " + condition.getTaskId() + ".xlsx"); log.debug("runDownloadTask.run", "准备生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx"; originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行 SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
Sheet sheet = wb.createSheet(); Sheet sheet = wb.createSheet();
Row row = sheet.createRow(0); Row row = sheet.createRow(0);
Cell cell; Cell cell;
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头 for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头
String colName = rs.getMetaData().getColumnLabel(j + 1); String colName = rs.getMetaData().getColumnLabel(j + 1);
cell = row.createCell(j); cell = row.createCell(j);
cell.setCellValue(colName); cell.setCellValue(colName);
} }
// 遍历输出行 // 遍历输出行
int rowCount = 0; int rowCount = 0;
while (rs.next()) { while (rs.next()) {
rowCount++; rowCount++;
row = sheet.createRow(rowCount); row = sheet.createRow(rowCount);
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) {
//String c = rs.getString(j + 1); //String c = rs.getString(j + 1);
//row.createCell(j).setCellValue(c); //row.createCell(j).setCellValue(c);
String cName = rs.getMetaData().getColumnName(j+1); String cName = rs.getMetaData().getColumnName(j+1);
List<String> cFilters = condition.getDecryptFilters(); List<String> cFilters = condition.getDecryptFilters();
if (task.getQueryDataType() == QueryDataType.FULL && cFilters.contains(cName)) { if (task.getQueryDataType() == QueryDataType.FULL && cFilters.contains(cName)) {
String tmpResult = rs.getString(j + 1); String tmpResult = rs.getString(j + 1);
if (StringUtils.isNotBlank(tmpResult)) tmpResult = DecryptUtils.getInstance().decrypt(tmpResult); if (StringUtils.isNotBlank(tmpResult)) tmpResult = DecryptUtils.getInstance().decrypt(tmpResult);
row.createCell(j).setCellValue(tmpResult); row.createCell(j).setCellValue(tmpResult);
} else { } else {
int cType = rs.getMetaData().getColumnType(j + 1); int cType = rs.getMetaData().getColumnType(j + 1);
switch (cType) { switch (cType) {
case Types.TIMESTAMP: case Types.TIMESTAMP:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : ""); row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : "");
break; break;
case Types.DATE: case Types.DATE:
row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : ""); row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : "");
break; break;
case Types.TIME: case Types.TIME:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : ""); row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : "");
break; break;
default: default:
if(cFilters.contains(cName)){ if(cFilters.contains(cName)){
row.createCell(j).setCellValue("******"); row.createCell(j).setCellValue("******");
}else { }else {
row.createCell(j).setCellValue(rs.getString(j + 1)); row.createCell(j).setCellValue(rs.getString(j + 1));
} }
break; break;
} }
} // IF ELSE OVER } // IF ELSE OVER
} // FOR OVER } // FOR OVER
} // WHILE OVER } // WHILE OVER
FileOutputStream fileOut = new FileOutputStream(originalFilePath); FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut); wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式 //fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut.close(); fileOut.close();
//wb.close(); //wb.close();
wb.dispose(); // SXSSFWorkbook 没有 close 方法 wb.dispose(); // SXSSFWorkbook 没有 close 方法
log.debug("runDownloadTask.run", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx"); log.debug("runDownloadTask.run", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
} // IF ELSE OVER } // IF ELSE OVER
// 如果指定压缩，则使用之 // 如果指定压缩，则使用之
//if (task.getFormat().equals("zip")) { //if (task.getFormat().equals("zip")) {
if (task.getUseCompress().equals(Global.YES)) { if (task.getUseCompress().equals(Global.YES)) {
log.debug("runDownloadTask.run", "准备生成自助指标压缩文件 " + condition.getTaskId() + ".zip"); log.debug("runDownloadTask.run", "准备生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip"; String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
File zipFile = new File(zipFilePath); File zipFile = new File(zipFilePath);
ZipOutputStream zos = null; ZipOutputStream zos = null;
byte[] buf = new byte[1024]; byte[] buf = new byte[1024];
int length = 0; int length = 0;
try { try {
OutputStream os = new FileOutputStream(zipFilePath); OutputStream os = new FileOutputStream(zipFilePath);
BufferedOutputStream bos = new BufferedOutputStream(os); BufferedOutputStream bos = new BufferedOutputStream(os);
zos = new ZipOutputStream(bos); zos = new ZipOutputStream(bos);
zos.setLevel(6); // 压缩率选择 0-9 zos.setLevel(6); // 压缩率选择 0-9
InputStream is = new FileInputStream(originalFilePath); InputStream is = new FileInputStream(originalFilePath);
BufferedInputStream bis = new BufferedInputStream(is); BufferedInputStream bis = new BufferedInputStream(is);
zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1))); zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
while ((length = bis.read(buf)) > 0) { while ((length = bis.read(buf)) > 0) {
zos.write(buf, 0, length); zos.write(buf, 0, length);
}
bis.close();
is.close();
//bos.close();
//os.close();
log.debug("runDownloadTask.run", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
} catch (Exception ex2) {
throw ex2;
} finally {
zos.closeEntry();
zos.close();
} }
bis.close();
is.close();
//bos.close();
//os.close();
log.debug("runDownloadTask.run", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
} catch (Exception ex2) {
throw ex2;
} finally {
zos.closeEntry();
zos.close();
} }
}
task.setStatus(DownloadTaskStatus.COMPLISHED); task.setStatus(DownloadTaskStatus.COMPLISHED);
task.setOverTime(new Date()); task.setOverTime(new Date());
String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx"; String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
task.setFilePath(task.getId() + taskFileExt); task.setFilePath(task.getId() + taskFileExt);
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task); DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
} catch (Exception ex) { } catch (Exception ex) {
ex.printStackTrace(); ex.printStackTrace();
// 标记任务异常 // 标记任务异常
task.setStatus(DownloadTaskStatus.ERROR); task.setStatus(DownloadTaskStatus.ERROR);
task.setOverTime(new Date()); task.setOverTime(new Date());
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task); DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
} finally { } finally {
try { try {
conn.close(); conn.close();
} catch (SQLException e) { } catch (SQLException e) {
e.printStackTrace(); e.printStackTrace();
}
} }
}
} // IF OVER
} // IF OVER } // IF OVER
} // IF OVER } // 没有任务则忽略
} // 没有任务则忽略 }catch (Exception e){
log.debug("自助指标下载异常", e.getMessage());
e.printStackTrace();
}
} // run Define Over } // run Define Over
}, interval * 1000, interval * 1000);// 配置中的值为毫秒 }, interval * 1000, interval * 1000);// 配置中的值为毫秒
} }
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
...@@ -358,158 +358,163 @@ public class FreeQueryServiceImpl implements IFreeQueryService { ...@@ -358,158 +358,163 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
this.downloadTaskTimer.schedule(new TimerTask() { this.downloadTaskTimer.schedule(new TimerTask() {
@Override @Override
public void run() { public void run() {
if (taskConditions != null && taskConditions.size() > 0) { try {
//FreeQueryTaskCondition condition = taskConditions.remove(0); // 移除并获取第一个任务条件 if (taskConditions != null && taskConditions.size() > 0) {
FreeQueryTaskCondition condition = null; //FreeQueryTaskCondition condition = taskConditions.remove(0); // 移除并获取第一个任务条件
for (int i = 0; i < taskConditions.size(); i++) { FreeQueryTaskCondition condition = null;
log.debug("自定义查询下载当前任务：", JSON.toJSONString(taskConditions.get(i))); for (int i = 0; i < taskConditions.size(); i++) {
if (taskConditions.get(i).getBuildPermitted().equals(Global.YES)) { log.debug("自定义查询下载当前任务：", JSON.toJSONString(taskConditions.get(i)));
condition = taskConditions.remove(i); // 移除并获取第一个任务条件 if (taskConditions.get(i).getBuildPermitted().equals(Global.YES)) {
break; condition = taskConditions.remove(i); // 移除并获取第一个任务条件
} // IF OVER break;
} // FOR OVER } // IF OVER
if (condition != null) { } // FOR OVER
// 更新任务状态 if (condition != null) {
DownloadTask task = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId()); // 更新任务状态
task.setStatus(DownloadTaskStatus.BUILDING); DownloadTask task = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId());
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task); task.setStatus(DownloadTaskStatus.BUILDING);
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
String fullQuery = condition.getSql();
Connection conn = MysqlHelper.getFreeQueryConnection(condition.getEnterpriseId());
if (conn != null) {
try {
// 设定为流式读取
Statement stat = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stat.setFetchSize(Integer.MIN_VALUE);
ResultSet rs = stat.executeQuery(fullQuery);
// 生成指定格式下载元文件
String originalFilePath = "";
if (task.getFormat().equals(DownloadFileFormat.CSV)) { // 如果指定为 CSV 格式
log.debug("runDownloadTask.run", "准备生成自定义查询下载文件 " + condition.getTaskId() + ".csv");
//String csvPath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv";
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv";
File tmp = new File(originalFilePath);
if (tmp.exists()) { // 删除可能存在的文件
tmp.delete();
}
//CSVWriter csvWriter = new CSVWriter(new FileWriter(csvPath), '\t'); String fullQuery = condition.getSql();
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(originalFilePath), Charset.forName("GBK"));
ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.NONE : CsvDataFilterMode.DESENSI, FreeQueryServiceImpl.getFieldsFilters(condition.getDesensiType())); Connection conn = MysqlHelper.getFreeQueryConnection(condition.getEnterpriseId());
CSVWriter writer = new CSVWriter(out, ','); if (conn != null) {
writer.setResultService(helper); try {
writer.writeAll(rs, true); // 设定为流式读取
writer.close(); Statement stat = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
out.close(); stat.setFetchSize(Integer.MIN_VALUE);
log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".csv"); ResultSet rs = stat.executeQuery(fullQuery);
} else {
log.debug("runDownloadTask.run", "准备生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx"); // 生成指定格式下载元文件
originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx"; String originalFilePath = "";
SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行 if (task.getFormat().equals(DownloadFileFormat.CSV)) { // 如果指定为 CSV 格式
Sheet sheet = wb.createSheet(); log.debug("runDownloadTask.run", "准备生成自定义查询下载文件 " + condition.getTaskId() + ".csv");
Row row = sheet.createRow(0); //String csvPath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv";
Cell cell; originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".csv";
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头 File tmp = new File(originalFilePath);
String colName = rs.getMetaData().getColumnLabel(j + 1); if (tmp.exists()) { // 删除可能存在的文件
cell = row.createCell(j); tmp.delete();
cell.setCellValue(colName); }
}
// 遍历输出行 //CSVWriter csvWriter = new CSVWriter(new FileWriter(csvPath), '\t');
int rowCount = 0; OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(originalFilePath), Charset.forName("GBK"));
while (rs.next()) { ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.NONE : CsvDataFilterMode.DESENSI, FreeQueryServiceImpl.getFieldsFilters(condition.getDesensiType()));
rowCount++; CSVWriter writer = new CSVWriter(out, ',');
row = sheet.createRow(rowCount); writer.setResultService(helper);
for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { writer.writeAll(rs, true);
//String c = rs.getString(j + 1); writer.close();
//row.createCell(j).setCellValue(c); out.close();
String cName = rs.getMetaData().getColumnName(j+1); log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".csv");
if (task.getQueryDataType() == QueryDataType.SAFE } else {
&& FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName)) { log.debug("runDownloadTask.run", "准备生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
row.createCell(j).setCellValue("******"); originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
} else { SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
int cType = rs.getMetaData().getColumnType(j + 1); Sheet sheet = wb.createSheet();
switch (cType) { Row row = sheet.createRow(0);
case Types.TIMESTAMP: Cell cell;
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : ""); for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头
break; String colName = rs.getMetaData().getColumnLabel(j + 1);
case Types.DATE: cell = row.createCell(j);
row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : ""); cell.setCellValue(colName);
break; }
case Types.TIME: // 遍历输出行
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : ""); int rowCount = 0;
break; while (rs.next()) {
default: rowCount++;
row.createCell(j).setCellValue(rs.getString(j + 1)); row = sheet.createRow(rowCount);
break; for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) {
//String c = rs.getString(j + 1);
//row.createCell(j).setCellValue(c);
String cName = rs.getMetaData().getColumnName(j+1);
if (task.getQueryDataType() == QueryDataType.SAFE
&& FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName)) {
row.createCell(j).setCellValue("******");
} else {
int cType = rs.getMetaData().getColumnType(j + 1);
switch (cType) {
case Types.TIMESTAMP:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : "");
break;
case Types.DATE:
row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : "");
break;
case Types.TIME:
row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : "");
break;
default:
row.createCell(j).setCellValue(rs.getString(j + 1));
break;
}
} }
} } // FOR OVER
} // FOR OVER } // WHILE OVER
} // WHILE OVER FileOutputStream fileOut = new FileOutputStream(originalFilePath);
FileOutputStream fileOut = new FileOutputStream(originalFilePath); wb.write(fileOut);
wb.write(fileOut); //fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式 fileOut.close();
fileOut.close(); //wb.close();
//wb.close(); wb.dispose(); // SXSSFWorkbook 没有 close 方法
wb.dispose(); // SXSSFWorkbook 没有 close 方法 log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx"); }
}
if (task.getUseCompress().equals(Global.YES)) { if (task.getUseCompress().equals(Global.YES)) {
log.debug("runDownloadTask.run", "准备生成自定义查询压缩文件 " + condition.getTaskId() + ".zip"); log.debug("runDownloadTask.run", "准备生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip"; String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
File zipFile = new File(zipFilePath); File zipFile = new File(zipFilePath);
ZipOutputStream zos = null; ZipOutputStream zos = null;
byte[] buf = new byte[1024]; byte[] buf = new byte[1024];
int length = 0; int length = 0;
try { try {
OutputStream os = new FileOutputStream(zipFilePath); OutputStream os = new FileOutputStream(zipFilePath);
BufferedOutputStream bos = new BufferedOutputStream(os); BufferedOutputStream bos = new BufferedOutputStream(os);
zos = new ZipOutputStream(bos); zos = new ZipOutputStream(bos);
zos.setLevel(6); // 压缩率选择 0-9 zos.setLevel(6); // 压缩率选择 0-9
InputStream is = new FileInputStream(originalFilePath); InputStream is = new FileInputStream(originalFilePath);
BufferedInputStream bis = new BufferedInputStream(is); BufferedInputStream bis = new BufferedInputStream(is);
zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1))); zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
while ((length = bis.read(buf)) > 0) { while ((length = bis.read(buf)) > 0) {
zos.write(buf, 0, length); zos.write(buf, 0, length);
}
bis.close();
is.close();
//bos.close();
//os.close();
log.debug("runDownloadTask.run", "已生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
} catch (Exception ex2) {
throw ex2;
} finally {
zos.closeEntry();
zos.close();
} }
bis.close();
is.close();
//bos.close();
//os.close();
log.debug("runDownloadTask.run", "已生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
} catch (Exception ex2) {
throw ex2;
} finally {
zos.closeEntry();
zos.close();
} }
}
task.setStatus(DownloadTaskStatus.COMPLISHED); task.setStatus(DownloadTaskStatus.COMPLISHED);
task.setOverTime(new java.util.Date()); task.setOverTime(new java.util.Date());
String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx"; String taskFileExt = task.getUseCompress().equals(Global.YES) ? ".zip" : task.getFormat().equals(DownloadFileFormat.CSV) ? ".csv" : ".xlsx";
task.setFilePath(task.getId() + taskFileExt); task.setFilePath(task.getId() + taskFileExt);
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task); DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
} catch (Exception ex) { } catch (Exception ex) {
ex.printStackTrace(); ex.printStackTrace();
// 标记任务异常 // 标记任务异常
task.setStatus(DownloadTaskStatus.ERROR); task.setStatus(DownloadTaskStatus.ERROR);
task.setOverTime(new Date()); task.setOverTime(new Date());
DownloadTaskServiceImpl.getInstance().updateDownloadTask(task); DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
} finally { } finally {
try { try {
conn.close(); conn.close();
} catch (SQLException e) { } catch (SQLException e) {
e.printStackTrace(); e.printStackTrace();
}
} }
} } // IF OVER
} // IF OVER } // IF OVER
} // IF OVER } // 没有任务则忽略
} // 没有任务则忽略 }catch (Exception e){
log.debug("自定义查询下载异常", e.getMessage());
e.printStackTrace();
}
} // run Define Over } // run Define Over
}, interval * 1000, interval * 1000);// 配置中的值为毫秒 }, interval * 1000, interval * 1000);// 配置中的值为毫秒
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment