Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
G
gic-cloud
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
data-hook
gic-cloud
Commits
b6f06a15
Commit
b6f06a15
authored
Jun 15, 2022
by
fudahua
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'developer' into 'master'
Developer See merge request
!50
parents
27cb83ac
7a5a41ce
Show whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
357 additions
and
121 deletions
+357
-121
FreeQueryTaskCondition.java
...ic/cloud/data/hook/api/entity/FreeQueryTaskCondition.java
+10
-0
FileUtil.java
...c/main/java/com/gic/cloud/data/hook/service/FileUtil.java
+62
-0
FlatQueryResultServiceImpl.java
...ud/data/hook/service/impl/FlatQueryResultServiceImpl.java
+122
-53
FreeQueryServiceImpl.java
...ic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
+163
-68
No files found.
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/entity/FreeQueryTaskCondition.java
View file @
b6f06a15
...
...
@@ -114,4 +114,14 @@ public class FreeQueryTaskCondition implements Serializable {
public
void
setDesensiType
(
Integer
desensiType
)
{
this
.
desensiType
=
desensiType
;
}
private
int
amount
;
public
int
getAmount
()
{
return
amount
;
}
public
void
setAmount
(
int
amount
)
{
this
.
amount
=
amount
;
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/FileUtil.java
0 → 100644
View file @
b6f06a15
package
com
.
gic
.
cloud
.
data
.
hook
.
service
;
import
com.ctrip.framework.apollo.Config
;
import
com.ctrip.framework.apollo.ConfigService
;
import
java.io.File
;
import
java.io.FileInputStream
;
import
java.io.IOException
;
import
java.io.OutputStream
;
import
java.util.List
;
import
java.util.zip.ZipEntry
;
import
java.util.zip.ZipOutputStream
;
public
class
FileUtil
{
private
static
final
int
BUFFER_SIZE
=
2
*
1024
;
/**
* 压缩zip
* @param srcFiles
* @param out
* @throws RuntimeException
*/
public
static
void
toZip
(
List
<
File
>
srcFiles
,
OutputStream
out
)
throws
RuntimeException
{
long
start
=
System
.
currentTimeMillis
();
ZipOutputStream
zos
=
null
;
try
{
zos
=
new
ZipOutputStream
(
out
);
for
(
File
srcFile
:
srcFiles
)
{
byte
[]
buf
=
new
byte
[
BUFFER_SIZE
];
zos
.
putNextEntry
(
new
ZipEntry
(
srcFile
.
getName
()));
int
len
;
FileInputStream
in
=
new
FileInputStream
(
srcFile
);
while
((
len
=
in
.
read
(
buf
))
!=
-
1
)
{
zos
.
write
(
buf
,
0
,
len
);
}
zos
.
closeEntry
();
in
.
close
();
}
long
end
=
System
.
currentTimeMillis
();
System
.
out
.
println
(
"压缩完成,耗时:"
+
(
end
-
start
)
+
" ms"
);
}
catch
(
Exception
e
)
{
throw
new
RuntimeException
(
"zip error from ZipUtils"
,
e
);
}
finally
{
if
(
zos
!=
null
)
{
try
{
zos
.
close
();
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
}
}
/**
* 获取限制调试
* @return
*/
public
static
Integer
getLimitSize
(){
Config
appConfig
=
ConfigService
.
getAppConfig
();
return
appConfig
.
getIntProperty
(
"xls.size.limit"
,
1000000
);
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
View file @
b6f06a15
...
...
@@ -76,6 +76,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private
static
final
Integer
SMALL_SIZE
=
2
;
private
static
final
Integer
XLS_SIZE
=
1000000
;
private
static
final
Map
<
String
,
String
>
bigTaskRunningMap
=
new
ConcurrentHashMap
<>();
@Autowired
...
...
@@ -809,7 +811,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
if
(
conn
!=
null
)
{
try
{
Statement
stat
=
conn
.
createStatement
();
Integer
limitSize
=
FileUtil
.
getLimitSize
();
// stat.setQueryTimeout(60 * 1000);
stat
.
execute
(
"REFRESH TABLE "
+
condition
.
getTableId
());
// 强制刷新表结构
ResultSet
rs
=
stat
.
executeQuery
(
fullQuery
);
...
...
@@ -817,6 +819,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
ResultSetHelper
helper
=
new
CsvResultSetHelper
(
queryDataType
,
condition
.
getDecryptFilters
(),
condition
.
getConditions
(),
condition
.
getEnterpriseIds
().
get
(
0
),
decryptKeyService
);
// 生成指定格式下载元文件
String
originalFilePath
=
""
;
boolean
zipFlag
=
false
;
List
<
File
>
listFile
=
new
ArrayList
<>();
if
(
task
.
getFormat
().
equals
(
DownloadFileFormat
.
CSV
))
{
// 如果指定为 CSV 格式
logger
.
info
(
"[ runDownloadTask.run ]: {}"
,
"准备生成自助指标下载文件 "
+
condition
.
getTaskId
()
+
".csv"
);
originalFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()
+
".csv"
;
...
...
@@ -832,69 +836,97 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
writer
.
close
();
out
.
close
();
//记得关闭资源
logger
.
info
(
"[ runDownloadTask.run ]: {}"
,
"已生成自助指标下载文件 "
+
condition
.
getTaskId
()
+
".csv"
);
listFile
.
add
(
new
File
(
originalFilePath
));
}
else
{
// 如果指定为 XLS 格式
logger
.
info
(
"[ runDownloadTask.run ]: {}"
,
"准备生成自助指标下载文件 "
+
condition
.
getTaskId
()
+
".xlsx"
);
int
filePos
=
0
;
if
(
condition
.
getAmount
()>
limitSize
){
int
num
=
(
condition
.
getAmount
()
/
limitSize
)+(
condition
.
getAmount
()%
limitSize
>
0
?
1
:
0
);
while
(
num
-->
0
)
{
originalFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()+
"-"
+
filePos
+
".xlsx"
;
filePos
++;
saveXlsSplit
(
originalFilePath
,
helper
,
rs
,
condition
);
File
file
=
new
File
(
originalFilePath
);
listFile
.
add
(
file
);
}
zipFlag
=
true
;
}
else
{
originalFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()
+
".xlsx"
;
SXSSFWorkbook
wb
=
new
SXSSFWorkbook
(
100
);
// 内存中保留 100 行
Sheet
sheet
=
wb
.
createSheet
();
Row
row
=
sheet
.
createRow
(
0
);
Cell
cell
;
String
[]
columnNames
=
helper
.
getColumnNames
(
rs
);
for
(
int
j
=
0
;
j
<
columnNames
.
length
;
j
++){
cell
=
row
.
createCell
(
j
);
cell
.
setCellValue
(
columnNames
[
j
]);
}
// 遍历输出行
int
rowCount
=
0
;
while
(
rs
.
next
())
{
rowCount
++;
row
=
sheet
.
createRow
(
rowCount
);
String
[]
columnValues
=
helper
.
getColumnValues
(
rs
,
true
,
""
,
""
);
for
(
int
j
=
0
;
j
<
columnValues
.
length
;
j
++){
row
.
createCell
(
j
).
setCellValue
(
columnValues
[
j
]);
saveXlsSplit
(
originalFilePath
,
helper
,
rs
,
condition
);
File
file
=
new
File
(
originalFilePath
);
listFile
.
add
(
file
);
}
}
// WHILE OVER
FileOutputStream
fileOut
=
new
FileOutputStream
(
originalFilePath
);
wb
.
write
(
fileOut
);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut
.
close
();
//wb.close();
wb
.
dispose
();
// SXSSFWorkbook 没有 close 方法
logger
.
info
(
"[ runDownloadTask.run ]: {}"
,
"已生成自助指标下载文件 "
+
condition
.
getTaskId
()
+
".xlsx"
);
String
zipFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()
+
".zip"
;
OutputStream
os
=
new
FileOutputStream
(
zipFilePath
);
FileUtil
.
toZip
(
listFile
,
os
);
// originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
// SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
// Sheet sheet = wb.createSheet();
// Row row = sheet.createRow(0);
// Cell cell;
// String[] columnNames = helper.getColumnNames(rs);
// for(int j =0; j<columnNames.length; j++){
// cell = row.createCell(j);
// cell.setCellValue(columnNames[j]);
// }
// // 遍历输出行
// int rowCount = 0;
// while (rs.next()) {
// rowCount++;
// row = sheet.createRow(rowCount);
// String[] columnValues = helper.getColumnValues(rs, true, "", "");
// for(int j=0; j<columnValues.length; j++){
// row.createCell(j).setCellValue(columnValues[j]);
// }
// } // WHILE OVER
// FileOutputStream fileOut = new FileOutputStream(originalFilePath);
// wb.write(fileOut);
// //fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
// fileOut.close();
// //wb.close();
// wb.dispose(); // SXSSFWorkbook 没有 close 方法
// logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标下载文件 " + condition.getTaskId() + ".xlsx");
}
// IF ELSE OVER
String
cloudFileUrl
=
"https://"
;
// 如果指定压缩,则使用之
//if (task.getFormat().equals("zip")) {
String
taskFileExt
=
task
.
getUseCompress
().
equals
(
Global
.
YES
)
?
".zip"
:
task
.
getFormat
().
equals
(
DownloadFileFormat
.
CSV
)
?
".csv"
:
".xlsx"
;
if
(
task
.
getUseCompress
().
equals
(
Global
.
YES
))
{
String
taskFileExt
=
task
.
getUseCompress
().
equals
(
Global
.
YES
)
||
zipFlag
?
".zip"
:
task
.
getFormat
().
equals
(
DownloadFileFormat
.
CSV
)
?
".csv"
:
".xlsx"
;
if
(
zipFlag
||
task
.
getUseCompress
().
equals
(
Global
.
YES
))
{
logger
.
info
(
"[ runDownloadTask.run ]: {}"
,
"准备生成自助指标压缩文件 "
+
condition
.
getTaskId
()
+
".zip"
);
String
zipFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()
+
".zip"
;
File
zipFile
=
new
File
(
zipFilePath
);
ZipOutputStream
zos
=
null
;
byte
[]
buf
=
new
byte
[
1024
];
int
length
=
0
;
try
{
OutputStream
os
=
new
FileOutputStream
(
zipFilePath
);
BufferedOutputStream
bos
=
new
BufferedOutputStream
(
os
);
zos
=
new
ZipOutputStream
(
bos
);
zos
.
setLevel
(
6
);
// 压缩率选择 0-9
InputStream
is
=
new
FileInputStream
(
originalFilePath
);
BufferedInputStream
bis
=
new
BufferedInputStream
(
is
);
zos
.
putNextEntry
(
new
ZipEntry
(
originalFilePath
.
substring
(
originalFilePath
.
lastIndexOf
(
"/"
)
+
1
)));
while
((
length
=
bis
.
read
(
buf
))
>
0
)
{
zos
.
write
(
buf
,
0
,
length
);
}
bis
.
close
();
is
.
close
();
//bos.close();
//os.close();
logger
.
info
(
"[ runDownloadTask.run ]: {}"
,
"已生成自助指标压缩文件 "
+
condition
.
getTaskId
()
+
".zip"
);
}
catch
(
Exception
ex2
)
{
throw
ex2
;
}
finally
{
zos
.
closeEntry
();
zos
.
close
();
}
File
zipFile
=
new
File
(
zipFilePath
);
FileUtil
.
toZip
(
listFile
,
os
);
//
//
// ZipOutputStream zos = null;
// byte[] buf = new byte[1024];
// int length = 0;
// try {
// OutputStream os = new FileOutputStream(zipFilePath);
// BufferedOutputStream bos = new BufferedOutputStream(os);
// zos = new ZipOutputStream(bos);
// zos.setLevel(6); // 压缩率选择 0-9
// InputStream is = new FileInputStream(originalFilePath);
// BufferedInputStream bis = new BufferedInputStream(is);
// zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
// while ((length = bis.read(buf)) > 0) {
// zos.write(buf, 0, length);
// }
// bis.close();
// is.close();
// //bos.close();
// //os.close();
// logger.info("[ runDownloadTask.run ]: {}", "已生成自助指标压缩文件 " + condition.getTaskId() + ".zip");
// } catch (Exception ex2) {
// throw ex2;
// } finally {
// zos.closeEntry();
// zos.close();
// }
logger
.
info
(
"[ 开始上传压缩文件到腾讯云 ]: {}"
,
task
.
getId
());
cloudFileUrl
+=
FileUploadUtil
.
simpleUploadFileFromLocal
(
zipFile
,
task
.
getName
()
+
"-"
+
task
.
getId
()+
taskFileExt
,
BucketNameEnum
.
COMPRESS_60000
.
getName
());
}
else
{
...
...
@@ -929,6 +961,43 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}
// IF OVER
}
private
void
saveXlsSplit
(
String
originalFilePath
,
ResultSetHelper
helper
,
ResultSet
rs
,
FlatQueryTaskCondition
condition
){
try
{
Integer
limitSize
=
FileUtil
.
getLimitSize
();
// String originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook
wb
=
new
SXSSFWorkbook
(
100
);
// 内存中保留 100 行
Sheet
sheet
=
wb
.
createSheet
();
Row
row
=
sheet
.
createRow
(
0
);
Cell
cell
;
String
[]
columnNames
=
helper
.
getColumnNames
(
rs
);
for
(
int
j
=
0
;
j
<
columnNames
.
length
;
j
++){
cell
=
row
.
createCell
(
j
);
cell
.
setCellValue
(
columnNames
[
j
]);
}
// 遍历输出行
int
rowCount
=
0
;
while
(
rowCount
<
limitSize
&&
rs
.
next
())
{
rowCount
++;
row
=
sheet
.
createRow
(
rowCount
);
String
[]
columnValues
=
helper
.
getColumnValues
(
rs
,
true
,
""
,
""
);
for
(
int
j
=
0
;
j
<
columnValues
.
length
;
j
++){
row
.
createCell
(
j
).
setCellValue
(
columnValues
[
j
]);
}
}
// WHILE OVER
FileOutputStream
fileOut
=
new
FileOutputStream
(
originalFilePath
);
wb
.
write
(
fileOut
);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
fileOut
.
close
();
//wb.close();
wb
.
dispose
();
// SXSSFWorkbook 没有 close 方法
logger
.
info
(
"[ runDownloadTask.run ]: {}"
,
"已生成自助指标下载文件 "
+
condition
.
getTaskId
()
+
".xlsx"
);
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
logger
.
info
(
"异常:{}"
,
e
);
}
}
/** 下载申请检查计时器 */
//private Timer applyTimer = new Timer();
...
...
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
View file @
b6f06a15
...
...
@@ -51,6 +51,8 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
/** 日志类 */
private
static
LogPak
log
=
new
LogPak
(
FreeQueryServiceImpl
.
class
);
private
static
final
Integer
XLS_SIZE
=
10000
;
/** csv / xls 下载目录 */
public
static
final
String
SAVE_FOLDER
=
"/usr/local/data-hook-file"
;
...
...
@@ -308,6 +310,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
// 生成条件
FreeQueryTaskCondition
condition
=
new
FreeQueryTaskCondition
();
condition
.
setAmount
(
amount
);
condition
.
setTaskId
(
curTask
.
getId
());
condition
.
setSql
(
sql
);
condition
.
setEnterpriseId
(
enterpriseId
);
...
...
@@ -334,6 +337,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
if
(
record
.
getDesensiType
()
!=
null
){
FreeQueryTaskCondition
condition
=
new
FreeQueryTaskCondition
();
condition
.
setTaskId
(
record
.
getId
());
condition
.
setAmount
(
record
.
getAmount
());
condition
.
setSql
(
record
.
getRealSql
());
condition
.
setEnterpriseId
(
record
.
getEnterpriseId
());
condition
.
setDesensiType
(
record
.
getDesensiType
());
...
...
@@ -394,12 +398,16 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
}
// IF OVER
}
// FOR OVER
if
(
condition
!=
null
)
{
Integer
limitSize
=
FileUtil
.
getLimitSize
();
// 更新任务状态
DownloadTask
task
=
DownloadTaskServiceImpl
.
getInstance
().
getDownloadTaskById
(
condition
.
getTaskId
());
task
.
setStatus
(
DownloadTaskStatus
.
BUILDING
);
DownloadTaskServiceImpl
.
getInstance
().
updateDownloadTask
(
task
);
String
fullQuery
=
condition
.
getSql
();
boolean
zipFlag
=
false
;
List
<
File
>
listFile
=
new
ArrayList
<>();
Connection
conn
=
MysqlHelper
.
getFreeQueryConnection
(
condition
.
getEnterpriseId
());
if
(
conn
!=
null
)
{
...
...
@@ -432,7 +440,161 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
log
.
debug
(
"runDownloadTask.run"
,
"已生成自定义查询下载文件 "
+
condition
.
getTaskId
()
+
".csv"
);
}
else
{
log
.
debug
(
"runDownloadTask.run"
,
"准备生成自定义查询下载文件 "
+
condition
.
getTaskId
()
+
".xlsx"
);
int
filePos
=
0
;
if
(
condition
.
getAmount
()>
limitSize
){
int
num
=
(
condition
.
getAmount
()
/
limitSize
)+(
condition
.
getAmount
()%
limitSize
>
0
?
1
:
0
);
while
(
num
-->
0
)
{
originalFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()+
"-"
+
filePos
+
".xlsx"
;
filePos
++;
saveXlsx
(
condition
,
originalFilePath
,
rs
,
task
);
File
file
=
new
File
(
originalFilePath
);
listFile
.
add
(
file
);
}
zipFlag
=
true
;
}
else
{
originalFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()
+
".xlsx"
;
saveXlsx
(
condition
,
originalFilePath
,
rs
,
task
);
File
file
=
new
File
(
originalFilePath
);
listFile
.
add
(
file
);
}
// SXSSFWorkbook wb = new SXSSFWorkbook(100); // 内存中保留 100 行
// Sheet sheet = wb.createSheet();
// Row row = sheet.createRow(0);
// Cell cell;
// for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) { // 遍历创建表头
// String colName = rs.getMetaData().getColumnLabel(j + 1);
// cell = row.createCell(j);
// cell.setCellValue(colName);
// }
// // 遍历输出行
// int rowCount = 0;
// while (rs.next()) {
// rowCount++;
// row = sheet.createRow(rowCount);
// for (int j = 0; j < rs.getMetaData().getColumnCount(); ++j) {
// //String c = rs.getString(j + 1);
// //row.createCell(j).setCellValue(c);
// String cName = rs.getMetaData().getColumnName(j+1);
// if (task.getQueryDataType() == QueryDataType.SAFE
// && (FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName) || FilterFieldUtils.FILETERS_USER_NAME.contains(cName))) {
// if(FilterFieldUtils.FILETERS_USER_NAME.contains(cName)){
// row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(rs.getString(j+1)));
// } else {
// row.createCell(j).setCellValue("******");
// }
// } else {
// int cType = rs.getMetaData().getColumnType(j + 1);
// switch (cType) {
// case Types.TIMESTAMP:
// row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? datetimeFormatter.format(rs.getTimestamp(j + 1)) : "");
// break;
// case Types.DATE:
// row.createCell(j).setCellValue(rs.getDate(j + 1) != null ? dateFormatter.format(rs.getDate(j + 1)) : "");
// break;
// case Types.TIME:
// row.createCell(j).setCellValue(rs.getTimestamp(j + 1) != null ? timeFormatter.format(rs.getTimestamp(j + 1)) : "");
// break;
// default:
// String string = rs.getString(j + 1);
// if(StringUtils.isNotBlank(string)){
// if(string.contains("E0")){
// string = string + "\t";
// }
// }
// row.createCell(j).setCellValue(string);
// break;
// }
// }
// } // FOR OVER
// } // WHILE OVER
// FileOutputStream fileOut = new FileOutputStream(originalFilePath);
// wb.write(fileOut);
// //fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
// fileOut.close();
// //wb.close();
// wb.dispose(); // SXSSFWorkbook 没有 close 方法
// log.debug("runDownloadTask.run", "已生成自定义查询下载文件 " + condition.getTaskId() + ".xlsx");
}
String
cloudFileUrl
=
"https://"
;
String
taskFileExt
=
task
.
getUseCompress
().
equals
(
Global
.
YES
)||
zipFlag
?
".zip"
:
task
.
getFormat
().
equals
(
DownloadFileFormat
.
CSV
)
?
".csv"
:
".xlsx"
;
if
(
task
.
getUseCompress
().
equals
(
Global
.
YES
)||
zipFlag
)
{
log
.
debug
(
"runDownloadTask.run"
,
"准备生成自定义查询压缩文件 "
+
condition
.
getTaskId
()
+
".zip"
);
// String zipFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".zip";
String
zipFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()
+
".zip"
;
OutputStream
os
=
new
FileOutputStream
(
zipFilePath
);
File
zipFile
=
new
File
(
zipFilePath
);
FileUtil
.
toZip
(
listFile
,
os
);
// File zipFile = new File(zipFilePath);
// ZipOutputStream zos = null;
// byte[] buf = new byte[1024];
// int length = 0;
// try {
// OutputStream os = new FileOutputStream(zipFilePath);
// BufferedOutputStream bos = new BufferedOutputStream(os);
// zos = new ZipOutputStream(bos);
// zos.setLevel(6); // 压缩率选择 0-9
// InputStream is = new FileInputStream(originalFilePath);
// BufferedInputStream bis = new BufferedInputStream(is);
// zos.putNextEntry(new ZipEntry(originalFilePath.substring(originalFilePath.lastIndexOf("/") + 1)));
// while ((length = bis.read(buf)) > 0) {
// zos.write(buf, 0, length);
// }
// bis.close();
// is.close();
// //bos.close();
// //os.close();
// log.debug("runDownloadTask.run", "已生成自定义查询压缩文件 " + condition.getTaskId() + ".zip");
// } catch (Exception ex2) {
// throw ex2;
// } finally {
// zos.closeEntry();
// zos.close();
// }
// log.debug("开始上传压缩文件到腾讯云", task.getId());
cloudFileUrl
+=
FileUploadUtil
.
simpleUploadFileFromLocal
(
zipFile
,
task
.
getName
()
+
"-"
+
task
.
getId
()+
taskFileExt
,
BucketNameEnum
.
COMPRESS_60000
.
getName
());
}
else
{
log
.
debug
(
"开始上传文件到腾讯云"
,
task
.
getId
());
cloudFileUrl
+=
FileUploadUtil
.
simpleUploadFileFromLocal
(
new
File
(
originalFilePath
),
task
.
getName
()
+
"-"
+
task
.
getId
()+
taskFileExt
,
BucketNameEnum
.
REPORT_50000
.
getName
());
}
log
.
debug
(
"上传腾讯云"
,
"地址为:"
+
cloudFileUrl
);
task
.
setStatus
(
DownloadTaskStatus
.
COMPLISHED
);
task
.
setOverTime
(
new
java
.
util
.
Date
());
task
.
setFilePath
(
cloudFileUrl
);
DownloadTaskServiceImpl
.
getInstance
().
updateDownloadTask
(
task
);
}
catch
(
Exception
ex
)
{
ex
.
printStackTrace
();
// 标记任务异常
task
.
setStatus
(
DownloadTaskStatus
.
ERROR
);
task
.
setOverTime
(
new
Date
());
DownloadTaskServiceImpl
.
getInstance
().
updateDownloadTask
(
task
);
}
finally
{
try
{
conn
.
close
();
}
catch
(
SQLException
e
)
{
e
.
printStackTrace
();
}
}
}
// IF OVER
}
// IF OVER
}
// 没有任务则忽略
}
catch
(
Exception
e
){
log
.
debug
(
"自定义查询下载异常"
,
e
.
getMessage
());
e
.
printStackTrace
();
}
}
},
interval
*
1000
,
interval
*
1000
,
TimeUnit
.
MILLISECONDS
);
}
private
void
saveXlsx
(
FreeQueryTaskCondition
condition
,
String
originalFilePath
,
ResultSet
rs
,
DownloadTask
task
)
throws
Exception
{
Integer
limitSize
=
FileUtil
.
getLimitSize
();
// originalFilePath = SAVE_FOLDER + "/" + condition.getTaskId() + ".xlsx";
SXSSFWorkbook
wb
=
new
SXSSFWorkbook
(
100
);
// 内存中保留 100 行
Sheet
sheet
=
wb
.
createSheet
();
Row
row
=
sheet
.
createRow
(
0
);
...
...
@@ -444,7 +606,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
}
// 遍历输出行
int
rowCount
=
0
;
while
(
rs
.
next
())
{
while
(
rowCount
<
limitSize
&&
rs
.
next
())
{
rowCount
++;
row
=
sheet
.
createRow
(
rowCount
);
for
(
int
j
=
0
;
j
<
rs
.
getMetaData
().
getColumnCount
();
++
j
)
{
...
...
@@ -491,73 +653,6 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
wb
.
dispose
();
// SXSSFWorkbook 没有 close 方法
log
.
debug
(
"runDownloadTask.run"
,
"已生成自定义查询下载文件 "
+
condition
.
getTaskId
()
+
".xlsx"
);
}
String
cloudFileUrl
=
"https://"
;
String
taskFileExt
=
task
.
getUseCompress
().
equals
(
Global
.
YES
)
?
".zip"
:
task
.
getFormat
().
equals
(
DownloadFileFormat
.
CSV
)
?
".csv"
:
".xlsx"
;
if
(
task
.
getUseCompress
().
equals
(
Global
.
YES
))
{
log
.
debug
(
"runDownloadTask.run"
,
"准备生成自定义查询压缩文件 "
+
condition
.
getTaskId
()
+
".zip"
);
String
zipFilePath
=
SAVE_FOLDER
+
"/"
+
condition
.
getTaskId
()
+
".zip"
;
File
zipFile
=
new
File
(
zipFilePath
);
ZipOutputStream
zos
=
null
;
byte
[]
buf
=
new
byte
[
1024
];
int
length
=
0
;
try
{
OutputStream
os
=
new
FileOutputStream
(
zipFilePath
);
BufferedOutputStream
bos
=
new
BufferedOutputStream
(
os
);
zos
=
new
ZipOutputStream
(
bos
);
zos
.
setLevel
(
6
);
// 压缩率选择 0-9
InputStream
is
=
new
FileInputStream
(
originalFilePath
);
BufferedInputStream
bis
=
new
BufferedInputStream
(
is
);
zos
.
putNextEntry
(
new
ZipEntry
(
originalFilePath
.
substring
(
originalFilePath
.
lastIndexOf
(
"/"
)
+
1
)));
while
((
length
=
bis
.
read
(
buf
))
>
0
)
{
zos
.
write
(
buf
,
0
,
length
);
}
bis
.
close
();
is
.
close
();
//bos.close();
//os.close();
log
.
debug
(
"runDownloadTask.run"
,
"已生成自定义查询压缩文件 "
+
condition
.
getTaskId
()
+
".zip"
);
}
catch
(
Exception
ex2
)
{
throw
ex2
;
}
finally
{
zos
.
closeEntry
();
zos
.
close
();
}
log
.
debug
(
"开始上传压缩文件到腾讯云"
,
task
.
getId
());
cloudFileUrl
+=
FileUploadUtil
.
simpleUploadFileFromLocal
(
zipFile
,
task
.
getName
()
+
"-"
+
task
.
getId
()+
taskFileExt
,
BucketNameEnum
.
COMPRESS_60000
.
getName
());
}
else
{
log
.
debug
(
"开始上传文件到腾讯云"
,
task
.
getId
());
cloudFileUrl
+=
FileUploadUtil
.
simpleUploadFileFromLocal
(
new
File
(
originalFilePath
),
task
.
getName
()
+
"-"
+
task
.
getId
()+
taskFileExt
,
BucketNameEnum
.
REPORT_50000
.
getName
());
}
log
.
debug
(
"上传腾讯云"
,
"地址为:"
+
cloudFileUrl
);
task
.
setStatus
(
DownloadTaskStatus
.
COMPLISHED
);
task
.
setOverTime
(
new
java
.
util
.
Date
());
task
.
setFilePath
(
cloudFileUrl
);
DownloadTaskServiceImpl
.
getInstance
().
updateDownloadTask
(
task
);
}
catch
(
Exception
ex
)
{
ex
.
printStackTrace
();
// 标记任务异常
task
.
setStatus
(
DownloadTaskStatus
.
ERROR
);
task
.
setOverTime
(
new
Date
());
DownloadTaskServiceImpl
.
getInstance
().
updateDownloadTask
(
task
);
}
finally
{
try
{
conn
.
close
();
}
catch
(
SQLException
e
)
{
e
.
printStackTrace
();
}
}
}
// IF OVER
}
// IF OVER
}
// 没有任务则忽略
}
catch
(
Exception
e
){
log
.
debug
(
"自定义查询下载异常"
,
e
.
getMessage
());
e
.
printStackTrace
();
}
}
},
interval
*
1000
,
interval
*
1000
,
TimeUnit
.
MILLISECONDS
);
}
/** 下载申请检查计时器 */
//private Timer applyTimer = new Timer();
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment