Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
G
gic-cloud
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
data-hook
gic-cloud
Commits
39223cb3
Commit
39223cb3
authored
Apr 07, 2023
by
fudahua
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'feature-2023-04' into 'developer'
Feature 2023 04 See merge request
!85
parents
da185f3a
f5adfddd
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
45 additions
and
34 deletions
+45
-34
FlatQueryResultServiceImpl.java
...ud/data/hook/service/impl/FlatQueryResultServiceImpl.java
+17
-5
ExceTest2.java
gic-cloud-data-hook-service/src/test/java/ExceTest2.java
+0
-18
ExcelTest.java
gic-cloud-data-hook-service/src/test/java/ExcelTest.java
+0
-4
Test.java
gic-cloud-data-hook-service/src/test/java/Test.java
+21
-0
ExceTest2.java
...ud-data-hook-service/src/test/java/com/gic/ExceTest2.java
+0
-0
hdfs.java
gic-cloud-data-hook-service/src/test/java/hdfs.java
+7
-7
No files found.
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
View file @
39223cb3
package
com
.
gic
.
cloud
.
data
.
hook
.
service
.
impl
;
package
com
.
gic
.
cloud
.
data
.
hook
.
service
.
impl
;
import
cn.hutool.core.date.StopWatch
;
import
cn.medubi.client.utils.LogPak
;
import
cn.medubi.client.utils.LogPak
;
import
com.alibaba.fastjson.JSON
;
import
com.alibaba.fastjson.JSON
;
import
com.alibaba.fastjson.JSONObject
;
import
com.alibaba.fastjson.JSONObject
;
...
@@ -877,11 +878,17 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -877,11 +878,17 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
String
dirName
=
"hdfs"
+
task
.
getId
();
String
dirName
=
"hdfs"
+
task
.
getId
();
String
path
=
HDFS_URL
+
"/"
+
dirName
;
String
path
=
HDFS_URL
+
"/"
+
dirName
;
try
{
try
{
StopWatch
stopWatch
=
StopWatch
.
create
(
"down"
);
stopWatch
.
start
();
logger
.
info
(
"下载开始"
);
boolean
downloadFlag
=
HDFSUtil
.
getInstance
().
downloadFile
(
path
,
SAVE_FOLDER
);
boolean
downloadFlag
=
HDFSUtil
.
getInstance
().
downloadFile
(
path
,
SAVE_FOLDER
);
if
(!
downloadFlag
)
{
if
(!
downloadFlag
)
{
logger
.
info
(
"下载失败:{}-{}"
,
path
,
JSONObject
.
toJSONString
(
task
));
logger
.
info
(
"下载失败:{}-{}"
,
path
,
JSONObject
.
toJSONString
(
task
));
return
;
return
;
}
}
stopWatch
.
stop
();
logger
.
info
(
"下载耗时:{}"
,
stopWatch
.
getLastTaskTimeMillis
());
stopWatch
.
start
();
List
<
String
>
xlsxFiles
=
new
ArrayList
<>();
List
<
String
>
xlsxFiles
=
new
ArrayList
<>();
AtomicInteger
count
=
new
AtomicInteger
(
0
);
AtomicInteger
count
=
new
AtomicInteger
(
0
);
AtomicReference
<
XlsxFileInfo
>
currentFile
=
new
AtomicReference
<>();
AtomicReference
<
XlsxFileInfo
>
currentFile
=
new
AtomicReference
<>();
...
@@ -893,11 +900,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -893,11 +900,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
xlsxFiles
.
add
(
currentFile
.
get
().
filepath
);
xlsxFiles
.
add
(
currentFile
.
get
().
filepath
);
}
}
saveXlsSplitNew
(
currentFile
.
get
().
filepath
,
cells
,
titles
,
currentFile
.
get
(),
count
,
false
);
saveXlsSplitNew
(
currentFile
.
get
().
filepath
,
cells
,
titles
,
currentFile
.
get
(),
count
,
false
);
count
.
incrementAndGet
();
});
});
//结束
//结束
saveXlsSplitNew
(
currentFile
.
get
().
filepath
,
null
,
null
,
currentFile
.
get
(),
count
,
true
);
saveXlsSplitNew
(
currentFile
.
get
().
filepath
,
null
,
null
,
currentFile
.
get
(),
count
,
true
);
stopWatch
.
stop
();
logger
.
info
(
"写入本地excel耗时:{}"
,
stopWatch
.
getLastTaskTimeMillis
());
stopWatch
.
start
();
//是否压缩
//是否压缩
boolean
zipFlag
=
(
xlsxFiles
.
size
()
>
1
)
?
true
:
false
;
boolean
zipFlag
=
(
xlsxFiles
.
size
()
>
1
)
?
true
:
false
;
String
cloudFileUrl
=
null
;
String
cloudFileUrl
=
null
;
...
@@ -915,7 +923,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -915,7 +923,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
logger
.
info
(
"[ 开始上传文件到腾讯云 ]: {}"
,
task
.
getId
());
logger
.
info
(
"[ 开始上传文件到腾讯云 ]: {}"
,
task
.
getId
());
cloudFileUrl
=
cloudFileUpload
(
task
.
getEnterpriseId
(),
new
File
(
currentFile
.
get
().
filepath
),
taskFileExt
.
substring
(
1
));
cloudFileUrl
=
cloudFileUpload
(
task
.
getEnterpriseId
(),
new
File
(
currentFile
.
get
().
filepath
),
taskFileExt
.
substring
(
1
));
}
}
stopWatch
.
stop
();
logger
.
info
(
"上传腾讯云耗时:{}"
,
stopWatch
.
getLastTaskTimeMillis
());
logger
.
info
(
"[ 上传腾讯云 ]: {}"
,
"地址为:"
+
cloudFileUrl
);
logger
.
info
(
"[ 上传腾讯云 ]: {}"
,
"地址为:"
+
cloudFileUrl
);
task
.
setStatus
(
DownloadTaskStatus
.
COMPLISHED
);
task
.
setStatus
(
DownloadTaskStatus
.
COMPLISHED
);
task
.
setOverTime
(
new
Date
());
task
.
setOverTime
(
new
Date
());
...
@@ -1225,8 +1234,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -1225,8 +1234,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
DataFormat
yyyyMMddDataFormat
=
xlsxFileInfo
.
workbook
.
createDataFormat
();
DataFormat
yyyyMMddDataFormat
=
xlsxFileInfo
.
workbook
.
createDataFormat
();
yyyyMMdd
.
setDataFormat
(
yyyyMMddDataFormat
.
getFormat
(
"yyyy-MM-dd"
));
yyyyMMdd
.
setDataFormat
(
yyyyMMddDataFormat
.
getFormat
(
"yyyy-MM-dd"
));
xlsxFileInfo
.
yyyyMMdd
=
yyyyMMdd
;
xlsxFileInfo
.
yyyyMMdd
=
yyyyMMdd
;
count
.
incrementAndGet
();
}
}
count
.
incrementAndGet
();
SXSSFWorkbook
wb
=
xlsxFileInfo
.
workbook
;
// 内存中保留 100 行
SXSSFWorkbook
wb
=
xlsxFileInfo
.
workbook
;
// 内存中保留 100 行
if
(!
endFlag
)
{
if
(!
endFlag
)
{
...
@@ -1265,7 +1274,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -1265,7 +1274,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}
}
Integer
limitSize
=
FileUtil
.
getLimitSize
();
Integer
limitSize
=
FileUtil
.
getLimitSize
();
int
c
=
count
.
get
();
int
c
=
count
.
get
();
if
(
c
>
limitSize
||
endFlag
)
{
if
(
c
%
5000
==
0
)
{
logger
.
info
(
"进度:{}-{}"
,
originalFilePath
,
c
);
}
if
(
c
>=
limitSize
.
intValue
()||
endFlag
)
{
logger
.
info
(
"{}-count:{}"
,
originalFilePath
,
count
.
get
());
logger
.
info
(
"{}-count:{}"
,
originalFilePath
,
count
.
get
());
FileOutputStream
fileOut
=
new
FileOutputStream
(
originalFilePath
);
FileOutputStream
fileOut
=
new
FileOutputStream
(
originalFilePath
);
wb
.
write
(
fileOut
);
wb
.
write
(
fileOut
);
...
...
gic-cloud-data-hook-service/src/test/java/ExceTest2.java
deleted
100644 → 0
View file @
da185f3a
import
org.apache.poi.ss.usermodel.*
;
import
org.apache.poi.xssf.streaming.SXSSFWorkbook
;
import
java.io.FileOutputStream
;
import
java.util.Date
;
public
class
ExceTest2
{
public
static
void
main
(
String
[]
args
)
{
int
n
=
25
;
int
c
=
100
;
Double
aDouble
=
Double
.
valueOf
((
double
)
n
/
c
);
System
.
out
.
println
(
aDouble
);
}
}
gic-cloud-data-hook-service/src/test/java/ExcelTest.java
View file @
39223cb3
...
@@ -21,10 +21,6 @@ public class ExcelTest {
...
@@ -21,10 +21,6 @@ public class ExcelTest {
DataFormat
format
=
wb
.
createDataFormat
();
DataFormat
format
=
wb
.
createDataFormat
();
cellStyle
.
setDataFormat
(
format
.
getFormat
(
"yyyy-MM-dd HH:mm:ss"
));
cellStyle
.
setDataFormat
(
format
.
getFormat
(
"yyyy-MM-dd HH:mm:ss"
));
// CellStyle cellStyle = wb.createCellStyle();
// DataFormat format = wb.createDataFormat();
// cellStyle.setDataFormat(format.getFormat("yyyy-MM-dd HH:mm:ss"));
int
n
=
0
;
int
n
=
0
;
do
{
do
{
Row
row
=
sheet
.
createRow
(
n
);
Row
row
=
sheet
.
createRow
(
n
);
...
...
gic-cloud-data-hook-service/src/test/java/Test.java
View file @
39223cb3
import
com.alibaba.fastjson.JSON
;
import
com.gic.cloud.data.hook.api.dto.DownloadTask
;
import
com.gic.cloud.data.hook.api.dto.DownloadTask
;
import
com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition
;
import
com.gic.cloud.data.hook.service.impl.DownloadTaskServiceImpl
;
import
com.gic.cloud.data.hook.service.impl.DownloadTaskServiceImpl
;
import
com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl
;
import
com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl
;
import
org.junit.runner.RunWith
;
import
org.junit.runner.RunWith
;
...
@@ -19,6 +21,7 @@ public class Test {
...
@@ -19,6 +21,7 @@ public class Test {
@org
.
junit
.
Test
@org
.
junit
.
Test
public
void
test
(){
public
void
test
(){
DownloadTask
downloadTask
=
downloadTaskService
.
getDownloadTaskById
(
"1680844420436"
);
DownloadTask
downloadTask
=
downloadTaskService
.
getDownloadTaskById
(
"1680844420436"
);
flatQueryResultService
.
takeFileNew
(
downloadTask
);
flatQueryResultService
.
takeFileNew
(
downloadTask
);
try
{
try
{
Thread
.
sleep
(
10000000000L
);
Thread
.
sleep
(
10000000000L
);
...
@@ -26,4 +29,22 @@ public class Test {
...
@@ -26,4 +29,22 @@ public class Test {
e
.
printStackTrace
();
e
.
printStackTrace
();
}
}
}
}
@org
.
junit
.
Test
public
void
test2
(){
String
json
=
"{\"allFields\":[\"use_time\",\"source_integral_log_id\",\"operation_integral_log_id\",\"member_name\",\"member_sxe\",\"phone_number\",\"card_num\",\"grade_name\",\"source_integral_change\",\"interval_remark\",\"remark\",\"integral_to_fee\",\"card_name\",\"card_code\",\"card_denomination\",\"cost_value\",\"status\",\"order_store_name\",\"order_store_code\",\"order_store_group_name\",\"order_store_type\",\"order_id\",\"enterprise_id\",\"integral_channel\",\"check_store_status\"],\"amount\":4509765,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗时间\",\"fieldName\":\"use_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"流水号\",\"fieldName\":\"source_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"关联流水号\",\"fieldName\":\"operation_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_sxe\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗额\",\"fieldName\":\"source_integral_change\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"事由\",\"fieldName\":\"interval_remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"备注\",\"fieldName\":\"remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"成本总额\",\"fieldName\":\"integral_to_fee\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券名称\",\"fieldName\":\"card_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券编码\",\"fieldName\":\"card_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券面额\",\"fieldName\":\"card_denomination\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券成本费用\",\"fieldName\":\"cost_value\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"优惠券状态\",\"fieldName\":\"status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店代码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店所属分组\",\"fieldName\":\"order_store_group_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"零售单号\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业id\",\"fieldName\":\"enterprise_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分来源渠道\",\"fieldName\":\"integral_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店状态\",\"fieldName\":\"check_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff8080817d9fbda8017dc20674f47fb6\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_integral_use_detail\",\"taskId\":\"1675670752773\"}"
;
FlatQueryTaskCondition
condition
=
JSON
.
parseObject
(
json
,
FlatQueryTaskCondition
.
class
);
String
fullQuery
=
flatQueryResultService
.
buildFlatQuerySQL
(
false
,
// 下载用途
condition
.
getTableId
(),
condition
.
getEnterpriseIds
(),
condition
.
getConditions
(),
condition
.
getOrderField
(),
condition
.
getOrderDir
(),
condition
.
getExecDistinct
(),
0
,
condition
.
getAuthStoreIdList
());
System
.
out
.
println
(
fullQuery
);
}
}
}
gic-cloud-data-hook-service/src/test/java/com/gic/ExceTest2.java
0 → 100644
View file @
39223cb3
This diff is collapsed.
Click to expand it.
gic-cloud-data-hook-service/src/test/java/hdfs.java
View file @
39223cb3
...
@@ -20,13 +20,13 @@ public class hdfs {
...
@@ -20,13 +20,13 @@ public class hdfs {
// e.printStackTrace();
// e.printStackTrace();
// }
// }
// HDFSUtil.getInstance().downloadFile("/data/emr/order-1.csv
","D:\\testorder");
HDFSUtil
.
getInstance
().
downloadFile
(
"/data/hook/hdfs1680836264869
"
,
"D:\\testorder"
);
Integer
[]
arr
=
new
Integer
[]{
1
,
2
,
3
,
4
,
5
,
6
};
//
Integer[] arr=new Integer[]{1,2,3,4,5,6};
Arrays
.
asList
(
arr
).
parallelStream
().
forEach
(
mid
->{
//
Arrays.asList(arr).parallelStream().forEach(mid->{
Thread
thread
=
Thread
.
currentThread
();
//
Thread thread = Thread.currentThread();
String
name
=
thread
.
getName
();
//
String name = thread.getName();
System
.
out
.
println
(
name
);
//
System.out.println(name);
});
//
});
// try{
// try{
// CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));
// CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment