Merge branch 'feature-2023-04' into 'developer'

Feature 2023 04

See merge request !85

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java

 package com.gic.cloud.data.hook.service.impl;
 
+import cn.hutool.core.date.StopWatch;
 import cn.medubi.client.utils.LogPak;
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
@@ -877,11 +878,17 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 		String dirName="hdfs"+task.getId();
 		String path=HDFS_URL+"/"+dirName;
 		try {
+			StopWatch stopWatch = StopWatch.create("down");
+			stopWatch.start();
+			logger.info("下载开始");
 			boolean downloadFlag = HDFSUtil.getInstance().downloadFile(path, SAVE_FOLDER);
 			if (!downloadFlag) {
 				logger.info("下载失败：{}-{}",path,JSONObject.toJSONString(task));
 				return;
 			}
+			stopWatch.stop();
+			logger.info("下载耗时：{}",stopWatch.getLastTaskTimeMillis());
+			stopWatch.start();
 			List<String> xlsxFiles=new ArrayList<>();
 			AtomicInteger count=new AtomicInteger(0);
 			AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
@@ -893,11 +900,12 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 					xlsxFiles.add(currentFile.get().filepath);
 				}
 				saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false);
-				count.incrementAndGet();
 			});
 			//结束
 			saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true);
-
+			stopWatch.stop();
+			logger.info("写入本地excel耗时：{}",stopWatch.getLastTaskTimeMillis());
+			stopWatch.start();
 			//是否压缩
 			boolean zipFlag = (xlsxFiles.size() > 1) ? true : false;
 			String cloudFileUrl=null;
@@ -915,7 +923,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 				logger.info("[ 开始上传文件到腾讯云 ]: {}", task.getId());
 				cloudFileUrl = cloudFileUpload(task.getEnterpriseId(), new File(currentFile.get().filepath), taskFileExt.substring(1));
 			}
-
+			stopWatch.stop();
+			logger.info("上传腾讯云耗时：{}",stopWatch.getLastTaskTimeMillis());
 			logger.info("[ 上传腾讯云 ]: {}", "地址为：" + cloudFileUrl);
 			task.setStatus(DownloadTaskStatus.COMPLISHED);
 			task.setOverTime(new Date());
@@ -1225,8 +1234,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 				DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat();
 				yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd"));
 				xlsxFileInfo.yyyyMMdd=yyyyMMdd;
-				count.incrementAndGet();
 			}
+			count.incrementAndGet();
 
 			SXSSFWorkbook wb = xlsxFileInfo.workbook; // 内存中保留 100 行
 			if (!endFlag) {
@@ -1265,7 +1274,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 			}
 			Integer limitSize = FileUtil.getLimitSize();
 			int c = count.get();
-			if (c>limitSize||endFlag) {
+			if (c%5000==0) {
+				logger.info("进度：{}-{}",originalFilePath,c);
+			}
+			if (c>=limitSize.intValue()||endFlag) {
 				logger.info("{}-count:{}",originalFilePath,count.get());
 				FileOutputStream fileOut = new FileOutputStream(originalFilePath);
 				wb.write(fileOut);

gic-cloud-data-hook-service/src/test/java/ExceTest2.java

-import org.apache.poi.ss.usermodel.*;
-import org.apache.poi.xssf.streaming.SXSSFWorkbook;
-
-import java.io.FileOutputStream;
-import java.util.Date;
-
-public class ExceTest2 {
-
-
-
-    public static void main(String[] args) {
-       int n=25;
-       int c=100;
-
-        Double aDouble = Double.valueOf((double) n / c);
-        System.out.println(aDouble);
-    }
-}

gic-cloud-data-hook-service/src/test/java/ExcelTest.java

@@ -21,10 +21,6 @@ public class ExcelTest {
             DataFormat format = wb.createDataFormat();
             cellStyle.setDataFormat(format.getFormat("yyyy-MM-dd HH:mm:ss"));
 
-//            CellStyle cellStyle = wb.createCellStyle();
-//            DataFormat format = wb.createDataFormat();
-//            cellStyle.setDataFormat(format.getFormat("yyyy-MM-dd HH:mm:ss"));
-
             int n=0;
             do {
                 Row row = sheet.createRow(n);

gic-cloud-data-hook-service/src/test/java/Test.java

+import com.alibaba.fastjson.JSON;
 import com.gic.cloud.data.hook.api.dto.DownloadTask;
+import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition;
 import com.gic.cloud.data.hook.service.impl.DownloadTaskServiceImpl;
 import com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl;
 import org.junit.runner.RunWith;
@@ -19,6 +21,7 @@ public class Test {
     @org.junit.Test
     public void test(){
         DownloadTask downloadTask = downloadTaskService.getDownloadTaskById("1680844420436");
+
         flatQueryResultService.takeFileNew(downloadTask);
         try {
             Thread.sleep(10000000000L);
@@ -26,4 +29,22 @@ public class Test {
             e.printStackTrace();
         }
     }
+
+    @org.junit.Test
+    public void test2(){
+        String json="{\"allFields\":[\"use_time\",\"source_integral_log_id\",\"operation_integral_log_id\",\"member_name\",\"member_sxe\",\"phone_number\",\"card_num\",\"grade_name\",\"source_integral_change\",\"interval_remark\",\"remark\",\"integral_to_fee\",\"card_name\",\"card_code\",\"card_denomination\",\"cost_value\",\"status\",\"order_store_name\",\"order_store_code\",\"order_store_group_name\",\"order_store_type\",\"order_id\",\"enterprise_id\",\"integral_channel\",\"check_store_status\"],\"amount\":4509765,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗时间\",\"fieldName\":\"use_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"流水号\",\"fieldName\":\"source_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"关联流水号\",\"fieldName\":\"operation_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_sxe\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗额\",\"fieldName\":\"source_integral_change\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"事由\",\"fieldName\":\"interval_remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"备注\",\"fieldName\":\"remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"成本总额\",\"fieldName\":\"integral_to_fee\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券名称\",\"fieldName\":\"card_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券编码\",\"fieldName\":\"card_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券面额\",\"fieldName\":\"card_denomination\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券成本费用\",\"fieldName\":\"cost_value\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"优惠券状态\",\"fieldName\":\"status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店代码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店所属分组\",\"fieldName\":\"order_store_group_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"零售单号\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业id\",\"fieldName\":\"enterprise_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分来源渠道\",\"fieldName\":\"integral_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店状态\",\"fieldName\":\"check_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff8080817d9fbda8017dc20674f47fb6\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_integral_use_detail\",\"taskId\":\"1675670752773\"}";
+        FlatQueryTaskCondition condition = JSON.parseObject(json, FlatQueryTaskCondition.class);
+        String fullQuery = flatQueryResultService.buildFlatQuerySQL(
+                false, // 下载用途
+                condition.getTableId(),
+                condition.getEnterpriseIds(),
+                condition.getConditions(),
+                condition.getOrderField(),
+                condition.getOrderDir(),
+                condition.getExecDistinct(),
+                0,
+                condition.getAuthStoreIdList());
+
+        System.out.println(fullQuery);
+    }
 }

gic-cloud-data-hook-service/src/test/java/hdfs.java

@@ -20,13 +20,13 @@ public class hdfs {
 //            e.printStackTrace();
 //        }
 
-//        HDFSUtil.getInstance().downloadFile("/data/emr/order-1.csv","D:\\testorder");
-        Integer[] arr=new Integer[]{1,2,3,4,5,6};
-        Arrays.asList(arr).parallelStream().forEach(mid->{
-            Thread thread = Thread.currentThread();
-            String name = thread.getName();
-            System.out.println(name);
-        });
+        HDFSUtil.getInstance().downloadFile("/data/hook/hdfs1680836264869","D:\\testorder");
+//        Integer[] arr=new Integer[]{1,2,3,4,5,6};
+//        Arrays.asList(arr).parallelStream().forEach(mid->{
+//            Thread thread = Thread.currentThread();
+//            String name = thread.getName();
+//            System.out.println(name);
+//        });
 
 //        try{
 //            CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));