Merge branch 'feature-2023-04' into 'developer'

Feature 2023 04

See merge request !83

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java

@@ -667,8 +667,9 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 		distService.scheduleAtFixedRate(new Runnable() {
 			@Override
 			public void run() {
+				String traceId = UUID.randomUUID().toString();
 				ProviderLocalTag providerLocalTag = ProviderLocalTag.tag.get();
-				providerLocalTag.traceId = UUID.randomUUID().toString();
+				providerLocalTag.traceId = traceId;
 				logger.info("执行hivefile");
 				try{
 					String lockKey="data:hook:hive";
@@ -683,6 +684,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 					}
 					RedisUtil.unlock(lockKey);
 					downloadTasks.parallelStream().forEach(mid->{
+						ProviderLocalTag localTag = ProviderLocalTag.tag.get();
+						localTag.traceId = traceId;
 						//下载处理
 						takeFileNew(mid);
 					});
@@ -932,6 +935,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 	private class XlsxFileInfo{
 		String filepath;
 		SXSSFWorkbook workbook;
+		CellStyle yyyyMMddhhmmss;
+		CellStyle yyyyMMdd;
 	}
 
 
@@ -1207,22 +1212,24 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 					cell = row.createCell(j);
 					cell.setCellValue(titles.get(j).getTitle());
 				}
-				count.incrementAndGet();
-			}
-			logger.info("{}-count:{}",originalFilePath,count.get());
 
-			SXSSFWorkbook wb = xlsxFileInfo.workbook; // 内存中保留 100 行
-			if (!endFlag) {
-				Sheet sheet = wb.getSheetAt(0);
 				//日期
-				CellStyle yyyyMMddhhmmss = wb.createCellStyle();
-				DataFormat dataFormat = wb.createDataFormat();
+				CellStyle yyyyMMddhhmmss = xlsxFileInfo.workbook.createCellStyle();
+				DataFormat dataFormat = xlsxFileInfo.workbook.createDataFormat();
 				yyyyMMddhhmmss.setDataFormat(dataFormat.getFormat("yyyy-MM-dd HH:mm:ss"));
+				xlsxFileInfo.yyyyMMddhhmmss=yyyyMMddhhmmss;
 
 				//日期
-				CellStyle yyyyMMdd = wb.createCellStyle();
-				DataFormat yyyyMMddDataFormat = wb.createDataFormat();
+				CellStyle yyyyMMdd = xlsxFileInfo.workbook.createCellStyle();
+				DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat();
 				yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd"));
+				xlsxFileInfo.yyyyMMdd=yyyyMMdd;
+				count.incrementAndGet();
+			}
+
+			SXSSFWorkbook wb = xlsxFileInfo.workbook; // 内存中保留 100 行
+			if (!endFlag) {
+				Sheet sheet = wb.getSheetAt(0);
 
 				Row row = sheet.createRow(count.get());
 				for (int j = 0; j < cells.length; j++) {
@@ -1236,10 +1243,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 					switch (type) {
 						case FlatQueryFieldType.DATE:
 							if (columnValue.length() == 10) {
-								midCell.setCellStyle(yyyyMMdd);
+								midCell.setCellStyle(xlsxFileInfo.yyyyMMdd);
 								midCell.setCellValue(DateUtils.parseDate(columnValue, new String[]{"yyyy-MM-dd"}));
 							} else {
-								midCell.setCellStyle(yyyyMMddhhmmss);
+								midCell.setCellStyle(xlsxFileInfo.yyyyMMddhhmmss);
 								midCell.setCellValue(DateUtils.parseDate(columnValue, new String[]{"yyyy-MM-dd HH:mm:ss"}));
 							}
 							break;
@@ -1258,6 +1265,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 			Integer limitSize = FileUtil.getLimitSize();
 			int c = count.get();
 			if (c>limitSize||endFlag) {
+				logger.info("{}-count:{}",originalFilePath,count.get());
 				FileOutputStream fileOut = new FileOutputStream(originalFilePath);
 				wb.write(fileOut);
 				//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式

gic-cloud-data-hook-service/src/test/java/Test.java

@@ -18,7 +18,7 @@ public class Test {
 
     @org.junit.Test
     public void test(){
-        DownloadTask downloadTask = downloadTaskService.getDownloadTaskById("1680832767459");
+        DownloadTask downloadTask = downloadTaskService.getDownloadTaskById("1680836264864");
         flatQueryResultService.takeFileNew(downloadTask);
         try {
             Thread.sleep(10000000000L);

gic-cloud-data-hook-service/src/test/java/hdfs.java

@@ -6,6 +6,7 @@ import org.apache.hadoop.fs.Path;
 
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.Arrays;
 
 public class hdfs {
     public static void main(String[] args) {
@@ -20,14 +21,20 @@ public class hdfs {
 //        }
 
 //        HDFSUtil.getInstance().downloadFile("/data/emr/order-1.csv","D:\\testorder");
+        Integer[] arr=new Integer[]{1,2,3,4,5,6};
+        Arrays.asList(arr).parallelStream().forEach(mid->{
+            Thread thread = Thread.currentThread();
+            String name = thread.getName();
+            System.out.println(name);
+        });
 
-        try{
-            CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));
-            String[] cells = csvReader.readNext();
-            System.out.println(cells);
-        }catch (Exception e) {
-
-        }
+//        try{
+//            CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));
+//            String[] cells = csvReader.readNext();
+//            System.out.println(cells);
+//        }catch (Exception e) {
+//
+//        }
 
     }
 }