Commit 6be51646 by fudahua

Merge branch 'feature-2023-04' into 'developer'

Feature 2023 04

See merge request !83
parents 624fb42a 233a403c
......@@ -667,8 +667,9 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
distService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
String traceId = UUID.randomUUID().toString();
ProviderLocalTag providerLocalTag = ProviderLocalTag.tag.get();
providerLocalTag.traceId = UUID.randomUUID().toString();
providerLocalTag.traceId = traceId;
logger.info("执行hivefile");
try{
String lockKey="data:hook:hive";
......@@ -683,6 +684,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}
RedisUtil.unlock(lockKey);
downloadTasks.parallelStream().forEach(mid->{
ProviderLocalTag localTag = ProviderLocalTag.tag.get();
localTag.traceId = traceId;
//下载处理
takeFileNew(mid);
});
......@@ -932,6 +935,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private class XlsxFileInfo{
String filepath;
SXSSFWorkbook workbook;
CellStyle yyyyMMddhhmmss;
CellStyle yyyyMMdd;
}
......@@ -1207,22 +1212,24 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
cell = row.createCell(j);
cell.setCellValue(titles.get(j).getTitle());
}
count.incrementAndGet();
}
logger.info("{}-count:{}",originalFilePath,count.get());
SXSSFWorkbook wb = xlsxFileInfo.workbook; // 内存中保留 100 行
if (!endFlag) {
Sheet sheet = wb.getSheetAt(0);
//日期
CellStyle yyyyMMddhhmmss = wb.createCellStyle();
DataFormat dataFormat = wb.createDataFormat();
CellStyle yyyyMMddhhmmss = xlsxFileInfo.workbook.createCellStyle();
DataFormat dataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMddhhmmss.setDataFormat(dataFormat.getFormat("yyyy-MM-dd HH:mm:ss"));
xlsxFileInfo.yyyyMMddhhmmss=yyyyMMddhhmmss;
//日期
CellStyle yyyyMMdd = wb.createCellStyle();
DataFormat yyyyMMddDataFormat = wb.createDataFormat();
CellStyle yyyyMMdd = xlsxFileInfo.workbook.createCellStyle();
DataFormat yyyyMMddDataFormat = xlsxFileInfo.workbook.createDataFormat();
yyyyMMdd.setDataFormat(yyyyMMddDataFormat.getFormat("yyyy-MM-dd"));
xlsxFileInfo.yyyyMMdd=yyyyMMdd;
count.incrementAndGet();
}
SXSSFWorkbook wb = xlsxFileInfo.workbook; // 内存中保留 100 行
if (!endFlag) {
Sheet sheet = wb.getSheetAt(0);
Row row = sheet.createRow(count.get());
for (int j = 0; j < cells.length; j++) {
......@@ -1236,10 +1243,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
switch (type) {
case FlatQueryFieldType.DATE:
if (columnValue.length() == 10) {
midCell.setCellStyle(yyyyMMdd);
midCell.setCellStyle(xlsxFileInfo.yyyyMMdd);
midCell.setCellValue(DateUtils.parseDate(columnValue, new String[]{"yyyy-MM-dd"}));
} else {
midCell.setCellStyle(yyyyMMddhhmmss);
midCell.setCellStyle(xlsxFileInfo.yyyyMMddhhmmss);
midCell.setCellValue(DateUtils.parseDate(columnValue, new String[]{"yyyy-MM-dd HH:mm:ss"}));
}
break;
......@@ -1258,6 +1265,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
Integer limitSize = FileUtil.getLimitSize();
int c = count.get();
if (c>limitSize||endFlag) {
logger.info("{}-count:{}",originalFilePath,count.get());
FileOutputStream fileOut = new FileOutputStream(originalFilePath);
wb.write(fileOut);
//fileOut.flush(); // SXSSFWorkbook 使用 auto-flush 模式
......
......@@ -18,7 +18,7 @@ public class Test {
@org.junit.Test
public void test(){
DownloadTask downloadTask = downloadTaskService.getDownloadTaskById("1680832767459");
DownloadTask downloadTask = downloadTaskService.getDownloadTaskById("1680836264864");
flatQueryResultService.takeFileNew(downloadTask);
try {
Thread.sleep(10000000000L);
......
......@@ -6,6 +6,7 @@ import org.apache.hadoop.fs.Path;
import java.io.FileReader;
import java.io.IOException;
import java.util.Arrays;
public class hdfs {
public static void main(String[] args) {
......@@ -20,14 +21,20 @@ public class hdfs {
// }
// HDFSUtil.getInstance().downloadFile("/data/emr/order-1.csv","D:\\testorder");
Integer[] arr=new Integer[]{1,2,3,4,5,6};
Arrays.asList(arr).parallelStream().forEach(mid->{
Thread thread = Thread.currentThread();
String name = thread.getName();
System.out.println(name);
});
try{
CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));
String[] cells = csvReader.readNext();
System.out.println(cells);
}catch (Exception e) {
}
// try{
// CSVReader csvReader = new CSVReader(new FileReader("C:\\Users\\hua\\Desktop\\part-00000-fa2dd286-1eda-452d-91a3-a222beb0f327-c000.csv"));
// String[] cells = csvReader.readNext();
// System.out.println(cells);
// }catch (Exception e) {
//
// }
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment