Skip to content

G
gic-cloud
Commit 5ace0ff6 by fudahua
Options

Merge branch 'fix-2023-04' into 'master' 

Fix 2023 04

See merge request !100
parents f22d725e 0e338de7
Showing with 182 additions and 10 deletions
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DingWarningUtil.java 0 → 100644
package com.gic.cloud.data.hook.service;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.gic.commons.util.DingtalkMessageUtil;
import com.gic.commons.util.TraceIdUtil;
import com.gic.dubbo.entity.ProviderLocalTag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class DingWarningUtil {
private static Logger logger = LoggerFactory.getLogger(DingWarningUtil.class);
private static final String dingdingurl="https://oapi.dingtalk.com/robot/send?access_token=1d6a0756fcfe5f215e3ed37d7e60960635d0009f71b476f3b402e7671fc2a8ca";
public static void sendCountWarning(String msg) {
Config appConfig = ConfigService.getAppConfig();
String url = appConfig.getProperty("ding.url", dingdingurl);
ProviderLocalTag localTag = ProviderLocalTag.tag.get();
String traceId = localTag.traceId;
try {
msg=msg+": "+traceId;
DingtalkMessageUtil.sendAlertMessage(msg,url);
}catch (Exception e) {
logger.info("异常：{}",e);
}
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/HDFSUtil.java
......@@ -16,6 +16,8 @@ public class HDFSUtil {
private static HDFSUtil hdfsUtil=null;
private static FileSystem fileSystem=null;
public static final String HDFS_URL = "/data/hook";
public static HDFSUtil getInstance(){
if (hdfsUtil==null) {
synchronized (HDFSUtil.class) {
......@@ -56,4 +58,26 @@ public class HDFSUtil {
return false;
}
}
public boolean deleteFile(String srcPath){
try {
fileSystem.delete(new Path(srcPath),true);
} catch (IOException e) {
e.printStackTrace();
}
return true;
}
public String getHdfsName(String id){
//下载文件
String dirName="hdfs"+id;
return dirName;
}
public String getHdfsPath(String id){
//下载文件
String dirName=getHdfsName(id);
String path=HDFS_URL+"/"+dirName;
return path;
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/DownloadTaskServiceImpl.java
......@@ -10,6 +10,7 @@ import com.gic.cloud.data.hook.api.entity.DownloadTaskStatus;
import com.gic.cloud.data.hook.api.entity.FreeQueryTaskCondition;
import com.gic.cloud.data.hook.api.entity.Global;
import com.gic.cloud.data.hook.api.service.IDownloadTaskService;
import com.gic.cloud.data.hook.service.HDFSUtil;
import com.gic.cloud.data.hook.service.MysqlHelper;
import com.gic.cloud.data.hook.service.dao.DownloadRecordDao;
import com.gic.cloud.data.hook.service.dao.DownloadTaskDao;
......@@ -165,6 +166,10 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
}
this.downloadTaskDao.deleteDownloadTask(task.getId());
CloudFileUtil.delFileByUrl(task.getFilePath());
String hdfsPath = HDFSUtil.getInstance().getHdfsPath(task.getId());
HDFSUtil.getInstance().deleteFile(hdfsPath);
return true;
} else return false;
}
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
......@@ -73,7 +73,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
public static final String SAVE_FOLDER = "/usr/local/data-hook-file";
// public static final String SAVE_FOLDER = "D:\\testorder";
public static final String HDFS_URL = "/data/hook";
// public static final String HDFS_URL = "/data/hook";
public static final String LOCK_KEY="data:hook:hive";
......@@ -919,8 +919,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
task.setDownloadWay(-1);
task.setFieldSize(condition.getAllFields().size());
//下载文件
String dirName="hdfs"+task.getId();
String path=HDFS_URL+"/"+dirName;
String dirName=HDFSUtil.getInstance().getHdfsName(task.getId());
String path=HDFSUtil.getInstance().getHdfsPath(task.getId());
HDFSUtil.getInstance().getHdfsName(task.getId());
try {
StopWatch stopWatch = StopWatch.create("down");
stopWatch.start();
......@@ -937,7 +939,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
AtomicInteger totalCount=new AtomicInteger(0);
AtomicInteger count=new AtomicInteger(0);
AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
readJsonFile(condition,dirName,(cells,titles,firstFlag)->{
if (count.get()==0) {
XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
xlsxFileInfo.filepath=SAVE_FOLDER + "/" + task.getId() + xlsxFiles.size() + ".xlsx";
......@@ -953,6 +955,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,queryDataType);
stopWatch.stop();
logger.info("写入本地excel耗时：{}, 数量: {}-》{}",stopWatch.getLastTaskTimeMillis(),task.getAmount(),total);
if (task.getAmount()!=total.intValue()) {
String msg="取数任务："+task.getId()+"下载数量对不上: 数量: "+task.getAmount()+"->"+total;
DingWarningUtil.sendCountWarning(msg);
}
stopWatch.start();
//是否压缩
boolean zipFlag = (xlsxFiles.size() > 1) ? true : false;
......@@ -1071,6 +1077,58 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}
/**
* 读物文件
* @param dirName
* @param func
*/
private void readJsonFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles();
List<FlatQueryCondition> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions();
List<String> keys = conditions.stream().map(mid -> mid.getFieldMark()).collect(Collectors.toList());
Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) {
if (!midFile.getName().endsWith("json")) {
continue;
}
try {
BufferedReader reader=new BufferedReader(new FileReader(midFile));
boolean first=true;
Exception exception=null;
try {
String line = reader.readLine();
while (line != null) {
List<String> cellList=new ArrayList<>();
JSONObject jsonObject = JSONObject.parseObject(line);
for (String key : keys) {
String cellVal = jsonObject.getString(key);
cellList.add(cellVal);
}
String[] cells = cellList.toArray(new String[]{});
func.deal(cells,conditions,first);
first=false;
line = reader.readLine();
}
logger.info("读取结束：{}",midFile.getName());
} catch (Exception e) {
exception=e;
}finally {
reader.close();
}
if (exception!=null) {
throw exception;
}
} catch (Exception e) {
logger.info("读取异常：{}",e);
throw new RuntimeException(e);
}
}
}
private void takeFile(FlatQueryTaskCondition condition, Connection conn, int downloadType){
if (condition != null) {
......
gic-cloud-data-hook-service/src/test/java/Test.java
......@@ -42,7 +42,7 @@ public class Test {
@org.junit.Test
public void test2(){
String json="{\"allFields\":[\"is_mbr\",\"main_store_name\",\"main_store_code\",\"card_num\",\"open_card_time\",\"clerk_name\",\"add_clerk_code\",\"clerk_store_name\",\"clerk_store_code\",\"add_buddy_time\",\"create_time\"],\"amount\":1169963,\"authStoreIdList\":[],\"buildPermitted\":\"1\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否会员\",\"fieldName\":\"is_mbr\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"是\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店名称\",\"fieldName\":\"main_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店code\",\"fieldName\":\"main_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"open_card_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购姓名\",\"fieldName\":\"clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购code\",\"fieldName\":\"add_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"导购门店名称\",\"fieldName\":\"clerk_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购所属门店code\",\"fieldName\":\"clerk_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"2023-02-28 23:59:59\",\"fieldMark\":\"添加好友时间\",\"fieldName\":\"add_buddy_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"2010-03-01 00:00:00\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好办导购与客户建立好友关系时间\",\"fieldName\":\"create_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[],\"enterpriseIds\":[\"ff80808180b3c54a0180bc3df3bb4bca\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":0,\"tableId\":\"extract_entwechat_detail_d\",\"taskId\":\"1678674610424\"}";
String json="{\"allFields\":[\"use_time\",\"source_integral_log_id\",\"operation_integral_log_id\",\"member_name\",\"member_sxe\",\"phone_number\",\"card_num\",\"grade_name\",\"source_integral_change\",\"interval_remark\",\"remark\",\"integral_to_fee\",\"card_name\",\"card_code\",\"card_denomination\",\"cost_value\",\"status\",\"order_store_name\",\"order_store_code\",\"order_store_group_name\",\"order_store_type\",\"order_id\",\"enterprise_id\",\"integral_channel\",\"check_store_status\"],\"amount\":4509765,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗时间\",\"fieldName\":\"use_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"流水号\",\"fieldName\":\"source_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"关联流水号\",\"fieldName\":\"operation_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_sxe\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗额\",\"fieldName\":\"source_integral_change\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"事由\",\"fieldName\":\"interval_remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"备注\",\"fieldName\":\"remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"成本总额\",\"fieldName\":\"integral_to_fee\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券名称\",\"fieldName\":\"card_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券编码\",\"fieldName\":\"card_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券面额\",\"fieldName\":\"card_denomination\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券成本费用\",\"fieldName\":\"cost_value\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"优惠券状态\",\"fieldName\":\"status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店代码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店所属分组\",\"fieldName\":\"order_store_group_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"零售单号\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业id\",\"fieldName\":\"enterprise_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分来源渠道\",\"fieldName\":\"integral_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店状态\",\"fieldName\":\"check_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff8080817d9fbda8017dc20674f47fb6\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_integral_use_detail\",\"taskId\":\"1675670752773\"}";
FlatQueryTaskCondition condition = JSON.parseObject(json, FlatQueryTaskCondition.class);
String fullQuery = flatQueryResultService.buildFlatQuerySQL(
false, // 下载用途
......
gic-cloud-data-hook-service/src/test/java/com/gic/ExceTest2.java
......@@ -8,9 +8,8 @@ import com.gic.cloud.data.hook.api.entity.FlatQueryTaskCondition;
import com.gic.cloud.data.hook.service.DecryptUtils;
import com.gic.cloud.data.hook.service.DownloadFunc;
import com.gic.cloud.data.hook.service.FileUtil;
import com.gic.cloud.data.hook.service.entity.ColumnInfo;
import com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.*;
......@@ -18,6 +17,7 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
......@@ -45,9 +45,11 @@ public class ExceTest2 {
if (!midFile.getName().endsWith("csv")) {
continue;
}
String lastcells=null;
String[] cells=null;
try {
CSVReader csvReader = new CSVReader(new FileReader(midFile));
String[] cells = csvReader.readNext();
CSVReader csvReader = new CSVReaderBuilder(new FileReader(midFile)).build();
cells = csvReader.readNext();
boolean first=true;
do {
if (titles == null) {
......@@ -62,15 +64,70 @@ public class ExceTest2 {
continue;
}
func.deal(cells,titles,first);
lastcells=JSONObject.toJSONString(cells);
}while ((cells = csvReader.readNext())!=null);
logger.info("读取结束：{}",midFile.getName());
csvReader.close();
} catch (Exception e) {
logger.info("lastcell: {}",lastcells);
logger.info("cells: {}",cells[0]);
logger.info("cells: {},{},{}",cells[1]);
logger.info("读取异常：{}",e);
}
}
}
/**
* 读物文件
* @param dirName
* @param func
*/
private static void readJsonFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles();
List<FlatQueryCondition> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions();
List<String> keys = conditions.stream().map(mid -> mid.getFieldMark()).collect(Collectors.toList());
Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) {
if (!midFile.getName().endsWith("json")) {
continue;
}
try {
BufferedReader reader=new BufferedReader(new FileReader(midFile));
boolean first=true;
Exception exception=null;
try {
String line = reader.readLine();
while (line != null) {
List<String> cellList=new ArrayList<>();
JSONObject jsonObject = JSONObject.parseObject(line);
for (String key : keys) {
String cellVal = jsonObject.getString(key);
cellList.add(cellVal);
}
String[] cells = cellList.toArray(new String[]{});
func.deal(cells,conditions,first);
first=false;
line = reader.readLine();
}
logger.info("读取结束：{}",midFile.getName());
} catch (Exception e) {
exception=e;
}finally {
reader.close();
}
if (exception!=null) {
throw exception;
}
} catch (Exception e) {
logger.info("读取异常：{}",e);
throw new RuntimeException(e);
}
}
}
public static class XlsxFileInfo{
String filepath;
SXSSFWorkbook workbook;
......@@ -178,7 +235,7 @@ public class ExceTest2 {
List<String> xlsxFiles=new ArrayList<>();
AtomicInteger count=new AtomicInteger(0);
AtomicReference<XlsxFileInfo> currentFile=new AtomicReference<>();
readCsvFile(condition,dirName,(cells,titles,firstFlag)->{
readJsonFile(condition,dirName,(cells,titles,firstFlag)->{
if (count.get()==0) {
XlsxFileInfo xlsxFileInfo = new XlsxFileInfo();
xlsxFileInfo.filepath=SAVE_FOLDER + "/" + taskId + xlsxFiles.size() + ".xlsx";
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment