Skip to content

G
gic-cloud
Commit dda1b337 by fudahua
Options

Merge branch 'feature-2023-04' into 'developer' 

Feature 2023 04

See merge request !86
parents 39223cb3 3697474e
Showing with 61 additions and 18 deletions
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/service/IFlatQueryResultService.java
......@@ -11,6 +11,12 @@ import java.util.List;
*/
public interface IFlatQueryResultService {
/**
* mq处理
* @param params
*/
public void dealFileMq(String params) ;
/** 自助指标查询
* @param tableId 指定的表明
* @param enterpriseIds 要查询的企业编号集合
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DecryptUtils.java
......@@ -3,6 +3,7 @@ package com.gic.cloud.data.hook.service;
import com.alibaba.dubbo.common.utils.StringUtils;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.gic.cloud.data.hook.api.dto.QueryDataType;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.service.impl.FreeQueryServiceImpl;
import com.gic.data.shield.SdkEnv;
......@@ -102,4 +103,26 @@ public class DecryptUtils {
}
}
/**
* 脱敏
* @param queryDataType
* @param columnVal
* @param condition
* @return
*/
public static String encryptionDeal(int queryDataType,String columnVal,FlatQueryCondition condition){
if(queryDataType == QueryDataType.FULL){
return columnVal;
} else {
if(condition.getEnableEncrypt()) {
return "******";
}else if (FilterFieldUtils.FILETERS_USER_NAME.contains(condition.getFieldName())) {
return DecryptUtils.dataSecurityProcessUserName(columnVal);
} else {
return columnVal;
}
}
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DownloadFunc.java
package com.gic.cloud.data.hook.service;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.service.entity.ColumnInfo;
import java.util.List;
public interface DownloadFunc {
public void deal(String[] cells, List<ColumnInfo> titles, boolean fileFirst);
public void deal(String[] cells, List<FlatQueryCondition> titles, boolean fileFirst);
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
......@@ -17,9 +17,11 @@ import com.gic.cloud.data.hook.service.dao.FlatQueryTableDao;
import com.gic.cloud.data.hook.service.entity.ColumnInfo;
import com.gic.cloud.data.hook.service.entity.CsvDataFilterMode;
import com.gic.cloud.data.hook.service.entity.CsvResultSetHelper;
import com.gic.commons.util.GICMQClientUtil;
import com.gic.dubbo.entity.ProviderLocalTag;
import com.gic.enterprise.api.dto.EnterpriseDTO;
import com.gic.enterprise.api.service.EnterpriseService;
import com.gic.mq.sdk.GicMQClient;
import com.gic.qcloud.FileRecordLogUtil;
import com.gic.redis.data.util.RedisUtil;
import com.gic.thirdparty.cloudfile.CloudFileUtil;
......@@ -122,6 +124,13 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
return zip.getOrgFileUrl();
}
@Override
public void dealFileMq(String params) {
logger.info("处理文件：{}",params);
DownloadTask downloadTask = JSON.parseObject(params, DownloadTask.class);
takeFileNew(downloadTask);
}
/** 生成自助指标查询最终 SQL
* @param queryOrDownload 查询或下载用途
* @param tableId
......@@ -675,21 +684,23 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
try{
String lockKey="data:hook:hive";
RedisUtil.lock(lockKey,2L);
List<DownloadTask> downloadTasks = DownloadTaskServiceImpl.getInstance().getDownloadTaskOfHasDownload(QueryDataSource.FLAT_QUERY,3);
List<DownloadTask> downloadTasks = DownloadTaskServiceImpl.getInstance().getDownloadTaskOfHasDownload(QueryDataSource.FLAT_QUERY,10);
if (CollectionUtils.isNotEmpty(downloadTasks)) {
for (DownloadTask downloadTask : downloadTasks) {
downloadTask.setStatus(DownloadTaskStatus.BUILDING);
downloadTask.setDownloadTime(new Date());
DownloadTaskServiceImpl.getInstance().updateDownloadTask(downloadTask);
GicMQClient clientInstance = GICMQClientUtil.getClientInstance();
clientInstance.sendMessage("dataHookFileDeal",JSONObject.toJSONString(downloadTask));
}
}
RedisUtil.unlock(lockKey);
downloadTasks.parallelStream().forEach(mid->{
ProviderLocalTag localTag = ProviderLocalTag.tag.get();
localTag.traceId = traceId;
//下载处理
takeFileNew(mid);
});
// downloadTasks.parallelStream().forEach(mid->{
// ProviderLocalTag localTag = ProviderLocalTag.tag.get();
// localTag.traceId = traceId;
// //下载处理
// takeFileNew(mid);
// });
}catch (Exception e){
logger.info("异常：{}",e);
logger.info("[ 自助指标下载异常 ]: {}", e.getMessage());
......@@ -867,6 +878,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
logger.info("不存在");
return;
}
Integer queryDataType = task.getQueryDataType();
FlatQueryTaskCondition condition = JSON.parseObject(task.getDownloadCondition(), FlatQueryTaskCondition.class);
if (!task.getStatus().equals(DownloadTaskStatus.BUILDING)) {
logger.info("文件hdfs没有处理完成：{}",JSONObject.toJSONString(task));
......@@ -899,10 +911,10 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
currentFile.set(xlsxFileInfo);
xlsxFiles.add(currentFile.get().filepath);
}
saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false);
saveXlsSplitNew(currentFile.get().filepath,cells,titles,currentFile.get(),count,false,queryDataType);
});
//结束
saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true);
saveXlsSplitNew(currentFile.get().filepath,null,null,currentFile.get(),count,true,queryDataType);
stopWatch.stop();
logger.info("写入本地excel耗时：{}",stopWatch.getLastTaskTimeMillis());
stopWatch.start();
......@@ -986,9 +998,9 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private void readCsvFile(FlatQueryTaskCondition condition, String dirName, DownloadFunc func){
File file = new File(SAVE_FOLDER+"/"+dirName);
File[] files = file.listFiles();
List<ColumnInfo> titles=null;
List<FlatQueryCondition> titles=null;
List<FlatQueryCondition> conditions = condition.getConditions();
Map<String, ColumnInfo> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid -> new ColumnInfo(mid.getFieldType(), mid.getFieldMark())));
Map<String, FlatQueryCondition> columnInfoMap = conditions.stream().collect(Collectors.toMap(mid -> mid.getFieldMark(), mid ->mid));
List<File> fileList = Arrays.stream(files).sorted(Comparator.comparing(File::getName)).collect(Collectors.toList());
for (File midFile : fileList) {
if (!midFile.getName().endsWith("csv")) {
......@@ -1210,7 +1222,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
* @param originalFilePath
* @param cells
*/
private void saveXlsSplitNew(String originalFilePath, String[] cells, List<ColumnInfo> titles, XlsxFileInfo xlsxFileInfo, AtomicInteger count,boolean endFlag){
private void saveXlsSplitNew(String originalFilePath, String[] cells, List<FlatQueryCondition> titles, XlsxFileInfo xlsxFileInfo, AtomicInteger count,boolean endFlag,int queryDataType){
try {
if (xlsxFileInfo.workbook==null) {
xlsxFileInfo.workbook=new SXSSFWorkbook(100);
......@@ -1220,7 +1232,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
logger.info("类型:{}",JSONObject.toJSONString(titles));
for(int j =0; j<titles.size(); j++){
cell = row.createCell(j);
cell.setCellValue(titles.get(j).getTitle());
cell.setCellValue(titles.get(j).getFieldMark());
}
//日期
......@@ -1243,7 +1255,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
Row row = sheet.createRow(count.get());
for (int j = 0; j < cells.length; j++) {
String type = titles.get(j).getType();
FlatQueryCondition flatQueryCondition = titles.get(j);
String type = titles.get(j).getFieldType();
Cell midCell = row.createCell(j);
String columnValue = cells[j];
if (StringUtils.isBlank(columnValue)) {
......@@ -1268,7 +1281,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}
break;
default:
midCell.setCellValue(columnValue);
midCell.setCellValue(DecryptUtils.encryptionDeal(queryDataType,columnValue,flatQueryCondition));
}
}
}
......@@ -1294,7 +1307,6 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}
private void saveXlsSplit(String originalFilePath,ResultSetHelper helper,ResultSet rs,FlatQueryTaskCondition condition){
try {
Integer limitSize = FileUtil.getLimitSize();
......@@ -1468,6 +1480,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}, interval*1000, interval * 1000, TimeUnit.MILLISECONDS);
}
/* *//** 运行下载申请任务
* @param interval
*//*
......
gic-cloud-data-hook-service/src/test/java/Test.java
......@@ -32,7 +32,7 @@ public class Test {
@org.junit.Test
public void test2(){
String json="{\"allFields\":[\"use_time\",\"source_integral_log_id\",\"operation_integral_log_id\",\"member_name\",\"member_sxe\",\"phone_number\",\"card_num\",\"grade_name\",\"source_integral_change\",\"interval_remark\",\"remark\",\"integral_to_fee\",\"card_name\",\"card_code\",\"card_denomination\",\"cost_value\",\"status\",\"order_store_name\",\"order_store_code\",\"order_store_group_name\",\"order_store_type\",\"order_id\",\"enterprise_id\",\"integral_channel\",\"check_store_status\"],\"amount\":4509765,\"authStoreIdList\":[],\"buildPermitted\":\"0\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗时间\",\"fieldName\":\"use_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"流水号\",\"fieldName\":\"source_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"关联流水号\",\"fieldName\":\"operation_integral_log_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员姓名\",\"fieldName\":\"member_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员性别\",\"fieldName\":\"member_sxe\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":true,\"extendFilter\":\"\",\"fieldMark\":\"手机号码\",\"fieldName\":\"phone_number\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员等级\",\"fieldName\":\"grade_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分消耗额\",\"fieldName\":\"source_integral_change\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"事由\",\"fieldName\":\"interval_remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"备注\",\"fieldName\":\"remark\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"成本总额\",\"fieldName\":\"integral_to_fee\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券名称\",\"fieldName\":\"card_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券编码\",\"fieldName\":\"card_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券面额\",\"fieldName\":\"card_denomination\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券成本费用\",\"fieldName\":\"cost_value\",\"fieldType\":\"number\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"优惠券状态\",\"fieldName\":\"status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店\",\"fieldName\":\"order_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店代码\",\"fieldName\":\"order_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店所属分组\",\"fieldName\":\"order_store_group_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店类型\",\"fieldName\":\"order_store_type\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"零售单号\",\"fieldName\":\"order_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"企业id\",\"fieldName\":\"enterprise_id\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"积分来源渠道\",\"fieldName\":\"integral_channel\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"卡券核销门店状态\",\"fieldName\":\"check_store_status\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[\"手机号码\"],\"enterpriseIds\":[\"ff8080817d9fbda8017dc20674f47fb6\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":1,\"tableId\":\"extract_integral_use_detail\",\"taskId\":\"1675670752773\"}";
String json="{\"allFields\":[\"is_mbr\",\"main_store_name\",\"main_store_code\",\"card_num\",\"open_card_time\",\"clerk_name\",\"add_clerk_code\",\"clerk_store_name\",\"clerk_store_code\",\"add_buddy_time\",\"create_time\"],\"amount\":1169963,\"authStoreIdList\":[],\"buildPermitted\":\"1\",\"conditions\":[{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"是否会员\",\"fieldName\":\"is_mbr\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"是\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店名称\",\"fieldName\":\"main_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员服务门店code\",\"fieldName\":\"main_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"会员卡号\",\"fieldName\":\"card_num\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"开卡时间\",\"fieldName\":\"open_card_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购姓名\",\"fieldName\":\"clerk_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购code\",\"fieldName\":\"add_clerk_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"导购门店名称\",\"fieldName\":\"clerk_store_name\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好友导购所属门店code\",\"fieldName\":\"clerk_store_code\",\"fieldType\":\"text\",\"filterTunnel\":\"\",\"mainFilter\":\"\"},{\"enableEncrypt\":false,\"extendFilter\":\"2023-02-28 23:59:59\",\"fieldMark\":\"添加好友时间\",\"fieldName\":\"add_buddy_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"2010-03-01 00:00:00\"},{\"enableEncrypt\":false,\"extendFilter\":\"\",\"fieldMark\":\"好办导购与客户建立好友关系时间\",\"fieldName\":\"create_time\",\"fieldType\":\"date\",\"filterTunnel\":\"\",\"mainFilter\":\"\"}],\"decryptFilters\":[],\"enterpriseIds\":[\"ff80808180b3c54a0180bc3df3bb4bca\"],\"execDistinct\":false,\"orderDir\":\"\",\"orderField\":\"\",\"queryDataType\":0,\"tableId\":\"extract_entwechat_detail_d\",\"taskId\":\"1678674610424\"}";
FlatQueryTaskCondition condition = JSON.parseObject(json, FlatQueryTaskCondition.class);
String fullQuery = flatQueryResultService.buildFlatQuerySQL(
false, // 下载用途
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment