Skip to content

G
gic-cloud
Commit 7108582d by 陶光胜
Options

Merge branch 'developer' into 'master' 

Developer

See merge request !24
parents f907ab13 6afec56c
Showing with 178 additions and 28 deletions
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/entity/FlatQueryTaskCondition.java
......@@ -108,6 +108,16 @@ public class FlatQueryTaskCondition {
return results;
}
public List<String> getAllFields(){
List<String> results = Lists.newArrayList();
if (this.conditions != null && this.conditions.size() > 0) {
for (FlatQueryCondition condition : this.conditions) {
results.add(condition.getFieldName());
} // FOR OVER
} // IF OVER
return results;
}
/** 排序字段指定 */
protected String orderField = "";
......
gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/service/IDownloadTaskService.java
......@@ -67,4 +67,6 @@ public interface IDownloadTaskService {
List<DownloadRecord> listUnDownloadTask(String source);
DownloadProcessDTO getDownloadProcess();
void checkTaskStatus(String param);
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DecryptUtils.java
package com.gic.cloud.data.hook.service;
import com.alibaba.dubbo.common.utils.StringUtils;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.service.impl.FreeQueryServiceImpl;
import com.gic.data.shield.SdkEnv;
import com.gic.data.shield.decrypt.DataDecryptionClient;
import com.gic.data.shield.exception.DecryptException;
import org.springframework.stereotype.Service;
import java.util.List;
public class DecryptUtils {
private static DataDecryptionClient decryptionClient = null;
......@@ -40,5 +45,27 @@ public class DecryptUtils {
}
}
public static String dataSecurityProcessUserName(String userName){
if(StringUtils.isBlank(userName)){
return userName;
}
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append(userName.charAt(0)).append("**");
return stringBuffer.toString();
}
public static boolean isName(String columnName, List<FlatQueryCondition> allFields){
if(StringUtils.isBlank(columnName)){
return false;
}
for(FlatQueryCondition condition : allFields){
if(condition.getFieldMark().equals(columnName)){
if(FreeQueryServiceImpl.FILETERS_USER_NAME.contains(condition.getFieldName())){
return true;
}
}
}
return false;
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/dao/DownloadRecordDao.java
......@@ -30,4 +30,6 @@ public interface DownloadRecordDao {
@Param("fuzzyRange") String fuzzyRange, @Param("fuzzy") String fuzzy);
List<DownloadRecord> listUnDownloadTask(@Param("source") String source);
List<DownloadRecord> listExpireTask();
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/dao/DownloadTaskDao.java
......@@ -64,4 +64,6 @@ public interface DownloadTaskDao {
DownloadProcessDTO getDownloadProcess();
int updateTaskStatusError(@Param("idList") List<String> idList);
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/entity/CsvResultSetHelper.java
package com.gic.cloud.data.hook.service.entity;
import com.alibaba.dubbo.common.utils.StringUtils;
import com.alibaba.fastjson.JSON;
import com.gic.cloud.data.hook.api.entity.FlatQueryCondition;
import com.gic.cloud.data.hook.service.DecryptUtils;
import com.gic.cloud.data.hook.service.impl.FreeQueryServiceImpl;
import com.opencsv.ResultSetHelper;
import java.io.IOException;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Types;
import java.text.SimpleDateFormat;
......@@ -19,9 +24,12 @@ public class CsvResultSetHelper implements ResultSetHelper {
/** 数据过滤用关键字集合 */
private List<String> filters = null;
public CsvResultSetHelper(String filterMode, List<String> filters) {
private List<FlatQueryCondition> allFields = null;
public CsvResultSetHelper(String filterMode, List<String> filters, List<FlatQueryCondition> allFields) {
this.filterMode = filterMode;
this.filters = filters;
this.allFields = allFields;
}
@Override
......@@ -68,26 +76,38 @@ public class CsvResultSetHelper implements ResultSetHelper {
boolean doDecrypt = false; // 是否进行解密
if (this.filterMode.equals(CsvDataFilterMode.DESENSI)) { // 如果需要脱敏
if (this.filters != null && this.filters.size() > 0) {
for (String filter : this.filters) {
if (columnName.contains(filter)) {
if(filters.contains(columnName)){
doDesensi = true;
break;
} // IF OVER
} // FOR OVER
} // IF OVER
}
}
if(allFields != null && isName(columnName)){
doDesensi = true;
}
if(allFields == null && FreeQueryServiceImpl.FILETERS_USER_NAME.contains(columnName)){
doDesensi = true;
}
} else if (this.filterMode.equals(CsvDataFilterMode.DECRYPT)) { // 如果需要解密
if (this.filters != null && this.filters.size() > 0) {
for (String filter : this.filters) {
if (columnName.equals(filter)) {
if(filters.contains(columnName)){
doDecrypt = true;
break;
} // IF OVER
} // FOR OVER
} // IF OVER
}
}
if(allFields != null && isName(columnName)){
doDecrypt = true;
}
if(allFields == null && FreeQueryServiceImpl.FILETERS_USER_NAME.contains(columnName)){
doDecrypt = true;
}
}
// 数据处理
if (doDesensi) { // 如果需要脱敏处理
if(allFields != null && isName(columnName)){
result[i] = DecryptUtils.dataSecurityProcessUserName(DecryptUtils.decrypt(resultSet.getString(columnName)));
} else if(allFields == null && FreeQueryServiceImpl.FILETERS_USER_NAME.contains(columnName)) {
result[i] = DecryptUtils.dataSecurityProcessUserName(resultSet.getString(columnName));
} else {
result[i] = "******";
}
} else if (doDecrypt) { // 如果需要解密处理
//System.out.println("CSV 解密字段名 " + columnName);
String tmpResult = resultSet.getString(columnName);
......@@ -118,4 +138,18 @@ public class CsvResultSetHelper implements ResultSetHelper {
} // IF OVER
return result;
}
private boolean isName(String columnName){
if(StringUtils.isBlank(columnName)){
return false;
}
for(FlatQueryCondition condition : allFields){
if(condition.getFieldMark().equals(columnName)){
if(FreeQueryServiceImpl.FILETERS_USER_NAME.contains(condition.getFieldName())){
return true;
}
}
}
return false;
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/DownloadTaskServiceImpl.java
......@@ -15,6 +15,7 @@ import com.gic.cloud.data.hook.service.dao.RiskModeRecordDao;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.opencsv.CSVWriter;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.ibatis.annotations.Param;
......@@ -89,6 +90,7 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
@Override
public Page<DownloadTask> getDownloadTaskPage(String userId, String fuzzy, Integer pageNum, Integer pageSize) {
checkTaskStatus("");
Page<DownloadTask> result = new Page<>();
PageHelper.startPage(pageNum, pageSize); // 设置分页参数
List<DownloadTask> query = downloadTaskDao.getDownloadTaskListByUserId(userId, fuzzy);
......@@ -260,5 +262,21 @@ public class DownloadTaskServiceImpl implements IDownloadTaskService {
return downloadProcess;
}
@Override
public void checkTaskStatus(String param) {
System.out.println("执行定时任务");
List<DownloadRecord> downloadRecordList = this.downloadRecordDao.listExpireTask();
List<String> idList = new ArrayList<>();
if(CollectionUtils.isNotEmpty(downloadRecordList)){
for(DownloadRecord downloadRecord : downloadRecordList){
idList.add(downloadRecord.getId());
}
}
if(CollectionUtils.isNotEmpty(idList)){
//设置超时任务的状态为error
this.downloadTaskDao.updateTaskStatusError(idList);
}
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
......@@ -271,22 +271,26 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
} else fieldResult = "";
} else fieldResult = "******";
} else fieldResult = rs.getObject(fieldColumnIndex);**/
if (FreeQueryServiceImpl.FILTERS_PHONE_ONLY.contains(fieldName) || FreeQueryServiceImpl.FILTERS_PHONE_AND_CARD.contains(fieldName)) {
if(queryDataType == QueryDataType.FULL){
String preResult = rs.getString(fieldColumnIndex);
if(dataPermission == 1 && FreeQueryServiceImpl.FILTERS_PHONE_ONLY.contains(fieldName)){
fieldResult = DecryptUtils.decrypt(preResult);
}else if(dataPermission == 2 && FreeQueryServiceImpl.FILTERS_PHONE_AND_CARD.contains(fieldName)){
} else if(dataPermission == 2 && FreeQueryServiceImpl.FILTERS_PHONE_AND_CARD.contains(fieldName)){
fieldResult = DecryptUtils.decrypt(preResult);
} else if (FreeQueryServiceImpl.FILETERS_USER_NAME.contains(fieldName)){
fieldResult = DecryptUtils.decrypt(preResult);
}else fieldResult = rs.getObject(fieldColumnIndex);
} else fieldResult = rs.getObject(fieldColumnIndex);
} else {
if(dataPermission == 1 && FreeQueryServiceImpl.FILTERS_PHONE_ONLY.contains(fieldName)){
fieldResult = "******";
}else if(dataPermission == 2 && FreeQueryServiceImpl.FILTERS_PHONE_AND_CARD.contains(fieldName)){
} else if(dataPermission == 2 && FreeQueryServiceImpl.FILTERS_PHONE_AND_CARD.contains(fieldName)){
fieldResult = "******";
} else if(FreeQueryServiceImpl.FILETERS_USER_NAME.contains(fieldName)){
fieldResult = DecryptUtils.dataSecurityProcessUserName(DecryptUtils.decrypt(rs.getObject(fieldColumnIndex)+""));
} else fieldResult = rs.getObject(fieldColumnIndex);
}
} else fieldResult = rs.getObject(fieldColumnIndex);
} // SWITCH OVER
result.add(fieldName, fieldResult);
// } // IF ELSE OVER
......@@ -576,7 +580,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}
//CSVWriter csvWriter = new CSVWriter(new FileWriter(csvPath), '\t');
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(originalFilePath), Charset.forName("GBK"));
ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.DECRYPT : CsvDataFilterMode.DESENSI, condition.getDecryptFilters());
ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.DECRYPT : CsvDataFilterMode.DESENSI, condition.getDecryptFilters(), condition.getConditions());
CSVWriter writer = new CSVWriter(out, ',');
writer.setResultService(helper);
writer.writeAll(rs, true);
......@@ -605,7 +609,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
//row.createCell(j).setCellValue(c);
String cName = rs.getMetaData().getColumnName(j+1);
List<String> cFilters = condition.getDecryptFilters();
if (task.getQueryDataType() == QueryDataType.FULL && cFilters.contains(cName)) {
if (task.getQueryDataType() == QueryDataType.FULL && (cFilters.contains(cName) || DecryptUtils.isName(cName, condition.getConditions()))) {
String tmpResult = rs.getString(j + 1);
if (StringUtils.isNotBlank(tmpResult)) tmpResult = DecryptUtils.getInstance().decrypt(tmpResult);
row.createCell(j).setCellValue(tmpResult);
......@@ -624,7 +628,9 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
default:
if(cFilters.contains(cName)){
row.createCell(j).setCellValue("******");
}else {
} else if(DecryptUtils.isName(cName, condition.getConditions())){
row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(DecryptUtils.decrypt(rs.getString(j + 1))));
} else {
row.createCell(j).setCellValue(rs.getString(j + 1));
}
break;
......@@ -765,7 +771,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
}
//CSVWriter csvWriter = new CSVWriter(new FileWriter(csvPath), '\t');
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(originalFilePath), Charset.forName("GBK"));
ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.DECRYPT : CsvDataFilterMode.DESENSI, condition.getDecryptFilters());
ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.DECRYPT : CsvDataFilterMode.DESENSI, condition.getDecryptFilters(), condition.getConditions());
CSVWriter writer = new CSVWriter(out, ',');
writer.setResultService(helper);
writer.writeAll(rs, true);
......@@ -794,7 +800,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
//row.createCell(j).setCellValue(c);
String cName = rs.getMetaData().getColumnName(j+1);
List<String> cFilters = condition.getDecryptFilters();
if (task.getQueryDataType() == QueryDataType.FULL && cFilters.contains(cName)) {
if (task.getQueryDataType() == QueryDataType.FULL && (cFilters.contains(cName) || DecryptUtils.isName(cName, condition.getConditions()))) {
String tmpResult = rs.getString(j + 1);
if (StringUtils.isNotBlank(tmpResult)) tmpResult = DecryptUtils.getInstance().decrypt(tmpResult);
row.createCell(j).setCellValue(tmpResult);
......@@ -813,6 +819,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
default:
if(cFilters.contains(cName)){
row.createCell(j).setCellValue("******");
}else if(DecryptUtils.isName(cName, condition.getConditions())){
row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(DecryptUtils.decrypt(rs.getString(j + 1))));
}else {
row.createCell(j).setCellValue(rs.getString(j + 1));
}
......@@ -903,6 +911,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
new BasicThreadFactory.Builder().namingPattern("applyTimer-%d").daemon(true).build());
public void runApplyTask(Integer interval){
Config config = ConfigService.getConfig("application");
final String url = config.getProperty("applyUrl", "");
applyService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
......@@ -912,7 +922,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
//log.debug("自助指标待审批任务状态变更", JSON.toJSONString(task));
if (!task.getApplyId().equals("")) {
// String applyStatusText = HttpUtils.doGet("http://www.gicdev.com/api-admin/apply-info?type=2&applyId=" + task.getApplyId());
String applyStatusText = HttpUtils.doGet("http://hope.demogic.com/api-admin/apply-info?type=2&applyId=" + task.getApplyId());
String applyStatusText = HttpUtils.doGet(url + task.getApplyId());
log.debug("runApplyTask.run", "查询自助指标查询 " + task.getId() + " 审核状态码" + applyStatusText);
if(StringUtils.isBlank(applyStatusText)){
break;
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
......@@ -2,6 +2,8 @@ package com.gic.cloud.data.hook.service.impl;
import cn.medubi.client.utils.LogPak;
import com.alibaba.fastjson.JSON;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.gic.cloud.data.hook.api.dto.*;
import com.gic.cloud.data.hook.api.dto.DataDesensiType;
import com.gic.cloud.data.hook.api.entity.*;
......@@ -56,6 +58,9 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
/** 脱敏字段 */
public static final List<String> FILTERS_PHONE_AND_CARD = Arrays.asList("card_num", "mobile", "phone", "enterprise_name", "phone_number", "receive_phone_number","receive_card_num","use_phone_number", "use_card_num","friend_card_num","from_card_num", "friend_phone_num","from_phone_num");
public static final List<String> FILETERS_USER_NAME = Arrays.asList("member_name","children_name","mbr_name","receive_member_name","use_member_name","name","bb_name","friend_mbr_name","from_mbr_name");
@Autowired
IDownloadTaskService downloadTaskService;
......@@ -229,8 +234,10 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
for (int i = 0; i < metaData.getColumnCount(); i++) {
String fieldName = metaData.getColumnLabel(i + 1);
Object filedValue;
if (queryDataType == QueryDataType.SAFE && FreeQueryServiceImpl.isFilterFields(desensiType, fieldName)) {
if (queryDataType == QueryDataType.SAFE && FreeQueryServiceImpl.isFilterFields(desensiType, fieldName)) {//手机号和卡号
filedValue = "******";
}else if(queryDataType == QueryDataType.SAFE && FILETERS_USER_NAME.contains(fieldName)){//用户名
filedValue = DecryptUtils.dataSecurityProcessUserName(rs.getString(fieldName));
} else {
switch (metaData.getColumnType(i + 1)) {
case Types.TIMESTAMP:
......@@ -405,7 +412,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
//CSVWriter csvWriter = new CSVWriter(new FileWriter(csvPath), '\t');
OutputStreamWriter out = new OutputStreamWriter(new FileOutputStream(originalFilePath), Charset.forName("GBK"));
ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.NONE : CsvDataFilterMode.DESENSI, FreeQueryServiceImpl.getFieldsFilters(condition.getDesensiType()));
ResultSetHelper helper = new CsvResultSetHelper(task.getQueryDataType() == QueryDataType.FULL ? CsvDataFilterMode.NONE : CsvDataFilterMode.DESENSI, FreeQueryServiceImpl.getFieldsFilters(condition.getDesensiType()), null);
CSVWriter writer = new CSVWriter(out, ',');
writer.setResultService(helper);
writer.writeAll(rs, true);
......@@ -434,8 +441,12 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
//row.createCell(j).setCellValue(c);
String cName = rs.getMetaData().getColumnName(j+1);
if (task.getQueryDataType() == QueryDataType.SAFE
&& FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName)) {
&& (FreeQueryServiceImpl.isFilterFields(condition.getDesensiType(), cName) || FreeQueryServiceImpl.FILETERS_USER_NAME.contains(cName))) {
if(FreeQueryServiceImpl.FILETERS_USER_NAME.contains(cName)){
row.createCell(j).setCellValue(DecryptUtils.dataSecurityProcessUserName(rs.getString(cName)));
} else {
row.createCell(j).setCellValue("******");
}
} else {
int cType = rs.getMetaData().getColumnType(j + 1);
switch (cType) {
......@@ -536,6 +547,8 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
* @param interval
*/
private void runApplyTask(Integer interval) {
Config config = ConfigService.getConfig("application");
final String url = config.getProperty("applyUrl", "");
this.freeApplyService.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
......@@ -545,7 +558,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
log.debug("自定义查询任务审批状态变更：", JSON.toJSONString(task));
if (!task.getApplyId().equals("")) {
// String applyStatusText = HttpUtils.doGet("http://www.gicdev.com/api-admin/apply-info?type=2&applyId=" + task.getApplyId());
String applyStatusText = HttpUtils.doGet("http://hope.demogic.com/api-admin/apply-info?type=2&applyId=" + task.getApplyId());
String applyStatusText = HttpUtils.doGet(url + task.getApplyId());
log.debug("runApplyTask.run", "查询自助指标查询 " + task.getId() + " 审核状态码" + applyStatusText);
if(StringUtils.isBlank(applyStatusText)){
break;
......
gic-cloud-data-hook-service/src/main/resources/mapper/DownloadRecordDao.xml
......@@ -117,4 +117,23 @@
and start_time &lt;= CONCAT(DATE_FORMAT(NOW(),'%Y-%m-%d'),' 23:59:59') order by start_time
</select>
<select id="listExpireTask" resultType="DownloadRecord">
SELECT
id,
enterprise_id,
name AS "task_name",
module_name,
amount,
query_data_type,
real_sql,
report_id,
desensi_type,
download_condition,
apply_permitted,
apply_status
FROM
dh_download_task
WHERE (status = 'building' or status = 'waiting')
and start_time &lt;= DATE_ADD(now(),INTERVAL -24 HOUR) order by start_time
</select>
</mapper>
\ No newline at end of file
gic-cloud-data-hook-service/src/main/resources/mapper/DownloadTaskDao.xml
......@@ -192,4 +192,15 @@
(status = 'building' or status = 'waiting') and start_time &gt;= CONCAT(DATE_FORMAT(NOW(),'%Y-%m-%d'),' 00:00:00')
and start_time &lt;= CONCAT(DATE_FORMAT(NOW(),'%Y-%m-%d'),' 23:59:59')
</select>
<update id="updateTaskStatusError">
UPDATE <include refid="updateTable"/> SET
status = 'error'
WHERE
id in
<foreach collection="idList" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</update>
</mapper>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment