Skip to content

G
gic-platform-enterprise
Commit cecd0a32 by 陶光胜
Options

报表中心

parent b2901a2e
Showing with 251 additions and 57 deletions
gic-platform-enterprise-api/src/main/java/com/gic/enterprise/constant/FieldFilterOptEnum.java 0 → 100644
package com.gic.enterprise.constant;
public enum FieldFilterOptEnum {
/**等于*/
EQ("=", "等于"),
/**包含**/
IN("in", "包含"),
/**包含**/
NOTIN("not in", "包含"),
/**大于等于**/
GTE(">=", "大于等于"),
/**小于等于**/
LTE("<=", "小于等于"),
/**大于**/
GT(">", "大于"),
/**小于**/
LT("<", "小于");
private String opt;
private String message;
FieldFilterOptEnum(String opt, String message){
this.opt = opt;
this.message = message;
}
public String getOpt() {
return opt;
}
public void setOpt(String opt) {
this.opt = opt;
}
public String getMessage() {
return message;
}
public void setMessage(String message) {
this.message = message;
}
}
gic-platform-enterprise-api/src/main/java/com/gic/enterprise/dto/HiveDataDownloadDTO.java
......@@ -8,8 +8,10 @@ import java.util.List;
public class HiveDataDownloadDTO implements Serializable {
private Integer enterpriseId;
/***hive查询sql**/
private String sql;
/**表名**/
private String tableName;
/**查询条件**/
private List<HiveQueryCondition> queryConditionList;
/**申请人用户id*/
private Integer applyUserId;
/**数据源（频道页），用 "-" 隔开上级和本级**/
......@@ -38,14 +40,6 @@ public class HiveDataDownloadDTO implements Serializable {
this.enterpriseId = enterpriseId;
}
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
public Integer getApplyUserId() {
return applyUserId;
}
......@@ -117,4 +111,20 @@ public class HiveDataDownloadDTO implements Serializable {
public void setExcelExtension(Integer excelExtension) {
this.excelExtension = excelExtension;
}
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public List<HiveQueryCondition> getQueryConditionList() {
return queryConditionList;
}
public void setQueryConditionList(List<HiveQueryCondition> queryConditionList) {
this.queryConditionList = queryConditionList;
}
}
gic-platform-enterprise-api/src/main/java/com/gic/enterprise/dto/HiveQueryCondition.java 0 → 100644
package com.gic.enterprise.dto;
import com.gic.enterprise.constant.FieldFilterOptEnum;
import com.gic.enterprise.constant.FieldTypeEnum;
import java.io.Serializable;
import java.util.List;
public class HiveQueryCondition implements Serializable {
/**字段名 enterprise_id*/
private String fieldName;
/**字段中文名 企业id**/
private String fieldMark;
/**字段类型**/
private FieldTypeEnum fieldTypeEnum;
/**需要过滤的value值**/
private List<?> filterValue;
/**查询条件操作符**/
private FieldFilterOptEnum fieldFilterOptEnum;
public String getFieldName() {
return fieldName;
}
public void setFieldName(String fieldName) {
this.fieldName = fieldName;
}
public String getFieldMark() {
return fieldMark;
}
public void setFieldMark(String fieldMark) {
this.fieldMark = fieldMark;
}
public FieldTypeEnum getFieldTypeEnum() {
return fieldTypeEnum;
}
public void setFieldTypeEnum(FieldTypeEnum fieldTypeEnum) {
this.fieldTypeEnum = fieldTypeEnum;
}
public List<?> getFilterValue() {
return filterValue;
}
public void setFilterValue(List<?> filterValue) {
this.filterValue = filterValue;
}
public FieldFilterOptEnum getFieldFilterOptEnum() {
return fieldFilterOptEnum;
}
public void setFieldFilterOptEnum(FieldFilterOptEnum fieldFilterOptEnum) {
this.fieldFilterOptEnum = fieldFilterOptEnum;
}
}
gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/outer/impl/HiveDataDownloadApiServiceImpl.java
......@@ -3,12 +3,12 @@ import com.alibaba.druid.sql.SQLUtils;
import com.alibaba.druid.sql.ast.SQLExpr;
import com.gic.api.base.commons.ServiceResponse;
import com.gic.commons.util.EntityUtil;
import com.gic.enterprise.constant.DownloadReportDataTypeEnum;
import com.gic.enterprise.constant.DownloadReportStatusEnum;
import com.gic.enterprise.constant.ExcelExtensionEnum;
import com.gic.enterprise.constant.*;
import com.gic.enterprise.dto.DownloadReportDTO;
import com.gic.enterprise.dto.HiveDataDownloadDTO;
import com.gic.enterprise.dto.HiveQueryCondition;
import com.gic.enterprise.entity.TabDownloadReport;
import com.gic.enterprise.error.ErrorCode;
import com.gic.enterprise.pojo.DownloadTaskPOJO;
import com.gic.enterprise.service.DownloadReportApiService;
import com.gic.enterprise.service.DownloadReportService;
......@@ -29,11 +29,9 @@ import javax.annotation.PostConstruct;
import java.io.File;
import java.sql.*;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.*;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
......@@ -76,52 +74,37 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
public ServiceResponse<Integer> createDownloadTask(HiveDataDownloadDTO hiveDataDownloadDTO) {
DownloadReportDTO downloadReportDTO = EntityUtil.changeEntityByJSON(DownloadReportDTO.class, hiveDataDownloadDTO);
downloadReportDTO.setDataCount(0);
String sql = "";
try{
sql = getSql(hiveDataDownloadDTO);
downloadReportDTO.setHiveSql(sql);
}catch (Exception e){
log.warn(e);
return ServiceResponse.failure(ErrorCode.SYSTEM_ERROR.getCode(), e.getMessage());
}
downloadReportDTO.setDataSource("hive");
ServiceResponse<Integer> response = downloadReportApiService.saveDownloadReport(downloadReportDTO);
if(!response.isSuccess()){
return response;
}
buildTask(response.getResult(), hiveDataDownloadDTO);
buildTask(response.getResult(), hiveDataDownloadDTO, sql);
return response;
}
private void buildTask(Integer reportId, HiveDataDownloadDTO hiveDataDownloadDTO){
private void buildTask(Integer reportId, HiveDataDownloadDTO hiveDataDownloadDTO, String sql){
DownloadTaskPOJO downloadTaskPOJO = new DownloadTaskPOJO();
downloadTaskPOJO.setReportId(reportId);
downloadTaskPOJO.setHiveSql(hiveDataDownloadDTO.getSql());
downloadTaskPOJO.setHiveSql(sql);
downloadTaskPOJO.setCanDownload(!DownloadReportDataTypeEnum.COMPLETE_DATA.getCode().equals(hiveDataDownloadDTO.getDataType()));
downloadTaskPOJO.setDesenField(hiveDataDownloadDTO.getDesenField());
downloadTaskPOJO.setExcelExtension(hiveDataDownloadDTO.getExcelExtension());
downloadTaskPOJO.setTableName(getTableName(hiveDataDownloadDTO.getSql()));
downloadTaskPOJO.setTableName(hiveDataDownloadDTO.getTableName());
downloadTaskPOJO.setFileName(hiveDataDownloadDTO.getFileName());
downloadTaskPOJO.setExcelExtension(hiveDataDownloadDTO.getExcelExtension());
taskList.add(downloadTaskPOJO);
}
public static void main(String[] args){
Connection conn = HiveConnUtils.getConnection();
Statement stat = null;
try {
stat = conn.createStatement();
//stat.execute("REFRESH TABLE extract_order_information"); // 强制刷新表结构 防止数据出错
String s = SQLUtils.formatHive("SELECT order_id as '订单号' FROM extract_order_information");
stat.execute(s);
} catch (Exception e) {
log.warn("下载数据出现异常", e);
// 任务状态修改
} finally {
try {
if(stat != null) {
stat.close();
}
if(conn != null) {
conn.close();
}
} catch (SQLException e) {
log.warn("stat 关闭异常", e);
}
}
}
private void runDownloadTask(Integer interval) {
downloadService.scheduleAtFixedRate(new Runnable() {
......@@ -204,15 +187,84 @@ public class HiveDataDownloadApiServiceImpl implements HiveDataDownloadApiServic
}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
}
private String getTableName(String sql){
if(StringUtils.isBlank(sql)){
return null;
public String getSql(HiveDataDownloadDTO hiveDataDownloadDTO) throws Exception {
List<HiveQueryCondition> queryConditionList = hiveDataDownloadDTO.getQueryConditionList();
String tableName = hiveDataDownloadDTO.getTableName();
StringBuffer sql = new StringBuffer(" select ");
int index = 1;
for(HiveQueryCondition condition: queryConditionList){
index++;
sql.append(condition.getFieldName()).append(" as `").append(condition.getFieldMark()).append("`");
if(index < queryConditionList.size()){
sql.append(",");
}
}
if(sql.contains("where")){
return sql.substring(sql.toLowerCase().indexOf("from")+4, sql.toLowerCase().indexOf("where"));
} else {
return sql.substring(sql.toLowerCase().indexOf("from")+4);
sql.append(" from ").append(tableName);
sql.append(" where 1=1 ");
boolean hasEnterpriseIdFilter = false;
for(HiveQueryCondition condition: queryConditionList){
if(condition.getFieldFilterOptEnum() != null){
if(condition.getFieldName().equals("enterprise_id") || condition.getFieldName().equals("ent_id")){
hasEnterpriseIdFilter = true;
}
sql.append(" and ").append(condition.getFieldName()).append(" ").append(condition.getFieldFilterOptEnum().getOpt());
if(FieldFilterOptEnum.IN.equals(condition.getFieldFilterOptEnum()) ||
FieldFilterOptEnum.NOTIN.equals(condition.getFieldFilterOptEnum())){
sql.append("(");
if(FieldTypeEnum.DATE.equals(condition.getFieldTypeEnum()) || FieldTypeEnum.STRING.equals(condition.getFieldTypeEnum())){
sql.append("'").append(condition.getFilterValue().get(0)).append("'");
} else {
sql.append(condition.getFilterValue().get(0));
}
sql.append(")");
} else {
if(FieldTypeEnum.DATE.equals(condition.getFieldTypeEnum()) || FieldTypeEnum.STRING.equals(condition.getFieldTypeEnum())){
sql.append("'").append(condition.getFilterValue().get(0)).append("'");
} else {
sql.append(condition.getFilterValue().get(0));
}
}
}
}
log.info("sql:{}", sql.toString());
if(!hasEnterpriseIdFilter){
throw new Exception("查询条件中必须要有企业id");
}
return sql.toString();
}
public static void main(String[] args){
HiveDataDownloadDTO dataDownloadDTO = new HiveDataDownloadDTO();
dataDownloadDTO.setEnterpriseId(1166);
dataDownloadDTO.setApplyUserId(13805);
dataDownloadDTO.setDataContent("用户-用户列表");
dataDownloadDTO.setDataType(DownloadReportDataTypeEnum.DESENSITIZATION_DATA.getCode());
dataDownloadDTO.setDataUrl("https://four.gicdev.com/member-tag/member-tag/tag-val-edit?id=264137629747888139&name=111111");
//dataDownloadDTO.setDesenField();
dataDownloadDTO.setExcelExtension(ExcelExtensionEnum.CSV.getCode());
dataDownloadDTO.setFileName("boomhivetest");
//dataDownloadDTO.setSql("SELECT order_id AS `订单号` FROM extract_order_information;");
dataDownloadDTO.setTableName("extract_order_information");
List<HiveQueryCondition> list = new ArrayList<>();
HiveQueryCondition queryCondition = new HiveQueryCondition();
queryCondition.setFieldName("order_id");
queryCondition.setFieldMark("订单号");
list.add(queryCondition);
HiveQueryCondition queryCondition1 = new HiveQueryCondition();
queryCondition1.setFieldName("enterprise_id");
queryCondition1.setFieldMark("企业id");
queryCondition1.setFieldFilterOptEnum(FieldFilterOptEnum.EQ);
queryCondition1.setFieldTypeEnum(FieldTypeEnum.STRING);
queryCondition1.setFilterValue(Arrays.asList("ff8080816a36326c016a53380d8b5f52"));
list.add(queryCondition1);
dataDownloadDTO.setQueryConditionList(list);
try{
String sql = new HiveDataDownloadApiServiceImpl().getSql(dataDownloadDTO);
System.out.println("sql = " + sql);
}catch (Exception e){
e.printStackTrace();
}
}
}
gic-platform-enterprise-service/src/test/java/com/gic/enterprise/service/outer/HiveDataDownloadApiServiceImplTest.java
......@@ -5,6 +5,7 @@ import com.gic.api.base.commons.ServiceResponse;
import com.gic.enterprise.constant.DownloadReportDataTypeEnum;
import com.gic.enterprise.constant.ExcelExtensionEnum;
import com.gic.enterprise.dto.HiveDataDownloadDTO;
import com.gic.enterprise.dto.HiveQueryCondition;
import com.gic.enterprise.service.EnterpriseApiService;
import com.gic.enterprise.service.EnterpriseInitService;
import com.gic.enterprise.service.HiveDataDownloadApiService;
......@@ -13,6 +14,9 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
import java.util.ArrayList;
import java.util.List;
/**
* @author guojx
* @date 2019/7/16 6:31 PM
......@@ -30,10 +34,15 @@ public class HiveDataDownloadApiServiceImplTest extends AbstractJUnit4SpringCont
dataDownloadDTO.setDataContent("用户-用户列表");
dataDownloadDTO.setDataType(DownloadReportDataTypeEnum.COMPLETE_DATA.getCode());
dataDownloadDTO.setDataUrl("https://four.gicdev.com/member-tag/member-tag/tag-val-edit?id=264137629747888139&name=111111");
//dataDownloadDTO.setDesenField();
dataDownloadDTO.setExcelExtension(ExcelExtensionEnum.CSV.getCode());
dataDownloadDTO.setFileName("boom hive test");
dataDownloadDTO.setSql("select order_id as '订单号' from extract_order_information");
dataDownloadDTO.setFileName("boomhivetest");
//dataDownloadDTO.setSql("SELECT order_id AS `订单号` FROM extract_order_information;");
dataDownloadDTO.setTableName("extract_order_information");
List<HiveQueryCondition> list = new ArrayList<>();
HiveQueryCondition queryCondition = new HiveQueryCondition();
queryCondition.setFieldName("order_id");
queryCondition.setFieldMark("订单号");
list.add(queryCondition);
dataDownloadDTO.setQueryConditionList(list);
ServiceResponse<Integer> downloadTask = hiveDataDownloadApiService.createDownloadTask(dataDownloadDTO);
System.out.println("downloadTask = " + JSON.toJSONString(downloadTask));
}
......
gic-platform-enterprise-web/src/main/java/com/gic/enterprise/web/controller/DownloadReportController.java
......@@ -9,8 +9,11 @@ import com.gic.download.utils.log.LogUtils;
import com.gic.enterprise.ano.IgnoreLogin;
import com.gic.enterprise.constant.DownloadReportDataTypeEnum;
import com.gic.enterprise.constant.ExcelExtensionEnum;
import com.gic.enterprise.constant.FieldFilterOptEnum;
import com.gic.enterprise.constant.FieldTypeEnum;
import com.gic.enterprise.dto.DownloadReportDTO;
import com.gic.enterprise.dto.HiveDataDownloadDTO;
import com.gic.enterprise.dto.HiveQueryCondition;
import com.gic.enterprise.qo.DownloadReportQO;
import com.gic.enterprise.response.EnterpriseRestResponse;
import com.gic.enterprise.service.DownloadReportApiService;
......@@ -25,6 +28,10 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
*
* @Description:
......@@ -95,7 +102,21 @@ public class DownloadReportController {
//dataDownloadDTO.setDesenField();
dataDownloadDTO.setExcelExtension(ExcelExtensionEnum.CSV.getCode());
dataDownloadDTO.setFileName("boomhivetest");
dataDownloadDTO.setSql("SELECT order_id AS `订单号` FROM extract_order_information;");
//dataDownloadDTO.setSql("SELECT order_id AS `订单号` FROM extract_order_information;");
dataDownloadDTO.setTableName("extract_order_information");
List<HiveQueryCondition> list = new ArrayList<>();
HiveQueryCondition queryCondition = new HiveQueryCondition();
queryCondition.setFieldName("order_id");
queryCondition.setFieldMark("订单号");
list.add(queryCondition);
HiveQueryCondition queryCondition1 = new HiveQueryCondition();
queryCondition1.setFieldName("enterprise_id");
queryCondition1.setFieldMark("企业id");
queryCondition1.setFieldFilterOptEnum(FieldFilterOptEnum.EQ);
queryCondition1.setFieldTypeEnum(FieldTypeEnum.STRING);
queryCondition1.setFilterValue(Arrays.asList("ff8080816a36326c016a53380d8b5f52"));
list.add(queryCondition1);
dataDownloadDTO.setQueryConditionList(list);
ServiceResponse<Integer> downloadTask = hiveDataDownloadApiService.createDownloadTask(dataDownloadDTO);
if(downloadTask.isSuccess()){
return RestResponse.success();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment