Merge remote-tracking branch 'origin/developer' into developer

gic-platform-enterprise-api/src/main/java/com/gic/enterprise/constant/FieldFilterOptEnum.java

+package com.gic.enterprise.constant;
+
+public enum FieldFilterOptEnum {
+    /**等于*/
+    EQ("=", "等于"),
+    /**包含**/
+    IN("in", "包含"),
+    /**包含**/
+    NOTIN("not in", "包含"),
+    /**大于等于**/
+    GTE(">=", "大于等于"),
+    /**小于等于**/
+    LTE("<=", "小于等于"),
+    /**大于**/
+    GT(">", "大于"),
+    /**小于**/
+    LT("<", "小于");
+
+    private String opt;
+    private String message;
+
+    FieldFilterOptEnum(String opt, String message){
+        this.opt = opt;
+        this.message = message;
+    }
+
+    public String getOpt() {
+        return opt;
+    }
+
+    public void setOpt(String opt) {
+        this.opt = opt;
+    }
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+}

gic-platform-enterprise-api/src/main/java/com/gic/enterprise/dto/HiveDataDownloadDTO.java

@@ -8,8 +8,10 @@ import java.util.List;
 
 public class HiveDataDownloadDTO implements Serializable {
     private Integer enterpriseId;
-    /***hive查询sql**/
-    private String sql;
+    /**表名**/
+    private String tableName;
+    /**查询条件**/
+    private List<HiveQueryCondition> queryConditionList;
     /**申请人用户id*/
     private Integer applyUserId;
     /**数据源（频道页），用  "-" 隔开上级和本级**/
@@ -38,14 +40,6 @@ public class HiveDataDownloadDTO implements Serializable {
         this.enterpriseId = enterpriseId;
     }
 
-    public String getSql() {
-        return sql;
-    }
-
-    public void setSql(String sql) {
-        this.sql = sql;
-    }
-
     public Integer getApplyUserId() {
         return applyUserId;
     }
@@ -117,4 +111,20 @@ public class HiveDataDownloadDTO implements Serializable {
     public void setExcelExtension(Integer excelExtension) {
         this.excelExtension = excelExtension;
     }
+
+    public String getTableName() {
+        return tableName;
+    }
+
+    public void setTableName(String tableName) {
+        this.tableName = tableName;
+    }
+
+    public List<HiveQueryCondition> getQueryConditionList() {
+        return queryConditionList;
+    }
+
+    public void setQueryConditionList(List<HiveQueryCondition> queryConditionList) {
+        this.queryConditionList = queryConditionList;
+    }
 }

gic-platform-enterprise-api/src/main/java/com/gic/enterprise/dto/HiveQueryCondition.java

+package com.gic.enterprise.dto;
+
+import com.gic.enterprise.constant.FieldFilterOptEnum;
+import com.gic.enterprise.constant.FieldTypeEnum;
+
+import java.io.Serializable;
+import java.util.List;
+
+public class HiveQueryCondition implements Serializable {
+    /**字段名 enterprise_id*/
+    private String fieldName;
+    /**字段中文名 企业id**/
+    private String fieldMark;
+    /**字段类型**/
+    private FieldTypeEnum fieldTypeEnum;
+    /**需要过滤的value值**/
+    private List<?> filterValue;
+    /**查询条件操作符**/
+    private FieldFilterOptEnum fieldFilterOptEnum;
+
+    public String getFieldName() {
+        return fieldName;
+    }
+
+    public void setFieldName(String fieldName) {
+        this.fieldName = fieldName;
+    }
+
+    public String getFieldMark() {
+        return fieldMark;
+    }
+
+    public void setFieldMark(String fieldMark) {
+        this.fieldMark = fieldMark;
+    }
+
+    public FieldTypeEnum getFieldTypeEnum() {
+        return fieldTypeEnum;
+    }
+
+    public void setFieldTypeEnum(FieldTypeEnum fieldTypeEnum) {
+        this.fieldTypeEnum = fieldTypeEnum;
+    }
+
+    public List<?> getFilterValue() {
+        return filterValue;
+    }
+
+    public void setFilterValue(List<?> filterValue) {
+        this.filterValue = filterValue;
+    }
+
+    public FieldFilterOptEnum getFieldFilterOptEnum() {
+        return fieldFilterOptEnum;
+    }
+
+    public void setFieldFilterOptEnum(FieldFilterOptEnum fieldFilterOptEnum) {
+        this.fieldFilterOptEnum = fieldFilterOptEnum;
+    }
+}

gic-platform-enterprise-service/src/main/java/com/gic/enterprise/pojo/DownloadTaskPOJO.java

@@ -5,7 +5,7 @@ import java.util.List;
 
 public class DownloadTaskPOJO {
     /***报告id***/
-    private String reportId;
+    private Integer reportId;
     /**表名**/
     private String tableName;
     /***hive查询语句***/
@@ -20,12 +20,11 @@ public class DownloadTaskPOJO {
     /**文件扩展 1.xlsx  2.xls  3.csv**/
     private Integer excelExtension;
 
-
-    public String getReportId() {
+    public Integer getReportId() {
         return reportId;
     }
 
-    public void setReportId(String reportId) {
+    public void setReportId(Integer reportId) {
         this.reportId = reportId;
     }
 

gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/DownloadReportService.java

@@ -98,7 +98,7 @@ public interface DownloadReportService {
      * @return void
      * @throws
      */
-    void updateDownloadStatus(String reportId, DownloadReportStatusEnum downloadReportStatusEnum);
+    void updateDownloadStatus(Integer reportId, DownloadReportStatusEnum downloadReportStatusEnum);
 
     /**
      * updateDataCount

gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/impl/DownloadReportServiceImpl.java

@@ -112,9 +112,9 @@ public class DownloadReportServiceImpl implements DownloadReportService {
     }
 
     @Override
-    public void updateDownloadStatus(String reportId, DownloadReportStatusEnum downloadReportStatusEnum) {
+    public void updateDownloadStatus(Integer downloadReportId, DownloadReportStatusEnum downloadReportStatusEnum) {
         TabDownloadReport downloadReport = new TabDownloadReport();
-        downloadReport.setReportId(reportId);
+        downloadReport.setDownloadReportId(downloadReportId);
         downloadReport.setStatus(downloadReportStatusEnum.getCode());
         tabDownloadReportMapper.updateByPrimaryKeySelective(downloadReport);
     }

gic-platform-enterprise-service/src/main/java/com/gic/enterprise/service/outer/impl/DownloadReportApiServiceImpl.java

@@ -25,6 +25,8 @@ import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.redisson.api.RMap;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -46,7 +48,7 @@ import java.util.concurrent.TimeUnit;
  */
 @Service("downloadReportApiService")
 public class DownloadReportApiServiceImpl implements DownloadReportApiService {
-
+    private static final Logger log = LogManager.getLogger(DownloadReportApiServiceImpl.class);
     @Autowired
     private DownloadReportService downloadReportService;
     @Autowired
@@ -71,21 +73,25 @@ public class DownloadReportApiServiceImpl implements DownloadReportApiService {
         checkReportHeart.scheduleAtFixedRate(new Runnable() {
             @Override
             public void run() {
-                RMap<Object, Object> map = RedisUtil.getRedisClient().getMap(reportKey);
-                if(MapUtils.isNotEmpty(map)){
-                    Iterator<Object> iterator = map.keySet().iterator();
-                    while (iterator.hasNext()){
-                        Object next = iterator.next();
-                        long putTime = (long) map.get(next);
-                        if((System.currentTimeMillis() - putTime) > 10*60*1000){
-                            //超过十分钟 如果报告还在生成中则设置为失败
-                            TabDownloadReport report = downloadReportService.getById((Integer) next);
-                            if(report != null && report.getStatus().equals(DownloadReportStatusEnum.BUILD.getCode())){
-                                downloadReportService.updateDownloadStatus(next.toString(), DownloadReportStatusEnum.ERROR);
+                try {
+                    RMap<Object, Object> map = RedisUtil.getRedisClient().getMap(reportKey);
+                    if(MapUtils.isNotEmpty(map)){
+                        Iterator<Object> iterator = map.keySet().iterator();
+                        while (iterator.hasNext()){
+                            Object next = iterator.next();
+                            long putTime = (long) map.get(next);
+                            if((System.currentTimeMillis() - putTime) > 10*60*1000){
+                                //超过十分钟 如果报告还在生成中则设置为失败
+                                TabDownloadReport report = downloadReportService.getById((Integer) next);
+                                if(report != null && report.getStatus().equals(DownloadReportStatusEnum.BUILD.getCode())){
+                                    downloadReportService.updateDownloadStatus((Integer)next, DownloadReportStatusEnum.ERROR);
+                                }
+                                map.remove(next);
                             }
-                            map.remove(next);
                         }
                     }
+                }catch (Throwable e){
+                    log.warn(e);
                 }
             }
         }, 3, 3, TimeUnit.SECONDS);

gic-platform-enterprise-service/src/main/java/com/gic/enterprise/utils/FileUtils.java

@@ -14,10 +14,7 @@ import org.apache.poi.ss.usermodel.Row;
 import org.apache.poi.ss.usermodel.Sheet;
 import org.apache.poi.xssf.streaming.SXSSFWorkbook;
 
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.OutputStreamWriter;
+import java.io.*;
 import java.nio.charset.Charset;
 import java.sql.ResultSet;
 import java.sql.Types;
@@ -102,21 +99,32 @@ public class FileUtils {
     }
 
     private static File createTmpFile(String path){
+        File dir = new File(SAVE_FOLDER);
+        if(!dir.exists()){
+            dir.mkdirs();
+        }
         File tmp = new File(path);
         if (tmp.exists()) { // 删除可能存在的文件
             tmp.delete();
         }
+        if(!tmp.exists()){
+            try {
+                tmp.createNewFile();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
         return tmp;
     }
 
     private static String getFilePath(String fileName, Integer fileExt){
         String originalFilePath = SAVE_FOLDER + File.separator + fileName ;
-        if(ExcelExtensionEnum.CSV.equals(fileExt)){
-            originalFilePath += ExcelExtensionEnum.CSV;
-        } else if(ExcelExtensionEnum.EXCEL_2003.equals(fileExt)){
-            originalFilePath += ExcelExtensionEnum.EXCEL_2003;
-        } else if(ExcelExtensionEnum.EXCEL_2007.equals(fileExt)){
-            originalFilePath += ExcelExtensionEnum.EXCEL_2007;
+        if(ExcelExtensionEnum.CSV.getCode() == fileExt){
+            originalFilePath += ExcelExtensionEnum.CSV.getExtension();
+        } else if(ExcelExtensionEnum.EXCEL_2003.getCode() == fileExt){
+            originalFilePath += ExcelExtensionEnum.EXCEL_2003.getExtension();
+        } else if(ExcelExtensionEnum.EXCEL_2007.getCode() == fileExt){
+            originalFilePath += ExcelExtensionEnum.EXCEL_2007.getExtension();
         }
         return originalFilePath;
     }

gic-platform-enterprise-service/src/test/java/com/gic/enterprise/service/outer/HiveDataDownloadApiServiceImplTest.java

@@ -5,6 +5,7 @@ import com.gic.api.base.commons.ServiceResponse;
 import com.gic.enterprise.constant.DownloadReportDataTypeEnum;
 import com.gic.enterprise.constant.ExcelExtensionEnum;
 import com.gic.enterprise.dto.HiveDataDownloadDTO;
+import com.gic.enterprise.dto.HiveQueryCondition;
 import com.gic.enterprise.service.EnterpriseApiService;
 import com.gic.enterprise.service.EnterpriseInitService;
 import com.gic.enterprise.service.HiveDataDownloadApiService;
@@ -13,6 +14,9 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.context.ContextConfiguration;
 import org.springframework.test.context.junit4.AbstractJUnit4SpringContextTests;
 
+import java.util.ArrayList;
+import java.util.List;
+
 /**
  * @author guojx
  * @date 2019/7/16 6:31 PM
@@ -30,10 +34,15 @@ public class HiveDataDownloadApiServiceImplTest extends AbstractJUnit4SpringCont
         dataDownloadDTO.setDataContent("用户-用户列表");
         dataDownloadDTO.setDataType(DownloadReportDataTypeEnum.COMPLETE_DATA.getCode());
         dataDownloadDTO.setDataUrl("https://four.gicdev.com/member-tag/member-tag/tag-val-edit?id=264137629747888139&name=111111");
-        //dataDownloadDTO.setDesenField();
-        dataDownloadDTO.setExcelExtension(ExcelExtensionEnum.CSV.getCode());
-        dataDownloadDTO.setFileName("boom hive test");
-        dataDownloadDTO.setSql("select order_id as '订单号' from extract_order_information");
+        dataDownloadDTO.setFileName("boomhivetest");
+        //dataDownloadDTO.setSql("SELECT order_id AS `订单号` FROM extract_order_information;");
+        dataDownloadDTO.setTableName("extract_order_information");
+        List<HiveQueryCondition> list = new ArrayList<>();
+        HiveQueryCondition queryCondition = new HiveQueryCondition();
+        queryCondition.setFieldName("order_id");
+        queryCondition.setFieldMark("订单号");
+        list.add(queryCondition);
+        dataDownloadDTO.setQueryConditionList(list);
         ServiceResponse<Integer> downloadTask = hiveDataDownloadApiService.createDownloadTask(dataDownloadDTO);
         System.out.println("downloadTask = " + JSON.toJSONString(downloadTask));
     }

gic-platform-enterprise-web/src/main/java/com/gic/enterprise/web/controller/DownloadReportController.java

 package com.gic.enterprise.web.controller;
 
+import com.alibaba.fastjson.JSON;
 import com.gic.api.base.commons.Page;
 import com.gic.api.base.commons.ServiceResponse;
+import com.gic.commons.annotation.HeaderSignIgnore;
 import com.gic.commons.webapi.reponse.RestResponse;
 import com.gic.download.utils.log.LogUtils;
 import com.gic.enterprise.ano.IgnoreLogin;
+import com.gic.enterprise.constant.DownloadReportDataTypeEnum;
+import com.gic.enterprise.constant.ExcelExtensionEnum;
+import com.gic.enterprise.constant.FieldFilterOptEnum;
+import com.gic.enterprise.constant.FieldTypeEnum;
 import com.gic.enterprise.dto.DownloadReportDTO;
+import com.gic.enterprise.dto.HiveDataDownloadDTO;
+import com.gic.enterprise.dto.HiveQueryCondition;
 import com.gic.enterprise.qo.DownloadReportQO;
+import com.gic.enterprise.response.EnterpriseRestResponse;
 import com.gic.enterprise.service.DownloadReportApiService;
+import com.gic.enterprise.service.HiveDataDownloadApiService;
 import com.gic.enterprise.utils.ResultControllerUtils;
 import com.gic.enterprise.utils.UserDetailUtils;
 import com.gic.enterprise.web.vo.DownloadReportVO;
@@ -18,6 +28,10 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.ResponseBody;
 import org.springframework.web.bind.annotation.RestController;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
 /**
  *
  * @Description:
@@ -29,6 +43,8 @@ public class DownloadReportController {
 
     @Autowired
     private DownloadReportApiService downloadReportApiService;
+    @Autowired
+    private HiveDataDownloadApiService hiveDataDownloadApiService;
 
     @RequestMapping("/list-download-report")
     public RestResponse listDownloadReport(@Validated({DownloadReportQO.SelectValidView.class}) DownloadReportQO downloadReportQO) {
@@ -66,8 +82,45 @@ public class DownloadReportController {
     @RequestMapping("receive-heart")
     @ResponseBody
     @IgnoreLogin
+    @HeaderSignIgnore
     public RestResponse receiveHeart(Integer reportId){
         ServiceResponse<Void> response = downloadReportApiService.receiveHeart(reportId);
         return RestResponse.success();
     }
+
+    @RequestMapping("download-hive-data")
+    @ResponseBody
+    @IgnoreLogin
+    @HeaderSignIgnore
+    public RestResponse downloadHiveData(){
+        HiveDataDownloadDTO dataDownloadDTO = new HiveDataDownloadDTO();
+        dataDownloadDTO.setEnterpriseId(1166);
+        dataDownloadDTO.setApplyUserId(13805);
+        dataDownloadDTO.setDataContent("用户-用户列表");
+        dataDownloadDTO.setDataType(DownloadReportDataTypeEnum.DESENSITIZATION_DATA.getCode());
+        dataDownloadDTO.setDataUrl("https://four.gicdev.com/member-tag/member-tag/tag-val-edit?id=264137629747888139&name=111111");
+        //dataDownloadDTO.setDesenField();
+        dataDownloadDTO.setExcelExtension(ExcelExtensionEnum.CSV.getCode());
+        dataDownloadDTO.setFileName("boomhivetest");
+        //dataDownloadDTO.setSql("SELECT order_id AS `订单号` FROM extract_order_information;");
+        dataDownloadDTO.setTableName("extract_order_information");
+        List<HiveQueryCondition> list = new ArrayList<>();
+        HiveQueryCondition queryCondition = new HiveQueryCondition();
+        queryCondition.setFieldName("order_id");
+        queryCondition.setFieldMark("订单号");
+        list.add(queryCondition);
+        HiveQueryCondition queryCondition1 = new HiveQueryCondition();
+        queryCondition1.setFieldName("enterprise_id");
+        queryCondition1.setFieldMark("企业id");
+        queryCondition1.setFieldFilterOptEnum(FieldFilterOptEnum.EQ);
+        queryCondition1.setFieldTypeEnum(FieldTypeEnum.STRING);
+        queryCondition1.setFilterValue(Arrays.asList("ff8080816a36326c016a53380d8b5f52"));
+        list.add(queryCondition1);
+        dataDownloadDTO.setQueryConditionList(list);
+        ServiceResponse<Integer> downloadTask = hiveDataDownloadApiService.createDownloadTask(dataDownloadDTO);
+        if(downloadTask.isSuccess()){
+            return RestResponse.success();
+        }
+        return EnterpriseRestResponse.failure(downloadTask);
+    }
 }

gic-platform-enterprise-web/src/main/resources/dubbo-gic-platform-enterprise-web.xml

@@ -160,4 +160,5 @@
     <dubbo:reference interface="com.gic.member.config.api.service.UserListFieldApiService" id="userListFieldApiService" timeout="6000" retries="0"/>
     <dubbo:reference interface="com.gic.store.service.StoreResourceApiService" id="storeResourceApiService" timeout="6000" retries="0"/>
     <dubbo:reference interface="com.gic.enterprise.service.CustomGuideTopApiService" id="customGuideTopApiService" timeout="6000" retries="0"/>
+    <dubbo:reference interface="com.gic.enterprise.service.HiveDataDownloadApiService" id="hiveDataDownloadApiService" timeout="6000" retries="0"/>
 </beans>