Merge branch 'developer' into 'master'

Developer

See merge request !49

gic-cloud-data-hook-api/.flattened-pom.xml

@@ -4,12 +4,12 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.gic</groupId>
   <artifactId>gic-cloud-data-hook-api</artifactId>
-  <version>2.28</version>
+  <version>3.0-SNAPSHOT</version>
   <dependencies>
     <dependency>
       <groupId>com.gic</groupId>
       <artifactId>gic-cloud-web-service-api</artifactId>
-      <version>3.73</version>
+      <version>3.0-SNAPSHOT</version>
       <scope>compile</scope>
       <exclusions>
         <exclusion>
@@ -25,7 +25,7 @@
     <dependency>
       <groupId>com.gic</groupId>
       <artifactId>gic-cloud-commons</artifactId>
-      <version>3.63</version>
+      <version>3.0-SNAPSHOT</version>
       <scope>compile</scope>
       <exclusions>
         <exclusion>
@@ -41,7 +41,7 @@
     <dependency>
       <groupId>com.gic</groupId>
       <artifactId>gic-platform-config</artifactId>
-      <version>2.19</version>
+      <version>3.0-SNAPSHOT</version>
       <scope>compile</scope>
     </dependency>
     <dependency>

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/dto/DownloadTask.java

@@ -366,4 +366,34 @@ public class DownloadTask implements Serializable {
 	public void setDownloadCondition(String downloadCondition) {
 		this.downloadCondition = downloadCondition;
 	}
+
+	private Date downloadTime;
+
+	public Date getDownloadTime() {
+		return downloadTime;
+	}
+
+	public void setDownloadTime(Date downloadTime) {
+		this.downloadTime = downloadTime;
+	}
+
+	private Integer downloadWay;
+
+	public Integer getDownloadWay() {
+		return downloadWay;
+	}
+
+	public void setDownloadWay(Integer downloadWay) {
+		this.downloadWay = downloadWay;
+	}
+
+	private Integer fieldSize;
+
+	public Integer getFieldSize() {
+		return fieldSize;
+	}
+
+	public void setFieldSize(Integer fieldSize) {
+		this.fieldSize = fieldSize;
+	}
 }

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/dto/FieldSizeQuery.java

+package com.gic.cloud.data.hook.api.dto;
+
+import java.io.Serializable;
+
+public class FieldSizeQuery implements Serializable {
+    private int size;
+
+    public int getSize() {
+        return size;
+    }
+
+    public void setSize(int size) {
+        this.size = size;
+    }
+}

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/dto/TaskDistCondition.java

+package com.gic.cloud.data.hook.api.dto;
+
+import java.io.Serializable;
+
+public class TaskDistCondition implements Serializable {
+    private TaskDistConditionRange fieldSize;
+    private TaskDistConditionRange resultCount;
+
+    public TaskDistConditionRange getFieldSize() {
+        return fieldSize;
+    }
+
+    public void setFieldSize(TaskDistConditionRange fieldSize) {
+        this.fieldSize = fieldSize;
+    }
+
+    public TaskDistConditionRange getResultCount() {
+        return resultCount;
+    }
+
+    public void setResultCount(TaskDistConditionRange resultCount) {
+        this.resultCount = resultCount;
+    }
+}

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/dto/TaskDistConditionRange.java

+package com.gic.cloud.data.hook.api.dto;
+
+import java.io.Serializable;
+
+public class TaskDistConditionRange implements Serializable {
+    private int start;
+    private int end;
+
+    public int getStart() {
+        return start;
+    }
+
+    public void setStart(int start) {
+        this.start = start;
+    }
+
+    public int getEnd() {
+        return end;
+    }
+
+    public void setEnd(int end) {
+        this.end = end;
+    }
+}

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/dto/TaskDistConfig.java

+package com.gic.cloud.data.hook.api.dto;
+
+import java.io.Serializable;
+import java.util.Date;
+import java.util.List;
+
+public class TaskDistConfig implements Serializable {
+    //快速执行队列
+    private List<TaskDistCondition> fast;
+    //中等可动态队列
+    private List<TaskDistCondition> middle;
+    //慢队列
+    private List<TaskDistCondition> slow;
+
+    public List<TaskDistCondition> getFast() {
+        return fast;
+    }
+
+    public void setFast(List<TaskDistCondition> fast) {
+        this.fast = fast;
+    }
+
+    public List<TaskDistCondition> getMiddle() {
+        return middle;
+    }
+
+    public void setMiddle(List<TaskDistCondition> middle) {
+        this.middle = middle;
+    }
+
+    public List<TaskDistCondition> getSlow() {
+        return slow;
+    }
+
+    public void setSlow(List<TaskDistCondition> slow) {
+        this.slow = slow;
+    }
+}

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/entity/DownloadWayType.java

+package com.gic.cloud.data.hook.api.entity;
+
+public class DownloadWayType {
+    /** 快速通道 */
+    public static final int smallConnect = 0;
+    /** 大查询通道 */
+    public static final int bigConnect = 1;
+    /** 巴拉通道 */
+    public static final int balaConnect = 2;
+}

gic-cloud-data-hook-api/src/main/java/com/gic/cloud/data/hook/api/entity/FlatQueryTaskCondition.java

@@ -196,4 +196,14 @@ public class FlatQueryTaskCondition {
 	public void setAuthStoreIdList(List<String> authStoreIdList) {
 		this.authStoreIdList = authStoreIdList;
 	}
+
+	private int amount;
+
+	public int getAmount() {
+		return amount;
+	}
+
+	public void setAmount(int amount) {
+		this.amount = amount;
+	}
 }

gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java

@@ -513,6 +513,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 		condition.setOrderField(orderField);
 		condition.setOrderDir(orderDir);
 		condition.setExecDistinct(execDistinct);
+		condition.setAmount(amount);
 		// 设置条件可否执行
 		if (curTask.getApplyPermitted().equals(Global.YES)) {
 			condition.setBuildPermitted(Global.YES);
@@ -617,6 +618,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 		}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
 	}
 
+	private
+
 
 	/** 下载任务执行计时器 */
 	//private Timer downloadTaskTimer = new Timer();
@@ -650,7 +653,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 								}
 							} // IF OVER
 						} // FOR OVER
-						takeFile(condition, connection);
+						takeFile(condition, connection,DownloadWayType.smallConnect);
 					} // 没有任务则忽略
 				}catch (Exception e){
 					logger.info("异常：{}",e);
@@ -702,7 +705,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 								}
 							} // IF OVER
 						} // FOR OVER
-						takeFile(condition, connection);
+						takeFile(condition, connection,DownloadWayType.bigConnect);
 						bigTaskRunningMap.remove(condition.getTaskId());
 					} // 没有任务则忽略
 				}catch (Exception e){
@@ -756,7 +759,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 							} // IF OVER
 						} // FOR OVER
 
-						takeFile(condition, connection);
+						takeFile(condition, connection,DownloadWayType.balaConnect);
 					} // 没有任务则忽略
 				}catch (Exception e){
 					logger.info("异常：{}",e);
@@ -775,11 +778,16 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
 		}, interval*1000, interval*1000, TimeUnit.MILLISECONDS);
 	}
 
-	private void takeFile(FlatQueryTaskCondition condition, Connection conn){
+	private void takeFile(FlatQueryTaskCondition condition, Connection conn, int downloadType){
 		if (condition != null) {
 			// 更新任务状态
 			DownloadTask task = DownloadTaskServiceImpl.getInstance().getDownloadTaskById(condition.getTaskId());
 			task.setStatus(DownloadTaskStatus.BUILDING);
+			task.setDownloadWay(downloadType);
+			task.setDownloadTime(new Date());
+			task.setFieldSize(condition.getAllFields().size());
+
+
 			DownloadTaskServiceImpl.getInstance().updateDownloadTask(task);
 			logger.info("[ runDownloadTask.run ]: {}", "自助指标下载任务执行：" + task.getId());
 			//初始化校验

gic-cloud-data-hook-service/src/main/resources/mapper/DownloadTaskDao.xml

@@ -152,7 +152,10 @@
 			apply_permitted = #{applyPermitted},
 			real_sql = #{realSql},
 			enterprise_id = #{enterpriseId},
-			report_id = #{reportId}
+			report_id = #{reportId},
+			download_time = #{downloadTime},
+			download_way = #{downloadWay},
+			field_size = #{fieldSize}
 		WHERE
 			id = #{id}
 	</update>

gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/web/FlatQueryController.java

 package com.gic.cloud.data.hook.web;
 
+import com.alibaba.fastjson.JSONObject;
+import com.ctrip.framework.apollo.Config;
+import com.ctrip.framework.apollo.ConfigService;
 import com.gic.clerk.api.constant.ThirdProjectEnum;
 import com.gic.clerk.api.dto.AuthorizedUser;
 import com.gic.clerk.api.dto.PowerClerkDTO;
@@ -24,6 +27,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.util.CollectionUtils;
+import org.springframework.util.StringUtils;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -43,6 +47,8 @@ public class FlatQueryController {
 
 	private static Logger logger= LoggerFactory.getLogger(FlatQueryController.class);
 
+	private static final int FIELD_SIZE=100;
+
 	/** 脱敏字段 */
 	public static final List<String> FILTERS_PHONE_ONLY = Arrays.asList("mobile", "phone", "enterprise_name", "phone_number", "receive_phone_number", "use_phone_number", "friend_phone_num","from_phone_num");
 
@@ -207,6 +213,64 @@ public class FlatQueryController {
 		return result;
 	}
 
+	/** 获取自助查询下载量评估
+	 * @param tableId
+	 * @param request
+	 * @param response
+	 * @return
+	 */
+	@RequestMapping("/get-query-field-size")
+	public FieldSizeQuery getQueryFieldSize(String tableId, HttpServletRequest request, HttpServletResponse response) {
+		FieldSizeQuery result = new FieldSizeQuery();
+		if (StringUtils.isEmpty(tableId)) {
+			logger.info("参数缺失tableID");
+			result.setSize(FIELD_SIZE);
+			return result;
+		}
+		String enterpriseId = SessionContextUtils.getLoginUserEnterpriseId();
+		int fieldSize = getFieldSize(enterpriseId, tableId);
+		result.setSize(fieldSize);
+		return result;
+	}
+
+	private int getFieldSize(String enterpriseId,String tableId) {
+
+		Config appConfig = ConfigService.getAppConfig();
+		String fieldSizeConfig = appConfig.getProperty("field.size", "");
+		if (StringUtils.isEmpty(fieldSizeConfig)) {
+			logger.info("没有在apoll配置");
+			return FIELD_SIZE;
+		}
+		JSONObject config = JSONObject.parseObject(fieldSizeConfig);
+		JSONObject defaultConfig = config.getJSONObject("default");
+		JSONObject tableConfig = config.getJSONObject(tableId);
+		if (null == tableConfig && defaultConfig == null) {
+			return FIELD_SIZE;
+		}
+		Integer num=null;
+		if (tableConfig!=null){
+			num = tableConfig.getInteger(enterpriseId);
+			if (num != null) {
+				return num;
+			}
+			num = tableConfig.getInteger("common");
+			if (num != null) {
+				return num;
+			}
+		}
+		if (defaultConfig!=null) {
+			num = defaultConfig.getInteger(enterpriseId);
+			if (num != null) {
+				return num;
+			}
+			num = defaultConfig.getInteger("common");
+			if (num != null) {
+				return num;
+			}
+		}
+		return FIELD_SIZE;
+	}
+
 	/** 生成自助查询下载任务
 	 * @param executeRequest
 	 * @param request