Skip to content

G
gic-cloud
Commit 6da39ffa by 陶光胜
Options

Merge branch 'developer' into 'master' 

Developer

See merge request !20
parents dda7a40b 98f07410
Showing with 324 additions and 39 deletions
gic-cloud-data-hook-service/dubbo/local/gic-member/dubbo.cache
#Dubbo Registry Cache
#Wed Sep 16 09:40:39 CST 2020
com.gic.cloud.data.hook.api.service.SearchLogService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.SearchLogService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.SearchLogService&methods\=saveLog&payload\=41557050&pid\=16652&side\=provider&timeout\=120000&timestamp\=1600140918276
com.gic.cloud.data.hook.api.service.IDownloadTaskService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IDownloadTaskService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IDownloadTaskService&methods\=createRiskModeRecord,listUnDownloadTask,createDownloadRecord,getRiskModeRecordPage,getDownloadRecordPage,deleteDownloadTask,getDownloadProcess,getDownloadTaskPage&payload\=41557050&pid\=16652&side\=provider&timeout\=120000&timestamp\=1600140918147
com.gic.cloud.data.hook.api.service.IFlatQueryTableService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryTableService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryTableService&methods\=queryFlatQueryFavo,queryFlatQueryTableById,queryFlatQueryTablePage,createFlatQueryFavo,removeFlatQueryFavo,queryFlatQueryTableDetailByTableId&payload\=41557050&pid\=16652&side\=provider&timeout\=120000&timestamp\=1600140918020
com.gic.cloud.data.hook.api.service.IFlatQueryResultService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryResultService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryResultService&methods\=getFlatQueryFilterData,initTask,estimateFlatQueryDownloadTask,buildFlatQueryDownloadTask,getFlatQueryResult&payload\=41557050&pid\=16652&side\=provider&timeout\=120000&timestamp\=1600140918093
com.gic.cloud.data.hook.api.service.IMyFlatQueryService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IMyFlatQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IMyFlatQueryService&methods\=createMyFlatQuery,queryMyFlatQueryPage,deleteMyFlatQuery,updateMyFlatQuery,queryMyFlatQueryById&payload\=41557050&pid\=16652&side\=provider&timeout\=120000&timestamp\=1600140918216
com.gic.cloud.data.hook.api.service.IFreeQueryService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IFreeQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFreeQueryService&methods\=getFreeQueryRecordByUserIdAndRecordId,createFreeQueryRecord,deleteFreeQueryRecord,getColumnsByTable,buildFreeQueryDownloadTask,updateFreeQueryRecord,getFreeQueryResult,initTask,getFreeQuerySource,getFreeQueryCount,getFreeQueryRecordListByUserId,getFreeQueryTableList&payload\=41557050&pid\=16652&side\=provider&timeout\=120000&timestamp\=1600140917339
#Thu Sep 09 18:47:36 CST 2021
com.gic.cloud.data.hook.api.service.IDownloadTaskService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IDownloadTaskService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IDownloadTaskService&methods\=createRiskModeRecord,createDownloadRecord,getRiskModeRecordPage,listUnDownloadTask,getDownloadRecordPage,deleteDownloadTask,getDownloadProcess,getDownloadTaskPage&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455577
com.gic.cloud.data.hook.api.service.SearchLogService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.SearchLogService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.SearchLogService&methods\=saveLog&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455823
com.gic.cloud.data.hook.api.service.IFlatQueryTableService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryTableService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryTableService&methods\=queryFlatQueryFavo,queryFlatQueryTableById,queryFlatQueryTablePage,createFlatQueryFavo,removeFlatQueryFavo,queryFlatQueryTableDetailByTableId&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455351
com.gic.cloud.data.hook.api.service.TableSyncRecordService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.TableSyncRecordService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.TableSyncRecordService&methods\=lastSyncDateTime&payload\=41557050&pid\=47060&side\=provider&timestamp\=1631184456001
com.gic.cloud.data.hook.api.service.IFlatQueryResultService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryResultService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryResultService&methods\=getFlatQueryFilterData,initTask,estimateFlatQueryDownloadTask,buildFlatQueryDownloadTask,getFlatQueryResult&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455489
com.gic.cloud.data.hook.api.service.IMyFlatQueryService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IMyFlatQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IMyFlatQueryService&methods\=queryMyFlatQueryPage,createMyFlatQuery,deleteMyFlatQuery,updateMyFlatQuery,queryMyFlatQueryById&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184455740
com.gic.cloud.data.hook.api.service.IFreeQueryService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFreeQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFreeQueryService&methods\=getFreeQueryRecordByUserIdAndRecordId,createFreeQueryRecord,buildFreeQueryDownloadTask,deleteFreeQueryRecord,getColumnsByTable,updateFreeQueryRecord,getFreeQueryResult,getFreeQuerySource,initTask,getFreeQueryCount,getFreeQueryRecordListByUserId,getFreeQueryTableList&payload\=41557050&pid\=47060&side\=provider&timeout\=120000&timestamp\=1631184454144
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DataInitUtils.java 0 → 100644
package com.gic.cloud.data.hook.service;
import cn.medubi.client.utils.LogPak;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.gic.cloud.data.hook.api.dto.DownloadRecord;
import com.gic.cloud.data.hook.api.dto.DownloadTask;
import java.net.InetAddress;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
public class DataInitUtils {
private static LogPak log = new LogPak(DataInitUtils.class);
private static String host = "";
static {
host = getHost();
}
public static List<DownloadRecord> listByHost(List<DownloadRecord> allList){
List<DownloadRecord> list = new ArrayList<>();
Config config = ConfigService.getConfig("application");
String customRouterHost = config.getProperty("customRouterHost", "");
String customRouterEnterpriseId = config.getProperty("customRouterEnterpriseId", "");
for(DownloadRecord downloadRecord : allList){
if(host.equals(customRouterHost)){
if(customRouterEnterpriseId.equals(downloadRecord.getEnterpriseId())){
list.add(downloadRecord);
}
} else {
if(!customRouterEnterpriseId.equals(downloadRecord.getEnterpriseId())){
list.add(downloadRecord);
}
}
}
return list;
}
public static List<DownloadTask> listTaskByHost(List<DownloadTask> allList){
List<DownloadTask> list = new ArrayList<>();
Config config = ConfigService.getConfig("application");
String customRouterHost = config.getProperty("customRouterHost", "");
String customRouterEnterpriseId = config.getProperty("customRouterEnterpriseId", "");
for(DownloadTask downloadTask : allList){
if(host.equals(customRouterHost)){
if(customRouterEnterpriseId.equals(downloadTask.getEnterpriseId())){
list.add(downloadTask);
}
} else {
if(!customRouterEnterpriseId.equals(downloadTask.getEnterpriseId())){
list.add(downloadTask);
}
}
}
return list;
}
private static String getHost(){
String localip = null;// 本地IP，如果没有配置外网IP则返回它
String netip = null;// 外网IP
try {
Enumeration netInterfaces = NetworkInterface.getNetworkInterfaces();
InetAddress ip = null;
boolean finded = false;// 是否找到外网IP
while (netInterfaces.hasMoreElements() && !finded) {
NetworkInterface ni = (NetworkInterface) netInterfaces.nextElement();
Enumeration address = ni.getInetAddresses();
while (address.hasMoreElements()) {
ip = (InetAddress) address.nextElement();
if (!ip.isSiteLocalAddress() && !ip.isLoopbackAddress() && ip.getHostAddress().indexOf(":") == -1) {// 外网IP
netip = ip.getHostAddress();
finded = true;
break;
} else if (ip.isSiteLocalAddress() && !ip.isLoopbackAddress() && ip.getHostAddress().indexOf(":") == -1) {// 内网IP
localip = ip.getHostAddress();
}
}
}
} catch (SocketException e) {
e.printStackTrace();
}
if (netip != null && !"".equals(netip)) {
return netip;
} else {
return localip;
}
}
public static void main(String[] args){
System.out.println("getHost() = " + getHost());
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/HiveHelper.java
......@@ -2,6 +2,8 @@ package com.gic.cloud.data.hook.service;
import cn.medubi.client.utils.LogPak;
import com.alibaba.fastjson.JSON;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
......@@ -11,6 +13,7 @@ import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.List;
/** Hive 辅助工具类
* @author Sam
......@@ -28,12 +31,15 @@ public class HiveHelper implements ApplicationContextAware {
private static DataSource balaDownloadSource = null;
private static DataSource balaSearchSource = null;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
log.debug("setApplicationContext", "准备初始化 Hive 连接池");
source = (DataSource)applicationContext.getBean("hiveSource");
downloadSource = (DataSource)applicationContext.getBean("downloadHiveSource");
balaDownloadSource = (DataSource)applicationContext.getBean("balaDownloadHiveSource");
balaSearchSource = (DataSource)applicationContext.getBean("balaSearchHiveSource");
}
/** 获取 Hive 数据源
......@@ -113,4 +119,36 @@ public class HiveHelper implements ApplicationContextAware {
} // TRY CATCH OVER
}
public static Connection getBalaSearchHiveConnection() {
Connection conn = null;
try {
// String url = "jdbc:hive2://115.159.205.44:10015/data_test";
// //String url = "jdbc:hive2://10.0.0.3:10015/data_test";
// Class.forName("org.apache.hive.jdbc.HiveDriver");
// conn = DriverManager.getConnection(url, "hadoop", "");
conn = balaSearchSource.getConnection();
return conn;
} catch (Exception ex) {
ex.printStackTrace();
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
return null;
} // TRY CATCH OVER
}
public static Connection getSearchHiveConnection(List<String> enterpriseIds){
Config config = ConfigService.getConfig("application");
String customRouterEnterpriseId = config.getProperty("customRouterEnterpriseId", "");
if(enterpriseIds.contains(customRouterEnterpriseId)){
return HiveHelper.getBalaSearchHiveConnection();
} else {
return HiveHelper.getHiveConnection();
}
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
......@@ -2,15 +2,14 @@ package com.gic.cloud.data.hook.service.impl;
import cn.medubi.client.utils.LogPak;
import com.alibaba.fastjson.JSON;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.gic.cloud.common.api.base.Page;
import com.gic.cloud.data.hook.api.dto.*;
import com.gic.cloud.data.hook.api.entity.*;
import com.gic.cloud.data.hook.api.service.IDownloadTaskService;
import com.gic.cloud.data.hook.api.service.IFlatQueryResultService;
import com.gic.cloud.data.hook.service.DecryptUtils;
import com.gic.cloud.data.hook.service.HiveHelper;
import com.gic.cloud.data.hook.service.HttpUtils;
import com.gic.cloud.data.hook.service.ServiceUtils;
import com.gic.cloud.data.hook.service.*;
import com.gic.cloud.data.hook.service.dao.FlatQueryTableDao;
import com.gic.cloud.data.hook.service.entity.CsvDataFilterMode;
import com.gic.cloud.data.hook.service.entity.CsvResultSetHelper;
......@@ -19,6 +18,7 @@ import com.google.common.collect.Lists;
import com.opencsv.CSVWriter;
import com.opencsv.ResultSetHelper;
import io.netty.handler.codec.http.HttpUtil;
import javafx.scene.chart.PieChart;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
......@@ -36,6 +36,7 @@ import java.io.*;
import java.math.BigDecimal;
import java.nio.charset.Charset;
import java.sql.*;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Date;
......@@ -203,7 +204,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
limitRange,
authStoreIdList);
// 进行查询
Connection conn = HiveHelper.getHiveConnection();
//获取连接根据企业判断，巴拉走单独的端口
Connection conn = HiveHelper.getSearchHiveConnection(enterpriseIds);
if (conn != null) {
Date startTime = new Date();
try {
......@@ -357,6 +359,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
public Integer estimateFlatQueryDownloadTask(String tableId, List<String> enterpriseIds,
List<FlatQueryCondition> queryConditions, String orderField,
String orderDir, Boolean execDistinct, List<String> authStoreIdList) {
log.debug("estimateFlatQueryDownloadTask", tableId);
Integer result = -1;
// 生成正式查询
String preQuery = this.buildFlatQuerySQL(
......@@ -370,7 +373,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
0,
authStoreIdList);
String finQuery = "SELECT COUNT(1) FROM (" + preQuery + ") AS tmpTable";
Connection conn = HiveHelper.getHiveConnection();
Connection conn = HiveHelper.getSearchHiveConnection(enterpriseIds);
if (conn != null) {
try {
Statement stat = conn.createStatement();
......@@ -476,6 +479,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
public void initTask(){
try {
List<DownloadRecord> list = this.downloadTaskService.listUnDownloadTask(QueryDataSource.FLAT_QUERY);
list = DataInitUtils.listByHost(list);
log.debug("flatInitList", JSON.toJSONString(list));
if(CollectionUtils.isNotEmpty(list)){
for(DownloadRecord record : list){
if(StringUtils.isNotBlank(record.getDownloadCondition())){
......@@ -505,6 +510,13 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private static SimpleDateFormat timeFormatter = new SimpleDateFormat("HH:mm:ss");
public static void main(String[] args) throws ParseException {
String date = "2021-09-01";
Date parse = dateFormatter.parse(date);
String format = datetimeFormatter.format(parse);
System.out.println("date = " + format);
}
/** 下载任务执行计时器 */
//private Timer downloadTaskTimer = new Timer();
ScheduledExecutorService downloadService = new ScheduledThreadPoolExecutor(1,
......@@ -895,12 +907,13 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
@Override
public void run() {
List<DownloadTask> waitingTasks = DownloadTaskServiceImpl.getInstance().getDownloadTaskOfWaiting(QueryDataSource.FLAT_QUERY);
waitingTasks = DataInitUtils.listTaskByHost(waitingTasks);
for (DownloadTask task : waitingTasks) {
log.debug("自助指标待审批任务状态变更", JSON.toJSONString(task));
//log.debug("自助指标待审批任务状态变更", JSON.toJSONString(task));
if (!task.getApplyId().equals("")) {
// String applyStatusText = HttpUtils.doGet("http://www.gicdev.com/api-admin/apply-info?type=2&applyId=" + task.getApplyId());
String applyStatusText = HttpUtils.doGet("http://hope.demogic.com/api-admin/apply-info?type=2&applyId=" + task.getApplyId());
log.debug("runApplyTask.run", "查询自助指标查询 " + task.getId() + " 审核状态码" + applyStatusText);
//log.debug("runApplyTask.run", "查询自助指标查询 " + task.getId() + " 审核状态码" + applyStatusText);
if(StringUtils.isBlank(applyStatusText)){
break;
}
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryTableServiceImpl.java
package com.gic.cloud.data.hook.service.impl;
import cn.medubi.client.utils.LogPak;
import com.alibaba.fastjson.JSON;
import com.gic.cloud.common.api.base.Page;
import com.gic.cloud.data.hook.api.dto.FlatQueryFavo;
......@@ -7,6 +8,7 @@ import com.gic.cloud.data.hook.api.dto.FlatQueryField;
import com.gic.cloud.data.hook.api.dto.FlatQueryTable;
import com.gic.cloud.data.hook.api.dto.FlatQueryTableDetail;
import com.gic.cloud.data.hook.api.service.IFlatQueryTableService;
import com.gic.cloud.data.hook.service.DataInitUtils;
import com.gic.cloud.data.hook.service.dao.FlatQueryFavoDao;
import com.gic.cloud.data.hook.service.dao.FlatQueryFieldDao;
import com.gic.cloud.data.hook.service.dao.FlatQueryTableDao;
......@@ -24,7 +26,7 @@ import java.util.List;
@Service
public class FlatQueryTableServiceImpl implements IFlatQueryTableService {
private org.slf4j.Logger logger= LoggerFactory.getLogger(IFlatQueryTableService.class);
private static LogPak log = new LogPak(FlatQueryTableServiceImpl.class);
/** 自助查询表信息 Dao */
@Autowired
......@@ -41,12 +43,13 @@ public class FlatQueryTableServiceImpl implements IFlatQueryTableService {
@Override
public FlatQueryTable queryFlatQueryTableById(String userId, String id) {
FlatQueryTable table = this.flatQueryTableDao.getFlatQueryTableById(userId, id);
logger.debug("json 格式为 " + JSON.toJSONString(table));
log.debug("json 格式为 :" , JSON.toJSONString(table));
return table;
}
@Override
public Page<FlatQueryTable> queryFlatQueryTablePage(String userId, String fuzzy, int pageNum, int pageSize, List<String> authTables) {
log.debug("queryFlatQueryTablePage", "start");
Page<FlatQueryTable> result = new Page<>();
PageHelper.startPage(pageNum, pageSize); // 设置分页参数
List<FlatQueryTable> query = flatQueryTableDao.getFlatQueryTable(userId, fuzzy, authTables);
......@@ -55,7 +58,7 @@ public class FlatQueryTableServiceImpl implements IFlatQueryTableService {
result.setList(preResult.getList()); // 设置数据集
result.setTotal(preResult.getTotal()); // 设置总数
result.setPageNum(preResult.getPageNum()); // 设置当前页数
logger.debug("json 格式为 " + JSON.toJSONString(result));
log.debug("json 格式为: " , JSON.toJSONString(result));
return result;
}
......
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
......@@ -7,10 +7,7 @@ import com.gic.cloud.data.hook.api.dto.DataDesensiType;
import com.gic.cloud.data.hook.api.entity.*;
import com.gic.cloud.data.hook.api.service.IDownloadTaskService;
import com.gic.cloud.data.hook.api.service.IFreeQueryService;
import com.gic.cloud.data.hook.service.HiveHelper;
import com.gic.cloud.data.hook.service.HttpUtils;
import com.gic.cloud.data.hook.service.MysqlHelper;
import com.gic.cloud.data.hook.service.ServiceUtils;
import com.gic.cloud.data.hook.service.*;
import com.gic.cloud.data.hook.service.dao.FreeQueryRecordDao;
import com.gic.cloud.data.hook.service.dao.FreeQuerySourceDao;
import com.gic.cloud.data.hook.service.entity.CsvDataFilterMode;
......@@ -102,6 +99,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
@Override
public List<FreeQueryTable> getFreeQueryTableList() {
log.debug("getFreeQueryTableList", "getFreeQueryTableList");
return MysqlHelper.getFreeQueryTableList();
}
......@@ -327,6 +325,8 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
public void initTask() {
try{
List<DownloadRecord> list = this.downloadTaskService.listUnDownloadTask(QueryDataSource.FREE_QUERY);
list = DataInitUtils.listByHost(list);
log.debug("freeInitList", JSON.toJSONString(list));
if(CollectionUtils.isNotEmpty(list)) {
for (DownloadRecord record : list) {
if(record.getDesensiType() != null){
......
gic-cloud-data-hook-service/src/main/resources/data-hook-flat-query-source.xml
......@@ -81,4 +81,27 @@
</property>
</bean>
<bean class="org.apache.commons.dbcp.BasicDataSource" id="balaSearchHiveSource" destroy-method="close">
<property name="driverClassName" value="org.apache.hive.jdbc.HiveDriver" />
<!--<property name="url" value="jdbc:hive2://115.159.205.44:10015/data_test" />-->
<property name="url" value="${hive.balasearch.url}" />
<property name="username" value="${hive.username}" />
<property name="password" value="" />
<property name="maxActive">
<value>20</value>
</property>
<property name="maxIdle">
<value>5</value>
</property>
<property name="minIdle">
<value>1</value>
</property>
<property name="testWhileIdle">
<value>true</value>
</property>
<property name="maxWait">
<value>120000</value>
</property>
</bean>
</beans>
\ No newline at end of file
gic-cloud-data-hook/pom.xml
......@@ -99,8 +99,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
<compilerArguments>
<verbose/>
......@@ -113,7 +113,7 @@
<artifactId>tomcat7-maven-plugin</artifactId>
<version>2.1</version>
<configuration>
<port>9090</port>
<port>9191</port>
<path>/</path>
<uriEncoding>UTF-8</uriEncoding>
<finalName>my-proxy</finalName>
......
gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/balance/CustomBalance.java 0 → 100644
package com.gic.cloud.data.hook.balance;
import com.alibaba.dubbo.common.URL;
import com.alibaba.dubbo.rpc.Invocation;
import com.alibaba.dubbo.rpc.Invoker;
import com.alibaba.dubbo.rpc.RpcException;
import com.alibaba.dubbo.rpc.cluster.LoadBalance;
import java.util.List;
public class CustomBalance implements LoadBalance {
@Override
public <T> Invoker<T> select(List<Invoker<T>> invokers, URL url, Invocation invocation) throws RpcException {
System.out.println("dubbo负载");
return invokers.get(0);
}
}
gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/router/CustomRouter.java 0 → 100644
package com.gic.cloud.data.hook.router;
import com.alibaba.dubbo.common.URL;
import com.alibaba.dubbo.rpc.Invocation;
import com.alibaba.dubbo.rpc.Invoker;
import com.alibaba.dubbo.rpc.RpcException;
import com.alibaba.dubbo.rpc.cluster.Router;
import com.ctrip.framework.apollo.Config;
import com.ctrip.framework.apollo.ConfigService;
import com.gic.web.common.utils.SessionContextUtils;
import java.util.ArrayList;
import java.util.List;
public class CustomRouter implements Router {
private URL url;
public CustomRouter(URL url){
this.url = url;
}
@Override
public URL getUrl() {
return url;
}
@Override
public <T> List<Invoker<T>> route(List<Invoker<T>> invokers, URL url, Invocation invocation) throws RpcException {
System.out.println("start route = " + System.currentTimeMillis());
if(url.getParameter("application", "").equals("gic-cloud-data-hook") &&
url.getParameter("interface").contains("com.gic.cloud.data.hook")){
String enterpriseId = SessionContextUtils.getLoginUserEnterpriseId();
Config config = ConfigService.getConfig("application");
String customRouterEnterpriseId = config.getProperty("customRouterEnterpriseId", "");
String customRouterHost = config.getProperty("customRouterHost", "");
System.out.println("end = " + System.currentTimeMillis());
List<Invoker<T>> list = new ArrayList<>();
if(enterpriseId.equals(customRouterEnterpriseId)){
//路由到新服务器
for(Invoker invoker : invokers){
if(invoker.getUrl().getHost().equals(customRouterHost)){
list.add(invoker);
}
}
} else {
for(Invoker invoker : invokers){
if(!invoker.getUrl().getHost().equals(customRouterHost)){
list.add(invoker);
}
}
}
System.out.println("end route = " + System.currentTimeMillis());
return list;
}
return invokers;
}
@Override
public int compareTo(Router o) {
return 0;
}
}
gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/router/RouterFactory.java 0 → 100644
package com.gic.cloud.data.hook.router;
import com.alibaba.dubbo.common.URL;
import com.alibaba.dubbo.common.extension.Activate;
import com.alibaba.dubbo.rpc.cluster.Router;
public class RouterFactory implements com.alibaba.dubbo.rpc.cluster.RouterFactory {
@Override
public Router getRouter(URL url) {
return new CustomRouter(url);
}
}
gic-cloud-data-hook/src/main/resources/META-INF/dubbo/com.alibaba.dubbo.rpc.cluster.LoadBalance 0 → 100644
customBalance=com.gic.cloud.data.hook.balance.CustomBalance
\ No newline at end of file
gic-cloud-data-hook/src/main/resources/META-INF/dubbo/com.alibaba.dubbo.rpc.cluster.RouterFactory 0 → 100644
customRouterFactory=com.gic.cloud.data.hook.router.RouterFactory
\ No newline at end of file
gic-cloud-data-hook/src/main/webapp/WEB-INF/applicationContext-conf.xml
......@@ -7,8 +7,8 @@
<!--<import resource="classpath:data-hook-init.xml"/>--><!-- data-hook 所有项目使用的 dubbo 环境配置（位于 data-hook-api/resource） -->
<!--<import resource="classpath*:applicationContext-sharding-db.xml"/>-->
<import resource="classpath*:applicationContext-init.xml"/>
<import resource="classpath*:dubbo-setting.xml"/>
<!-- <import resource="data-hook-dubbo-settings.xml"/>--><!-- data-hook 所有项目初始化参数加载（位于 data-hook-api/resource）-->
<!--<import resource="classpath*:dubbo-setting.xml"/>-->
<import resource="data-hook-dubbo-settings.xml"/><!-- data-hook 所有项目初始化参数加载（位于 data-hook-api/resource）-->
<import resource="data-hook-dubbo-config.xml"/><!-- data-hook web 项目使用的 dubbo 引用服务配置 -->
</beans>
gic-cloud-data-hook/src/main/webapp/WEB-INF/data-hook-dubbo-config.xml
......@@ -13,8 +13,7 @@
<!-- 应用名称 -->
<dubbo:application name="gic-cloud-data-hook"/>
<!-- 外部服务端口 -->
<dubbo:protocol name="dubbo" port="2181"/>
<dubbo:protocol name="dubbo" port="2181"></dubbo:protocol>
<!-- 引用的 Dubbo 服务 -->
<dubbo:reference interface="com.gic.cloud.data.hook.api.service.IFlatQueryTableService" id="flatQueryTableService" timeout="120000" retries="0" />
<dubbo:reference interface="com.gic.cloud.data.hook.api.service.IFlatQueryResultService" id="flatQueryResultService" timeout="120000" retries="0" />
......
gic-cloud-data-hook/src/main/webapp/WEB-INF/data-hook-dubbo-settings.xml
......@@ -2,19 +2,32 @@
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:dubbo="http://code.alibabatech.com/schema/dubbo"
xmlns:p="http://www.springframework.org/schema/p"
xmlns:apollo="http://www.ctrip.com/schema/apollo"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
http://code.alibabatech.com/schema/dubbo
http://code.alibabatech.com/schema/dubbo/dubbo.xsd">
http://code.alibabatech.com/schema/dubbo http://code.alibabatech.com/schema/dubbo/dubbo.xsd
http://www.ctrip.com/schema/apollo
http://www.ctrip.com/schema/apollo.xsd">
<!--<context:property-placeholder location="classpath:jdbc.properties" ignore-unresolvable="true"/>-->
<apollo:config namespaces="COMMON.dubbo, COMMON.jdbc"/>
<dubbo:monitor protocol="registry"/>
<dubbo:registry address="${zookeeper_host}" protocol="dubbo" id="remoteAddress" timeout="20000"
register="${shall_register:true}"
file="dubbo/remote/${dubbo.registry.file}/dubbo.cache"
default="true">
<dubbo:parameter key="router" value="customRouterFactory"/>
</dubbo:registry>
<dubbo:registry address="zookeeper://localhost:2181" protocol="dubbo" id="localAddress" timeout="10000"
file="dubbo/local/${dubbo.registry.file}/dubbo.cache" default="false">
<dubbo:parameter key="router" value="customRouterFactory"/>
</dubbo:registry>
<!--<dubbo:registry address="zookeeper://123.207.187.158:2199" protocol="dubbo" id="remoteAddress" timeout="20000" file="${catalina.home}/dubbo/${dubbo.registry.file}/dubbo.cache"/>-->
<!--<dubbo:registry address="zookeeper://123.207.187.158:2199" protocol="dubbo" id="remoteAddress" register="false" timeout="120000" file="dubbo/remote/${dubbo.registry.file}/dubbo.cache"/>-->
<dubbo:registry address="zookeeper://localhost:2181" protocol="dubbo" id="localAddress" timeout="120000" file="dubbo/local/${dubbo.registry.file}/dubbo.cache"/>
<dubbo:registry address="zookeeper://localhost:2181" protocol="dubbo" id="remoteAddress" timeout="120000" file="dubbo/local/${dubbo.registry.file}/dubbo.cache"/>
<dubbo:provider timeout="120000" retries="0" registry="localAddress,remoteAddress"/>
<dubbo:consumer timeout="120000" retries="0" check="false" registry="localAddress,remoteAddress"/>
<dubbo:provider timeout="10000" retries="0" registry="${registry:remoteAddress}" threads="${jdbc.maxsize}"
queues="10000"
dispather="maintenance" loadbalance="customBalance"/>
<!--<bean id="brave" class="com.github.kristofa.brave.dubbo.BraveFactoryBean" p:serviceName="${dubbo.registry.file}" p:zipkinHost="http://123.207.187.158:9411/" p:rate="1.0" />-->
<dubbo:consumer timeout="10000" retries="0" check="false" registry="${registry:remoteAddress}"/>
</beans>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment