Skip to content
Projects
Groups
Snippets
Help
This project
Loading...
Sign in / Register
Toggle navigation
G
gic-cloud
Overview
Overview
Details
Activity
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
data-hook
gic-cloud
Commits
70e937d2
Commit
70e937d2
authored
Sep 10, 2021
by
陶光胜
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
取数平台调整
parent
0cad84ad
Hide whitespace changes
Inline
Side-by-side
Showing
15 changed files
with
260 additions
and
34 deletions
+260
-34
dubbo.cache
...loud-data-hook-service/dubbo/local/gic-member/dubbo.cache
+8
-7
DataInitUtils.java
...n/java/com/gic/cloud/data/hook/service/DataInitUtils.java
+51
-0
HiveHelper.java
...main/java/com/gic/cloud/data/hook/service/HiveHelper.java
+38
-0
FlatQueryResultServiceImpl.java
...ud/data/hook/service/impl/FlatQueryResultServiceImpl.java
+15
-6
FreeQueryServiceImpl.java
...ic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
+2
-4
data-hook-flat-query-source.xml
...ervice/src/main/resources/data-hook-flat-query-source.xml
+24
-0
pom.xml
gic-cloud-data-hook/pom.xml
+3
-3
CustomBalance.java
...n/java/com/gic/cloud/data/hook/balance/CustomBalance.java
+17
-0
CustomRouter.java
...ain/java/com/gic/cloud/data/hook/router/CustomRouter.java
+59
-0
RouterFactory.java
...in/java/com/gic/cloud/data/hook/router/RouterFactory.java
+12
-0
com.alibaba.dubbo.rpc.cluster.LoadBalance
.../META-INF/dubbo/com.alibaba.dubbo.rpc.cluster.LoadBalance
+2
-0
com.alibaba.dubbo.rpc.cluster.RouterFactory
...ETA-INF/dubbo/com.alibaba.dubbo.rpc.cluster.RouterFactory
+2
-0
applicationContext-conf.xml
...-hook/src/main/webapp/WEB-INF/applicationContext-conf.xml
+2
-2
data-hook-dubbo-config.xml
...a-hook/src/main/webapp/WEB-INF/data-hook-dubbo-config.xml
+1
-2
data-hook-dubbo-settings.xml
...hook/src/main/webapp/WEB-INF/data-hook-dubbo-settings.xml
+24
-10
No files found.
gic-cloud-data-hook-service/dubbo/local/gic-member/dubbo.cache
View file @
70e937d2
#Dubbo Registry Cache
#Dubbo Registry Cache
#Wed Sep 16 09:40:39 CST 2020
#Thu Sep 09 18:47:36 CST 2021
com.gic.cloud.data.hook.api.service.SearchLogService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.SearchLogService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.SearchLogService&methods\=saveLog&payload\=41557050&pid\=16652&side\=provider&timeout\=120000×tamp\=1600140918276
com.gic.cloud.data.hook.api.service.IDownloadTaskService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IDownloadTaskService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IDownloadTaskService&methods\=createRiskModeRecord,createDownloadRecord,getRiskModeRecordPage,listUnDownloadTask,getDownloadRecordPage,deleteDownloadTask,getDownloadProcess,getDownloadTaskPage&payload\=41557050&pid\=47060&side\=provider&timeout\=120000×tamp\=1631184455577
com.gic.cloud.data.hook.api.service.IDownloadTaskService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IDownloadTaskService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IDownloadTaskService&methods\=createRiskModeRecord,listUnDownloadTask,createDownloadRecord,getRiskModeRecordPage,getDownloadRecordPage,deleteDownloadTask,getDownloadProcess,getDownloadTaskPage&payload\=41557050&pid\=16652&side\=provider&timeout\=120000×tamp\=1600140918147
com.gic.cloud.data.hook.api.service.SearchLogService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.SearchLogService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.SearchLogService&methods\=saveLog&payload\=41557050&pid\=47060&side\=provider&timeout\=120000×tamp\=1631184455823
com.gic.cloud.data.hook.api.service.IFlatQueryTableService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryTableService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryTableService&methods\=queryFlatQueryFavo,queryFlatQueryTableById,queryFlatQueryTablePage,createFlatQueryFavo,removeFlatQueryFavo,queryFlatQueryTableDetailByTableId&payload\=41557050&pid\=16652&side\=provider&timeout\=120000×tamp\=1600140918020
com.gic.cloud.data.hook.api.service.IFlatQueryTableService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryTableService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryTableService&methods\=queryFlatQueryFavo,queryFlatQueryTableById,queryFlatQueryTablePage,createFlatQueryFavo,removeFlatQueryFavo,queryFlatQueryTableDetailByTableId&payload\=41557050&pid\=47060&side\=provider&timeout\=120000×tamp\=1631184455351
com.gic.cloud.data.hook.api.service.IFlatQueryResultService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryResultService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryResultService&methods\=getFlatQueryFilterData,initTask,estimateFlatQueryDownloadTask,buildFlatQueryDownloadTask,getFlatQueryResult&payload\=41557050&pid\=16652&side\=provider&timeout\=120000×tamp\=1600140918093
com.gic.cloud.data.hook.api.service.TableSyncRecordService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.TableSyncRecordService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.TableSyncRecordService&methods\=lastSyncDateTime&payload\=41557050&pid\=47060&side\=provider×tamp\=1631184456001
com.gic.cloud.data.hook.api.service.IMyFlatQueryService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IMyFlatQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IMyFlatQueryService&methods\=createMyFlatQuery,queryMyFlatQueryPage,deleteMyFlatQuery,updateMyFlatQuery,queryMyFlatQueryById&payload\=41557050&pid\=16652&side\=provider&timeout\=120000×tamp\=1600140918216
com.gic.cloud.data.hook.api.service.IFlatQueryResultService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFlatQueryResultService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFlatQueryResultService&methods\=getFlatQueryFilterData,initTask,estimateFlatQueryDownloadTask,buildFlatQueryDownloadTask,getFlatQueryResult&payload\=41557050&pid\=47060&side\=provider&timeout\=120000×tamp\=1631184455489
com.gic.cloud.data.hook.api.service.IFreeQueryService=empty\://192.168.1.217\:20338/com.gic.cloud.data.hook.api.service.IFreeQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFreeQueryService&methods\=getFreeQueryRecordByUserIdAndRecordId,createFreeQueryRecord,deleteFreeQueryRecord,getColumnsByTable,buildFreeQueryDownloadTask,updateFreeQueryRecord,getFreeQueryResult,initTask,getFreeQuerySource,getFreeQueryCount,getFreeQueryRecordListByUserId,getFreeQueryTableList&payload\=41557050&pid\=16652&side\=provider&timeout\=120000×tamp\=1600140917339
com.gic.cloud.data.hook.api.service.IMyFlatQueryService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IMyFlatQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IMyFlatQueryService&methods\=queryMyFlatQueryPage,createMyFlatQuery,deleteMyFlatQuery,updateMyFlatQuery,queryMyFlatQueryById&payload\=41557050&pid\=47060&side\=provider&timeout\=120000×tamp\=1631184455740
com.gic.cloud.data.hook.api.service.IFreeQueryService=empty\://172.16.40.36\:20338/com.gic.cloud.data.hook.api.service.IFreeQueryService?anyhost\=true&application\=gic-cloud-data-hook&category\=configurators&check\=false&default.dispatcher\=maintenance&default.queues\=10000&default.retries\=0&default.threads\=30&default.timeout\=10000&dubbo\=2.8.5&generic\=false&interface\=com.gic.cloud.data.hook.api.service.IFreeQueryService&methods\=getFreeQueryRecordByUserIdAndRecordId,createFreeQueryRecord,buildFreeQueryDownloadTask,deleteFreeQueryRecord,getColumnsByTable,updateFreeQueryRecord,getFreeQueryResult,getFreeQuerySource,initTask,getFreeQueryCount,getFreeQueryRecordListByUserId,getFreeQueryTableList&payload\=41557050&pid\=47060&side\=provider&timeout\=120000×tamp\=1631184454144
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/DataInitUtils.java
0 → 100644
View file @
70e937d2
package
com
.
gic
.
cloud
.
data
.
hook
.
service
;
import
cn.medubi.client.utils.LogPak
;
import
com.ctrip.framework.apollo.Config
;
import
com.ctrip.framework.apollo.ConfigService
;
import
com.gic.cloud.data.hook.api.dto.DownloadRecord
;
import
com.gic.cloud.data.hook.service.impl.FlatQueryResultServiceImpl
;
import
java.net.InetAddress
;
import
java.util.ArrayList
;
import
java.util.List
;
public
class
DataInitUtils
{
private
static
LogPak
log
=
new
LogPak
(
DataInitUtils
.
class
);
public
static
List
<
DownloadRecord
>
listByHost
(
List
<
DownloadRecord
>
allList
){
List
<
DownloadRecord
>
list
=
new
ArrayList
<>();
Config
config
=
ConfigService
.
getConfig
(
"application"
);
String
customRouterHost
=
config
.
getProperty
(
"customRouterHost"
,
""
);
String
customRouterEnterpriseId
=
config
.
getProperty
(
"customRouterEnterpriseId"
,
""
);
String
host
=
getHost
();
log
.
debug
(
"查询本机的内网ip"
,
host
);
for
(
DownloadRecord
downloadRecord
:
allList
){
if
(
host
.
equals
(
customRouterHost
)){
if
(
customRouterEnterpriseId
.
equals
(
downloadRecord
.
getEnterpriseId
())){
list
.
add
(
downloadRecord
);
}
}
else
{
if
(!
customRouterEnterpriseId
.
equals
(
downloadRecord
.
getEnterpriseId
())){
list
.
add
(
downloadRecord
);
}
}
}
return
list
;
}
private
static
String
getHost
(){
try
{
String
hostAddress
=
InetAddress
.
getLocalHost
().
getHostAddress
();
return
hostAddress
;
}
catch
(
Exception
e
){
e
.
printStackTrace
();
}
return
""
;
}
public
static
void
main
(
String
[]
args
){
System
.
out
.
println
(
"getHost() = "
+
getHost
());
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/HiveHelper.java
View file @
70e937d2
...
@@ -2,6 +2,8 @@ package com.gic.cloud.data.hook.service;
...
@@ -2,6 +2,8 @@ package com.gic.cloud.data.hook.service;
import
cn.medubi.client.utils.LogPak
;
import
cn.medubi.client.utils.LogPak
;
import
com.alibaba.fastjson.JSON
;
import
com.alibaba.fastjson.JSON
;
import
com.ctrip.framework.apollo.Config
;
import
com.ctrip.framework.apollo.ConfigService
;
import
org.springframework.beans.BeansException
;
import
org.springframework.beans.BeansException
;
import
org.springframework.context.ApplicationContext
;
import
org.springframework.context.ApplicationContext
;
import
org.springframework.context.ApplicationContextAware
;
import
org.springframework.context.ApplicationContextAware
;
...
@@ -11,6 +13,7 @@ import javax.sql.DataSource;
...
@@ -11,6 +13,7 @@ import javax.sql.DataSource;
import
java.sql.Connection
;
import
java.sql.Connection
;
import
java.sql.DriverManager
;
import
java.sql.DriverManager
;
import
java.sql.SQLException
;
import
java.sql.SQLException
;
import
java.util.List
;
/** Hive 辅助工具类
/** Hive 辅助工具类
* @author Sam
* @author Sam
...
@@ -28,12 +31,15 @@ public class HiveHelper implements ApplicationContextAware {
...
@@ -28,12 +31,15 @@ public class HiveHelper implements ApplicationContextAware {
private
static
DataSource
balaDownloadSource
=
null
;
private
static
DataSource
balaDownloadSource
=
null
;
private
static
DataSource
balaSearchSource
=
null
;
@Override
@Override
public
void
setApplicationContext
(
ApplicationContext
applicationContext
)
throws
BeansException
{
public
void
setApplicationContext
(
ApplicationContext
applicationContext
)
throws
BeansException
{
log
.
debug
(
"setApplicationContext"
,
"准备初始化 Hive 连接池"
);
log
.
debug
(
"setApplicationContext"
,
"准备初始化 Hive 连接池"
);
source
=
(
DataSource
)
applicationContext
.
getBean
(
"hiveSource"
);
source
=
(
DataSource
)
applicationContext
.
getBean
(
"hiveSource"
);
downloadSource
=
(
DataSource
)
applicationContext
.
getBean
(
"downloadHiveSource"
);
downloadSource
=
(
DataSource
)
applicationContext
.
getBean
(
"downloadHiveSource"
);
balaDownloadSource
=
(
DataSource
)
applicationContext
.
getBean
(
"balaDownloadHiveSource"
);
balaDownloadSource
=
(
DataSource
)
applicationContext
.
getBean
(
"balaDownloadHiveSource"
);
balaSearchSource
=
(
DataSource
)
applicationContext
.
getBean
(
"balaSearchHiveSource"
);
}
}
/** 获取 Hive 数据源
/** 获取 Hive 数据源
...
@@ -113,4 +119,36 @@ public class HiveHelper implements ApplicationContextAware {
...
@@ -113,4 +119,36 @@ public class HiveHelper implements ApplicationContextAware {
}
// TRY CATCH OVER
}
// TRY CATCH OVER
}
}
public
static
Connection
getBalaSearchHiveConnection
()
{
Connection
conn
=
null
;
try
{
// String url = "jdbc:hive2://115.159.205.44:10015/data_test";
// //String url = "jdbc:hive2://10.0.0.3:10015/data_test";
// Class.forName("org.apache.hive.jdbc.HiveDriver");
// conn = DriverManager.getConnection(url, "hadoop", "");
conn
=
balaSearchSource
.
getConnection
();
return
conn
;
}
catch
(
Exception
ex
)
{
ex
.
printStackTrace
();
if
(
conn
!=
null
)
{
try
{
conn
.
close
();
}
catch
(
SQLException
e
)
{
e
.
printStackTrace
();
}
}
return
null
;
}
// TRY CATCH OVER
}
public
static
Connection
getSearchHiveConnection
(
List
<
String
>
enterpriseIds
){
Config
config
=
ConfigService
.
getConfig
(
"application"
);
String
customRouterEnterpriseId
=
config
.
getProperty
(
"customRouterEnterpriseId"
,
""
);
if
(
enterpriseIds
.
contains
(
customRouterEnterpriseId
)){
return
HiveHelper
.
getBalaSearchHiveConnection
();
}
else
{
return
HiveHelper
.
getHiveConnection
();
}
}
}
}
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FlatQueryResultServiceImpl.java
View file @
70e937d2
...
@@ -2,15 +2,14 @@ package com.gic.cloud.data.hook.service.impl;
...
@@ -2,15 +2,14 @@ package com.gic.cloud.data.hook.service.impl;
import
cn.medubi.client.utils.LogPak
;
import
cn.medubi.client.utils.LogPak
;
import
com.alibaba.fastjson.JSON
;
import
com.alibaba.fastjson.JSON
;
import
com.ctrip.framework.apollo.Config
;
import
com.ctrip.framework.apollo.ConfigService
;
import
com.gic.cloud.common.api.base.Page
;
import
com.gic.cloud.common.api.base.Page
;
import
com.gic.cloud.data.hook.api.dto.*
;
import
com.gic.cloud.data.hook.api.dto.*
;
import
com.gic.cloud.data.hook.api.entity.*
;
import
com.gic.cloud.data.hook.api.entity.*
;
import
com.gic.cloud.data.hook.api.service.IDownloadTaskService
;
import
com.gic.cloud.data.hook.api.service.IDownloadTaskService
;
import
com.gic.cloud.data.hook.api.service.IFlatQueryResultService
;
import
com.gic.cloud.data.hook.api.service.IFlatQueryResultService
;
import
com.gic.cloud.data.hook.service.DecryptUtils
;
import
com.gic.cloud.data.hook.service.*
;
import
com.gic.cloud.data.hook.service.HiveHelper
;
import
com.gic.cloud.data.hook.service.HttpUtils
;
import
com.gic.cloud.data.hook.service.ServiceUtils
;
import
com.gic.cloud.data.hook.service.dao.FlatQueryTableDao
;
import
com.gic.cloud.data.hook.service.dao.FlatQueryTableDao
;
import
com.gic.cloud.data.hook.service.entity.CsvDataFilterMode
;
import
com.gic.cloud.data.hook.service.entity.CsvDataFilterMode
;
import
com.gic.cloud.data.hook.service.entity.CsvResultSetHelper
;
import
com.gic.cloud.data.hook.service.entity.CsvResultSetHelper
;
...
@@ -36,6 +35,7 @@ import java.io.*;
...
@@ -36,6 +35,7 @@ import java.io.*;
import
java.math.BigDecimal
;
import
java.math.BigDecimal
;
import
java.nio.charset.Charset
;
import
java.nio.charset.Charset
;
import
java.sql.*
;
import
java.sql.*
;
import
java.text.ParseException
;
import
java.text.SimpleDateFormat
;
import
java.text.SimpleDateFormat
;
import
java.util.*
;
import
java.util.*
;
import
java.util.Date
;
import
java.util.Date
;
...
@@ -203,7 +203,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -203,7 +203,8 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
limitRange
,
limitRange
,
authStoreIdList
);
authStoreIdList
);
// 进行查询
// 进行查询
Connection
conn
=
HiveHelper
.
getHiveConnection
();
//获取连接根据企业判断,巴拉走单独的端口
Connection
conn
=
HiveHelper
.
getSearchHiveConnection
(
enterpriseIds
);
if
(
conn
!=
null
)
{
if
(
conn
!=
null
)
{
Date
startTime
=
new
Date
();
Date
startTime
=
new
Date
();
try
{
try
{
...
@@ -370,7 +371,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -370,7 +371,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
0
,
0
,
authStoreIdList
);
authStoreIdList
);
String
finQuery
=
"SELECT COUNT(1) FROM ("
+
preQuery
+
") AS tmpTable"
;
String
finQuery
=
"SELECT COUNT(1) FROM ("
+
preQuery
+
") AS tmpTable"
;
Connection
conn
=
HiveHelper
.
get
HiveConnection
(
);
Connection
conn
=
HiveHelper
.
get
SearchHiveConnection
(
enterpriseIds
);
if
(
conn
!=
null
)
{
if
(
conn
!=
null
)
{
try
{
try
{
Statement
stat
=
conn
.
createStatement
();
Statement
stat
=
conn
.
createStatement
();
...
@@ -476,6 +477,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -476,6 +477,7 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
public
void
initTask
(){
public
void
initTask
(){
try
{
try
{
List
<
DownloadRecord
>
list
=
this
.
downloadTaskService
.
listUnDownloadTask
(
QueryDataSource
.
FLAT_QUERY
);
List
<
DownloadRecord
>
list
=
this
.
downloadTaskService
.
listUnDownloadTask
(
QueryDataSource
.
FLAT_QUERY
);
list
=
DataInitUtils
.
listByHost
(
list
);
if
(
CollectionUtils
.
isNotEmpty
(
list
)){
if
(
CollectionUtils
.
isNotEmpty
(
list
)){
for
(
DownloadRecord
record
:
list
){
for
(
DownloadRecord
record
:
list
){
if
(
StringUtils
.
isNotBlank
(
record
.
getDownloadCondition
())){
if
(
StringUtils
.
isNotBlank
(
record
.
getDownloadCondition
())){
...
@@ -505,6 +507,13 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
...
@@ -505,6 +507,13 @@ public class FlatQueryResultServiceImpl implements IFlatQueryResultService {
private
static
SimpleDateFormat
timeFormatter
=
new
SimpleDateFormat
(
"HH:mm:ss"
);
private
static
SimpleDateFormat
timeFormatter
=
new
SimpleDateFormat
(
"HH:mm:ss"
);
public
static
void
main
(
String
[]
args
)
throws
ParseException
{
String
date
=
"2021-09-01"
;
Date
parse
=
dateFormatter
.
parse
(
date
);
String
format
=
datetimeFormatter
.
format
(
parse
);
System
.
out
.
println
(
"date = "
+
format
);
}
/** 下载任务执行计时器 */
/** 下载任务执行计时器 */
//private Timer downloadTaskTimer = new Timer();
//private Timer downloadTaskTimer = new Timer();
ScheduledExecutorService
downloadService
=
new
ScheduledThreadPoolExecutor
(
1
,
ScheduledExecutorService
downloadService
=
new
ScheduledThreadPoolExecutor
(
1
,
...
...
gic-cloud-data-hook-service/src/main/java/com/gic/cloud/data/hook/service/impl/FreeQueryServiceImpl.java
View file @
70e937d2
...
@@ -7,10 +7,7 @@ import com.gic.cloud.data.hook.api.dto.DataDesensiType;
...
@@ -7,10 +7,7 @@ import com.gic.cloud.data.hook.api.dto.DataDesensiType;
import
com.gic.cloud.data.hook.api.entity.*
;
import
com.gic.cloud.data.hook.api.entity.*
;
import
com.gic.cloud.data.hook.api.service.IDownloadTaskService
;
import
com.gic.cloud.data.hook.api.service.IDownloadTaskService
;
import
com.gic.cloud.data.hook.api.service.IFreeQueryService
;
import
com.gic.cloud.data.hook.api.service.IFreeQueryService
;
import
com.gic.cloud.data.hook.service.HiveHelper
;
import
com.gic.cloud.data.hook.service.*
;
import
com.gic.cloud.data.hook.service.HttpUtils
;
import
com.gic.cloud.data.hook.service.MysqlHelper
;
import
com.gic.cloud.data.hook.service.ServiceUtils
;
import
com.gic.cloud.data.hook.service.dao.FreeQueryRecordDao
;
import
com.gic.cloud.data.hook.service.dao.FreeQueryRecordDao
;
import
com.gic.cloud.data.hook.service.dao.FreeQuerySourceDao
;
import
com.gic.cloud.data.hook.service.dao.FreeQuerySourceDao
;
import
com.gic.cloud.data.hook.service.entity.CsvDataFilterMode
;
import
com.gic.cloud.data.hook.service.entity.CsvDataFilterMode
;
...
@@ -327,6 +324,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
...
@@ -327,6 +324,7 @@ public class FreeQueryServiceImpl implements IFreeQueryService {
public
void
initTask
()
{
public
void
initTask
()
{
try
{
try
{
List
<
DownloadRecord
>
list
=
this
.
downloadTaskService
.
listUnDownloadTask
(
QueryDataSource
.
FREE_QUERY
);
List
<
DownloadRecord
>
list
=
this
.
downloadTaskService
.
listUnDownloadTask
(
QueryDataSource
.
FREE_QUERY
);
list
=
DataInitUtils
.
listByHost
(
list
);
if
(
CollectionUtils
.
isNotEmpty
(
list
))
{
if
(
CollectionUtils
.
isNotEmpty
(
list
))
{
for
(
DownloadRecord
record
:
list
)
{
for
(
DownloadRecord
record
:
list
)
{
if
(
record
.
getDesensiType
()
!=
null
){
if
(
record
.
getDesensiType
()
!=
null
){
...
...
gic-cloud-data-hook-service/src/main/resources/data-hook-flat-query-source.xml
View file @
70e937d2
...
@@ -81,4 +81,27 @@
...
@@ -81,4 +81,27 @@
</property>
</property>
</bean>
</bean>
<bean
class=
"org.apache.commons.dbcp.BasicDataSource"
id=
"balaSearchHiveSource"
destroy-method=
"close"
>
<property
name=
"driverClassName"
value=
"org.apache.hive.jdbc.HiveDriver"
/>
<!--<property name="url" value="jdbc:hive2://115.159.205.44:10015/data_test" />-->
<property
name=
"url"
value=
"${hive.balasearch.url}"
/>
<property
name=
"username"
value=
"${hive.username}"
/>
<property
name=
"password"
value=
""
/>
<property
name=
"maxActive"
>
<value>
20
</value>
</property>
<property
name=
"maxIdle"
>
<value>
5
</value>
</property>
<property
name=
"minIdle"
>
<value>
1
</value>
</property>
<property
name=
"testWhileIdle"
>
<value>
true
</value>
</property>
<property
name=
"maxWait"
>
<value>
120000
</value>
</property>
</bean>
</beans>
</beans>
\ No newline at end of file
gic-cloud-data-hook/pom.xml
View file @
70e937d2
...
@@ -99,8 +99,8 @@
...
@@ -99,8 +99,8 @@
<artifactId>
maven-compiler-plugin
</artifactId>
<artifactId>
maven-compiler-plugin
</artifactId>
<version>
3.1
</version>
<version>
3.1
</version>
<configuration>
<configuration>
<source>
1.
7
</source>
<source>
1.
8
</source>
<target>
1.
7
</target>
<target>
1.
8
</target>
<encoding>
UTF-8
</encoding>
<encoding>
UTF-8
</encoding>
<compilerArguments>
<compilerArguments>
<verbose/>
<verbose/>
...
@@ -113,7 +113,7 @@
...
@@ -113,7 +113,7 @@
<artifactId>
tomcat7-maven-plugin
</artifactId>
<artifactId>
tomcat7-maven-plugin
</artifactId>
<version>
2.1
</version>
<version>
2.1
</version>
<configuration>
<configuration>
<port>
9
090
</port>
<port>
9
191
</port>
<path>
/
</path>
<path>
/
</path>
<uriEncoding>
UTF-8
</uriEncoding>
<uriEncoding>
UTF-8
</uriEncoding>
<finalName>
my-proxy
</finalName>
<finalName>
my-proxy
</finalName>
...
...
gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/balance/CustomBalance.java
0 → 100644
View file @
70e937d2
package
com
.
gic
.
cloud
.
data
.
hook
.
balance
;
import
com.alibaba.dubbo.common.URL
;
import
com.alibaba.dubbo.rpc.Invocation
;
import
com.alibaba.dubbo.rpc.Invoker
;
import
com.alibaba.dubbo.rpc.RpcException
;
import
com.alibaba.dubbo.rpc.cluster.LoadBalance
;
import
java.util.List
;
public
class
CustomBalance
implements
LoadBalance
{
@Override
public
<
T
>
Invoker
<
T
>
select
(
List
<
Invoker
<
T
>>
invokers
,
URL
url
,
Invocation
invocation
)
throws
RpcException
{
System
.
out
.
println
(
"dubbo负载"
);
return
invokers
.
get
(
0
);
}
}
gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/router/CustomRouter.java
0 → 100644
View file @
70e937d2
package
com
.
gic
.
cloud
.
data
.
hook
.
router
;
import
com.alibaba.dubbo.common.URL
;
import
com.alibaba.dubbo.rpc.Invocation
;
import
com.alibaba.dubbo.rpc.Invoker
;
import
com.alibaba.dubbo.rpc.RpcException
;
import
com.alibaba.dubbo.rpc.cluster.Router
;
import
com.ctrip.framework.apollo.Config
;
import
com.ctrip.framework.apollo.ConfigService
;
import
com.gic.web.common.utils.SessionContextUtils
;
import
java.util.ArrayList
;
import
java.util.List
;
public
class
CustomRouter
implements
Router
{
private
URL
url
;
public
CustomRouter
(
URL
url
){
this
.
url
=
url
;
}
@Override
public
URL
getUrl
()
{
return
url
;
}
@Override
public
<
T
>
List
<
Invoker
<
T
>>
route
(
List
<
Invoker
<
T
>>
invokers
,
URL
url
,
Invocation
invocation
)
throws
RpcException
{
if
(
url
.
getParameter
(
"application"
,
""
).
equals
(
"gic-cloud-data-hook"
)){
String
enterpriseId
=
SessionContextUtils
.
getLoginUserEnterpriseId
();
Config
config
=
ConfigService
.
getConfig
(
"application"
);
String
customRouterEnterpriseId
=
config
.
getProperty
(
"customRouterEnterpriseId"
,
""
);
String
customRouterHost
=
config
.
getProperty
(
"customRouterHost"
,
""
);
List
<
Invoker
<
T
>>
list
=
new
ArrayList
<>();
if
(
enterpriseId
.
equals
(
customRouterEnterpriseId
)){
//路由到新服务器
for
(
Invoker
invoker
:
invokers
){
if
(
invoker
.
getUrl
().
getHost
().
equals
(
customRouterHost
)){
list
.
add
(
invoker
);
}
}
}
else
{
for
(
Invoker
invoker
:
invokers
){
if
(!
invoker
.
getUrl
().
getHost
().
equals
(
customRouterHost
)){
list
.
add
(
invoker
);
}
}
}
return
list
;
}
return
invokers
;
}
@Override
public
int
compareTo
(
Router
o
)
{
return
0
;
}
}
gic-cloud-data-hook/src/main/java/com/gic/cloud/data/hook/router/RouterFactory.java
0 → 100644
View file @
70e937d2
package
com
.
gic
.
cloud
.
data
.
hook
.
router
;
import
com.alibaba.dubbo.common.URL
;
import
com.alibaba.dubbo.common.extension.Activate
;
import
com.alibaba.dubbo.rpc.cluster.Router
;
public
class
RouterFactory
implements
com
.
alibaba
.
dubbo
.
rpc
.
cluster
.
RouterFactory
{
@Override
public
Router
getRouter
(
URL
url
)
{
return
new
CustomRouter
(
url
);
}
}
gic-cloud-data-hook/src/main/resources/META-INF/dubbo/com.alibaba.dubbo.rpc.cluster.LoadBalance
0 → 100644
View file @
70e937d2
customBalance=com.gic.cloud.data.hook.balance.CustomBalance
\ No newline at end of file
gic-cloud-data-hook/src/main/resources/META-INF/dubbo/com.alibaba.dubbo.rpc.cluster.RouterFactory
0 → 100644
View file @
70e937d2
customRouterFactory=com.gic.cloud.data.hook.router.RouterFactory
\ No newline at end of file
gic-cloud-data-hook/src/main/webapp/WEB-INF/applicationContext-conf.xml
View file @
70e937d2
...
@@ -7,8 +7,8 @@
...
@@ -7,8 +7,8 @@
<!--<import resource="classpath:data-hook-init.xml"/>--><!-- data-hook 所有项目使用的 dubbo 环境配置(位于 data-hook-api/resource) -->
<!--<import resource="classpath:data-hook-init.xml"/>--><!-- data-hook 所有项目使用的 dubbo 环境配置(位于 data-hook-api/resource) -->
<!--<import resource="classpath*:applicationContext-sharding-db.xml"/>-->
<!--<import resource="classpath*:applicationContext-sharding-db.xml"/>-->
<import
resource=
"classpath*:applicationContext-init.xml"
/>
<import
resource=
"classpath*:applicationContext-init.xml"
/>
<
import
resource=
"classpath*:dubbo-setting.xml"
/
>
<
!--<import resource="classpath*:dubbo-setting.xml"/>--
>
<!-- <import resource="data-hook-dubbo-settings.xml"/>--
><!-- data-hook 所有项目初始化参数加载(位于 data-hook-api/resource)-->
<import
resource=
"data-hook-dubbo-settings.xml"
/
>
<!-- data-hook 所有项目初始化参数加载(位于 data-hook-api/resource)-->
<import
resource=
"data-hook-dubbo-config.xml"
/>
<!-- data-hook web 项目使用的 dubbo 引用服务配置 -->
<import
resource=
"data-hook-dubbo-config.xml"
/>
<!-- data-hook web 项目使用的 dubbo 引用服务配置 -->
</beans>
</beans>
gic-cloud-data-hook/src/main/webapp/WEB-INF/data-hook-dubbo-config.xml
View file @
70e937d2
...
@@ -13,8 +13,7 @@
...
@@ -13,8 +13,7 @@
<!-- 应用名称 -->
<!-- 应用名称 -->
<dubbo:application
name=
"gic-cloud-data-hook"
/>
<dubbo:application
name=
"gic-cloud-data-hook"
/>
<!-- 外部服务端口 -->
<!-- 外部服务端口 -->
<dubbo:protocol
name=
"dubbo"
port=
"2181"
/>
<dubbo:protocol
name=
"dubbo"
port=
"2181"
></dubbo:protocol>
<!-- 引用的 Dubbo 服务 -->
<!-- 引用的 Dubbo 服务 -->
<dubbo:reference
interface=
"com.gic.cloud.data.hook.api.service.IFlatQueryTableService"
id=
"flatQueryTableService"
timeout=
"120000"
retries=
"0"
/>
<dubbo:reference
interface=
"com.gic.cloud.data.hook.api.service.IFlatQueryTableService"
id=
"flatQueryTableService"
timeout=
"120000"
retries=
"0"
/>
<dubbo:reference
interface=
"com.gic.cloud.data.hook.api.service.IFlatQueryResultService"
id=
"flatQueryResultService"
timeout=
"120000"
retries=
"0"
/>
<dubbo:reference
interface=
"com.gic.cloud.data.hook.api.service.IFlatQueryResultService"
id=
"flatQueryResultService"
timeout=
"120000"
retries=
"0"
/>
...
...
gic-cloud-data-hook/src/main/webapp/WEB-INF/data-hook-dubbo-settings.xml
View file @
70e937d2
...
@@ -2,19 +2,32 @@
...
@@ -2,19 +2,32 @@
<beans
xmlns=
"http://www.springframework.org/schema/beans"
<beans
xmlns=
"http://www.springframework.org/schema/beans"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xmlns:xsi=
"http://www.w3.org/2001/XMLSchema-instance"
xmlns:dubbo=
"http://code.alibabatech.com/schema/dubbo"
xmlns:dubbo=
"http://code.alibabatech.com/schema/dubbo"
xmlns:p=
"http://www.springframework.org/schema/p"
xmlns:apollo=
"http://www.ctrip.com/schema/apollo"
xsi:schemaLocation=
"http://www.springframework.org/schema/beans
xsi:schemaLocation=
"http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
http://www.springframework.org/schema/beans/spring-beans-2.5.xsd
http://code.alibabatech.com/schema/dubbo
http://code.alibabatech.com/schema/dubbo http://code.alibabatech.com/schema/dubbo/dubbo.xsd
http://code.alibabatech.com/schema/dubbo/dubbo.xsd"
>
http://www.ctrip.com/schema/apollo
http://www.ctrip.com/schema/apollo.xsd"
>
<!--<context:property-placeholder location="classpath:jdbc.properties" ignore-unresolvable="true"/>-->
<apollo:config
namespaces=
"COMMON.dubbo, COMMON.jdbc"
/>
<dubbo:monitor
protocol=
"registry"
/>
<dubbo:registry
address=
"${zookeeper_host}"
protocol=
"dubbo"
id=
"remoteAddress"
timeout=
"20000"
register=
"${shall_register:true}"
file=
"dubbo/remote/${dubbo.registry.file}/dubbo.cache"
default=
"true"
>
<dubbo:parameter
key=
"router"
value=
"customRouterFactory"
/>
</dubbo:registry>
<dubbo:registry
address=
"zookeeper://localhost:2181"
protocol=
"dubbo"
id=
"localAddress"
timeout=
"10000"
file=
"dubbo/local/${dubbo.registry.file}/dubbo.cache"
default=
"false"
>
<dubbo:parameter
key=
"router"
value=
"customRouterFactory"
/>
</dubbo:registry>
<!--<dubbo:registry address="zookeeper://123.207.187.158:2199" protocol="dubbo" id="remoteAddress" timeout="20000" file="${catalina.home}/dubbo/${dubbo.registry.file}/dubbo.cache"/>-->
<!--<dubbo:registry address="zookeeper://123.207.187.158:2199" protocol="dubbo" id="remoteAddress" register="false" timeout="120000" file="dubbo/remote/${dubbo.registry.file}/dubbo.cache"/>-->
<dubbo:provider
timeout=
"10000"
retries=
"0"
registry=
"${registry:remoteAddress}"
threads=
"${jdbc.maxsize}"
<dubbo:registry
address=
"zookeeper://localhost:2181"
protocol=
"dubbo"
id=
"localAddress"
timeout=
"120000"
file=
"dubbo/local/${dubbo.registry.file}/dubbo.cache"
/>
queues=
"10000"
<dubbo:registry
address=
"zookeeper://localhost:2181"
protocol=
"dubbo"
id=
"remoteAddress"
timeout=
"120000"
file=
"dubbo/local/${dubbo.registry.file}/dubbo.cache"
/>
dispather=
"maintenance"
loadbalance=
"customBalance"
/>
<dubbo:provider
timeout=
"120000"
retries=
"0"
registry=
"localAddress,remoteAddress"
/>
<dubbo:consumer
timeout=
"120000"
retries=
"0"
check=
"false"
registry=
"localAddress,remoteAddress"
/>
<
!--<bean id="brave" class="com.github.kristofa.brave.dubbo.BraveFactoryBean" p:serviceName="${dubbo.registry.file}" p:zipkinHost="http://123.207.187.158:9411/" p:rate="1.0" />--
>
<
dubbo:consumer
timeout=
"10000"
retries=
"0"
check=
"false"
registry=
"${registry:remoteAddress}"
/
>
</beans>
</beans>
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment