public/dataease
1
0
Fork 0
mirror of https://github.com/dataease/dataease.git synced 2026-06-17 04:51:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' into pr@dev@oracle

Browse Source
This commit is contained in:
taojinlong
2021-06-30 10:53:08 +08:00
parent 69321b08a0 ac06b6fa94
commit b57e71cee6
30 changed files with 498 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java
View File

@@ -2,9 +2,18 @@ package io.dataease.base.mapper.ext;
import io.dataease.controller.request.chart.ChartViewRequest;
import io.dataease.dto.chart.ChartViewDTO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import java.util.List;
@Mapper
public interface ExtChartViewMapper {
List<ChartViewDTO> search(ChartViewRequest request);
void chartCopy(@Param("newChartId")String newChartId,@Param("oldChartId")String oldChartId);
@Select("select id from chart_view where table_id = #{tableId}")
List<String> allViewIds(@Param("tableId") String tableId);
}

41
backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml
View File

@@ -27,4 +27,45 @@
</select>
<insert id="chartCopy">
INSERT INTO chart_view (
`id`,
`name`,
`scene_id`,
`table_id`,
`type`,
`title`,
`x_axis`,
`y_axis`,
`custom_attr`,
`custom_style`,
`custom_filter`,
`create_by`,
`create_time`,
`update_time`,
`snapshot`,
`style_priority`
) SELECT
#{newChartId},
GET_CHART_VIEW_COPY_NAME ( #{oldChartId} ),
`scene_id`,
`table_id`,
`type`,
`title`,
`x_axis`,
`y_axis`,
`custom_attr`,
`custom_style`,
`custom_filter`,
`create_by`,
`create_time`,
`update_time`,
`snapshot`,
`style_priority`
FROM
chart_view
WHERE
id = #{oldChartId}
</insert>
</mapper>

6
backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java Normal file
View File

@@ -0,0 +1,6 @@
package io.dataease.commons.constants;
public class JdbcConstants {
public final static String VIEW_CACHE_KEY = "view_cache";
}

5
backend/src/main/java/io/dataease/controller/chart/ChartViewController.java
View File

@@ -55,4 +55,9 @@ public class ChartViewController {
public Map<String, Object> chartDetail(@PathVariable String id) {
return chartViewService.getChartDetail(id);
}
@PostMapping("chartCopy/{id}")
public String chartCopy(@PathVariable String id) {
return chartViewService.chartCopy(id);
}
}

22
backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java
View File

@@ -11,7 +11,6 @@ import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.exception.DataEaseException;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import java.beans.PropertyVetoException;
import java.sql.*;
import java.util.*;
@@ -23,14 +22,29 @@ public class JdbcProvider extends DatasourceProvider {
private static int initPoolSize = 1;
private static int maxConnections = 1;
/**
* 增加缓存机制 key 由 'provider_sql_' dsr.datasource.id dsr.table dsr.query共4部分组成，命中则使用缓存直接返回不再执行sql逻辑
* @param dsr
* @return
* @throws Exception
*/
/**
* 这里使用声明式缓存不是很妥当
* 改为chartViewService中使用编程式缓存
@Cacheable(
value = JdbcConstants.JDBC_PROVIDER_KEY,
key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query",
condition = "#dsr.pageSize == null || #dsr.pageSize == 0L"
)
*/
@Override
public List<String[]> getData(DatasourceRequest datasourceRequest) throws Exception {
public List<String[]> getData(DatasourceRequest dsr) throws Exception {
List<String[]> list = new LinkedList<>();
Connection connection = null;
try {
connection = getConnectionFromPool(datasourceRequest);
connection = getConnectionFromPool(dsr);
Statement stat = connection.createStatement();
ResultSet rs = stat.executeQuery(datasourceRequest.getQuery());
ResultSet rs = stat.executeQuery(dsr.getQuery());
list = fetchResult(rs);
} catch (SQLException e) {
DataEaseException.throwException(e);

7
backend/src/main/java/io/dataease/listener/util/CacheUtils.java
View File

@@ -24,18 +24,19 @@ public class CacheUtils {
return element.getObjectValue();
}
private static void put(String cacheName, Object key, Object value, Integer ttl, Integer tti) {
public static void put(String cacheName, Object key, Object value, Integer ttl, Integer tti) {
Element e = new Element(key, value);
//不设置则使用xml配置
if (ttl != null)
if (ttl != null) {
e.setEternal(false);
e.setTimeToLive(ttl);
}
if (tti != null)
e.setTimeToIdle(tti);
cache(cacheName).put(e);
}
private static boolean remove(String cacheName, Object key) {
public static boolean remove(String cacheName, Object key) {
return cache(cacheName).remove(key);
}

4
backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java
View File

@@ -237,7 +237,7 @@ public class DorisQueryProvider extends QueryProvider {
String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}",
"(" + sql + ") AS tmp",
StringUtils.join(resultFilter, " "),
StringUtils.join(yOrder, ","));
ObjectUtils.isNotEmpty(yOrder) ? StringUtils.join(yOrder, ",") : "null");
return filterSql;
}
}
@@ -318,7 +318,7 @@ public class DorisQueryProvider extends QueryProvider {
String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}",
"(" + sql + ") AS tmp",
StringUtils.join(resultFilter, " "),
StringUtils.join(order, ","));
ObjectUtils.isNotEmpty(order) ? StringUtils.join(order, ",") : "null");
return filterSql;
}
}

4
backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java
View File

@@ -245,7 +245,7 @@ public class MysqlQueryProvider extends QueryProvider {
String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}",
"(" + sql + ") AS tmp",
StringUtils.join(resultFilter, " "),
StringUtils.join(yOrder, ","));
ObjectUtils.isNotEmpty(yOrder) ? StringUtils.join(yOrder, ",") : "null");
return filterSql;
}
}
@@ -326,7 +326,7 @@ public class MysqlQueryProvider extends QueryProvider {
String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}",
"(" + sql + ") AS tmp",
StringUtils.join(resultFilter, " "),
StringUtils.join(order, ","));
ObjectUtils.isNotEmpty(order) ? StringUtils.join(order, ",") : "null");
return filterSql;
}
}

74
backend/src/main/java/io/dataease/service/chart/ChartViewService.java
View File

@@ -6,6 +6,7 @@ import io.dataease.base.domain.*;
import io.dataease.base.mapper.ChartViewMapper;
import io.dataease.base.mapper.ext.ExtChartGroupMapper;
import io.dataease.base.mapper.ext.ExtChartViewMapper;
import io.dataease.commons.constants.JdbcConstants;
import io.dataease.commons.utils.AuthUtils;
import io.dataease.commons.utils.BeanUtils;
import io.dataease.commons.utils.CommonBeanFactory;
@@ -20,6 +21,7 @@ import io.dataease.datasource.service.DatasourceService;
import io.dataease.dto.chart.*;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.i18n.Translator;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.QueryProvider;
import io.dataease.service.dataset.DataSetTableFieldsService;
import io.dataease.service.dataset.DataSetTableService;
@@ -32,6 +34,7 @@ import javax.annotation.Resource;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.*;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
/**
@@ -53,6 +56,9 @@ public class ChartViewService {
@Resource
private ExtChartGroupMapper extChartGroupMapper;
//默认使用非公平
private ReentrantLock lock = new ReentrantLock();
public ChartViewWithBLOBs save(ChartViewWithBLOBs chartView) {
checkName(chartView);
long timestamp = System.currentTimeMillis();
@@ -65,6 +71,10 @@ public class ChartViewService {
chartView.setUpdateTime(timestamp);
chartViewMapper.insertSelective(chartView);
}
Optional.ofNullable(chartView.getId()).ifPresent(id -> {
CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, id);
});
return chartView;
}
@@ -188,6 +198,17 @@ public class ChartViewService {
}
}
data = datasourceProvider.getData(datasourceRequest);
/**
* 直连不实用缓存
String key = "provider_sql_"+datasourceRequest.getDatasource().getId() + "_" + datasourceRequest.getTable() + "_" +datasourceRequest.getQuery();
Object cache;
if ((cache = CacheUtils.get(JdbcConstants.JDBC_PROVIDER_KEY, key)) == null) {
data = datasourceProvider.getData(datasourceRequest);
CacheUtils.put(JdbcConstants.JDBC_PROVIDER_KEY,key ,data, null, null);
}else {
data = (List<String[]>) cache;
}
*/
} else if (table.getMode() == 1) {// 抽取
// 连接doris，构建doris数据源查询
Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource");
@@ -202,7 +223,23 @@ public class ChartViewService {
} else {
datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, customFilter, extFilterList));
}
data = datasourceProvider.getData(datasourceRequest);
/*// 定时抽取使用缓存
Object cache;
// 仪表板有参数不实用缓存
if (CollectionUtils.isNotEmpty(requestList.getFilter())) {
data = datasourceProvider.getData(datasourceRequest);
}
// 仪表板无参数 且 未缓存过该视图 则查询后缓存
else if ((cache = CacheUtils.get(JdbcConstants.VIEW_CACHE_KEY, id)) == null) {
lock.lock();
data = datasourceProvider.getData(datasourceRequest);
CacheUtils.put(JdbcConstants.VIEW_CACHE_KEY, id, data, null, null);
}
// 仪表板有缓存 使用缓存
else {
data = (List<String[]>) cache;
}*/
data = cacheViewData(datasourceProvider, datasourceRequest, id);
}
if (StringUtils.containsIgnoreCase(view.getType(), "pie") && data.size() > 1000) {
data = data.subList(0, 1000);
@@ -269,6 +306,35 @@ public class ChartViewService {
return dto;
}
/**
* 避免缓存击穿
* 虽然流量不一定能够达到击穿的水平
* @param datasourceProvider
* @param datasourceRequest
* @param viewId
* @return
* @throws Exception
*/
public List<String[]> cacheViewData(DatasourceProvider datasourceProvider, DatasourceRequest datasourceRequest, String viewId) throws Exception{
List<String[]> result ;
Object cache = CacheUtils.get(JdbcConstants.VIEW_CACHE_KEY, viewId);
if (cache == null) {
if (lock.tryLock()) {// 获取锁成功
result = datasourceProvider.getData(datasourceRequest);
if (result != null) {
CacheUtils.put(JdbcConstants.VIEW_CACHE_KEY, viewId, result, null, null);
}
lock.unlock();
}else {//获取锁失败
Thread.sleep(100);//避免CAS自旋频率过大 占用cpu资源过高
result = cacheViewData(datasourceProvider, datasourceRequest, viewId);
}
}else {
result = (List<String[]>)cache;
}
return result;
}
private void checkName(ChartViewWithBLOBs chartView) {
// if (StringUtils.isEmpty(chartView.getId())) {
// return;
@@ -310,4 +376,10 @@ public class ChartViewService {
public ChartViewWithBLOBs findOne(String id) {
return chartViewMapper.selectByPrimaryKey(id);
}
public String chartCopy(String id) {
String newChartId = UUID.randomUUID().toString();
extChartViewMapper.chartCopy(newChartId,id);
return newChartId;
}
}

46
backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java
View File

@@ -5,11 +5,14 @@ import io.dataease.base.domain.*;
import io.dataease.base.mapper.DatasetTableMapper;
import io.dataease.base.mapper.DatasetTableTaskMapper;
import io.dataease.base.mapper.DatasourceMapper;
import io.dataease.base.mapper.ext.ExtChartViewMapper;
import io.dataease.commons.constants.JdbcConstants;
import io.dataease.commons.constants.JobStatus;
import io.dataease.commons.constants.ScheduleType;
import io.dataease.commons.constants.UpdateType;
import io.dataease.commons.utils.CommonBeanFactory;
import io.dataease.commons.utils.DorisTableUtils;
import io.dataease.commons.utils.HttpClientUtil;
import io.dataease.commons.utils.LogUtil;
import io.dataease.datasource.constants.DatasourceTypes;
import io.dataease.datasource.dto.*;
@@ -19,6 +22,7 @@ import io.dataease.datasource.provider.ProviderFactory;
import io.dataease.datasource.request.DatasourceRequest;
import io.dataease.dto.dataset.DataTableInfoDTO;
import io.dataease.exception.DataEaseException;
import io.dataease.listener.util.CacheUtils;
import io.dataease.provider.QueryProvider;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.io.FileUtils;
@@ -27,6 +31,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.poi.hssf.usermodel.HSSFSheet;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.xssf.usermodel.XSSFSheet;
@@ -59,6 +64,7 @@ import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassDef;
import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassMeta;
import org.pentaho.di.www.SlaveServerJobStatus;
import org.quartz.JobExecutionContext;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
@@ -93,6 +99,9 @@ public class ExtractDataService {
@Resource
private DatasetTableTaskMapper datasetTableTaskMapper;
@Resource
private ExtChartViewMapper extChartViewMapper;
private static String lastUpdateTime = "${__last_update_time__}";
private static String currentUpdateTime = "${__current_update_time__}";
private static String separator = "|DE|";
@@ -274,7 +283,15 @@ public class ExtractDataService {
}
}
break;
}
}
//侵入式清除下属视图缓存
List<String> viewIds = extChartViewMapper.allViewIds(datasetTableId);
if (CollectionUtils.isNotEmpty(viewIds)){
viewIds.forEach(viewId -> {
CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, viewId);
});
}
}
private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed, Long execTime) {
@@ -840,16 +857,19 @@ public class ExtractDataService {
}
public boolean isKettleRunning() {
try {
if (!InetAddress.getByName(carte).isReachable(1000)) {
return false;
}
}catch (Exception e){
return false;
}
HttpGet getMethod = new HttpGet("http://" + carte + ":" + port);
HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder();
clientBuilder.setConnectionTimeout(1);
clientBuilder.setCredentials(user, passwd);
CloseableHttpClient httpClient = clientBuilder.build();
try {
if (!InetAddress.getByName(carte).isReachable(1000)) {
return false;
}
HttpClient httpClient;
HttpGet getMethod = new HttpGet("http://" + carte + ":" + port);
HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder();
clientBuilder.setConnectionTimeout(1);
clientBuilder.setCredentials(user, passwd);
httpClient = clientBuilder.build();
HttpResponse httpResponse = httpClient.execute(getMethod);
int statusCode = httpResponse.getStatusLine().getStatusCode();
if (statusCode != -1 && statusCode < 400) {
@@ -859,6 +879,12 @@ public class ExtractDataService {
}
} catch (Exception e) {
return false;
}finally {
try {
httpClient.close();
} catch (Exception e) {
LoggerFactory.getLogger(HttpClientUtil.class).error("HttpClient关闭连接失败", e);
}
}
}