diff --git a/backend/pom.xml b/backend/pom.xml index 1a0ad78d0b..500999f194 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -355,7 +355,7 @@ https://github.com/dataease/dataease-plugins--> io.dataease - dataease-plugin-xpack + dataease-plugin-interface 1.0 diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java b/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java index fd823e2637..89ae4b74d8 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java @@ -2,9 +2,18 @@ package io.dataease.base.mapper.ext; import io.dataease.controller.request.chart.ChartViewRequest; import io.dataease.dto.chart.ChartViewDTO; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; import java.util.List; +@Mapper public interface ExtChartViewMapper { List search(ChartViewRequest request); + + void chartCopy(@Param("newChartId")String newChartId,@Param("oldChartId")String oldChartId); + + @Select("select id from chart_view where table_id = #{tableId}") + List allViewIds(@Param("tableId") String tableId); } diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml b/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml index 13fdcb64e6..042283e2f9 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.xml @@ -27,4 +27,45 @@ + + + INSERT INTO chart_view ( + `id`, + `name`, + `scene_id`, + `table_id`, + `type`, + `title`, + `x_axis`, + `y_axis`, + `custom_attr`, + `custom_style`, + `custom_filter`, + `create_by`, + `create_time`, + `update_time`, + `snapshot`, + `style_priority` + ) SELECT + #{newChartId}, + GET_CHART_VIEW_COPY_NAME ( #{oldChartId} ), + `scene_id`, + `table_id`, + `type`, + `title`, + `x_axis`, + `y_axis`, + `custom_attr`, + `custom_style`, + `custom_filter`, + `create_by`, + `create_time`, + `update_time`, + `snapshot`, + `style_priority` + FROM + chart_view + WHERE + id = #{oldChartId} + diff --git a/backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java b/backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java new file mode 100644 index 0000000000..d617122096 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java @@ -0,0 +1,6 @@ +package io.dataease.commons.constants; + +public class JdbcConstants { + + public final static String VIEW_CACHE_KEY = "view_cache"; +} diff --git a/backend/src/main/java/io/dataease/controller/chart/ChartViewController.java b/backend/src/main/java/io/dataease/controller/chart/ChartViewController.java index 207d64b188..7d0260887e 100644 --- a/backend/src/main/java/io/dataease/controller/chart/ChartViewController.java +++ b/backend/src/main/java/io/dataease/controller/chart/ChartViewController.java @@ -55,4 +55,9 @@ public class ChartViewController { public Map chartDetail(@PathVariable String id) { return chartViewService.getChartDetail(id); } + + @PostMapping("chartCopy/{id}") + public String chartCopy(@PathVariable String id) { + return chartViewService.chartCopy(id); + } } diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index 04a87ebde9..e4a389f19a 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -11,7 +11,6 @@ import io.dataease.datasource.request.DatasourceRequest; import io.dataease.exception.DataEaseException; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; - import java.beans.PropertyVetoException; import java.sql.*; import java.util.*; @@ -23,14 +22,29 @@ public class JdbcProvider extends DatasourceProvider { private static int initPoolSize = 1; private static int maxConnections = 1; + /** + * 增加缓存机制 key 由 'provider_sql_' dsr.datasource.id dsr.table dsr.query共4部分组成，命中则使用缓存直接返回不再执行sql逻辑 + * @param dsr + * @return + * @throws Exception + */ + /** + * 这里使用声明式缓存不是很妥当 + * 改为chartViewService中使用编程式缓存 + @Cacheable( + value = JdbcConstants.JDBC_PROVIDER_KEY, + key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query", + condition = "#dsr.pageSize == null || #dsr.pageSize == 0L" + ) + */ @Override - public List getData(DatasourceRequest datasourceRequest) throws Exception { + public List getData(DatasourceRequest dsr) throws Exception { List list = new LinkedList<>(); Connection connection = null; try { - connection = getConnectionFromPool(datasourceRequest); + connection = getConnectionFromPool(dsr); Statement stat = connection.createStatement(); - ResultSet rs = stat.executeQuery(datasourceRequest.getQuery()); + ResultSet rs = stat.executeQuery(dsr.getQuery()); list = fetchResult(rs); } catch (SQLException e) { DataEaseException.throwException(e); diff --git a/backend/src/main/java/io/dataease/listener/util/CacheUtils.java b/backend/src/main/java/io/dataease/listener/util/CacheUtils.java index 09e656315a..443a9dc3fd 100644 --- a/backend/src/main/java/io/dataease/listener/util/CacheUtils.java +++ b/backend/src/main/java/io/dataease/listener/util/CacheUtils.java @@ -24,18 +24,19 @@ public class CacheUtils { return element.getObjectValue(); } - private static void put(String cacheName, Object key, Object value, Integer ttl, Integer tti) { + public static void put(String cacheName, Object key, Object value, Integer ttl, Integer tti) { Element e = new Element(key, value); //不设置则使用xml配置 - if (ttl != null) + if (ttl != null) { e.setEternal(false); e.setTimeToLive(ttl); + } if (tti != null) e.setTimeToIdle(tti); cache(cacheName).put(e); } - private static boolean remove(String cacheName, Object key) { + public static boolean remove(String cacheName, Object key) { return cache(cacheName).remove(key); } diff --git a/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java b/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java index 00bf32f724..aaedfd0524 100644 --- a/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/doris/DorisQueryProvider.java @@ -237,7 +237,7 @@ public class DorisQueryProvider extends QueryProvider { String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}", "(" + sql + ") AS tmp", StringUtils.join(resultFilter, " "), - StringUtils.join(yOrder, ",")); + ObjectUtils.isNotEmpty(yOrder) ? StringUtils.join(yOrder, ",") : "null"); return filterSql; } } @@ -318,7 +318,7 @@ public class DorisQueryProvider extends QueryProvider { String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}", "(" + sql + ") AS tmp", StringUtils.join(resultFilter, " "), - StringUtils.join(order, ",")); + ObjectUtils.isNotEmpty(order) ? StringUtils.join(order, ",") : "null"); return filterSql; } } diff --git a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java index 4feaad6579..1b6362b326 100644 --- a/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java +++ b/backend/src/main/java/io/dataease/provider/mysql/MysqlQueryProvider.java @@ -245,7 +245,7 @@ public class MysqlQueryProvider extends QueryProvider { String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}", "(" + sql + ") AS tmp", StringUtils.join(resultFilter, " "), - StringUtils.join(yOrder, ",")); + ObjectUtils.isNotEmpty(yOrder) ? StringUtils.join(yOrder, ",") : "null"); return filterSql; } } @@ -326,7 +326,7 @@ public class MysqlQueryProvider extends QueryProvider { String filterSql = MessageFormat.format("SELECT * FROM {0} WHERE 1=1 {1} ORDER BY {2}", "(" + sql + ") AS tmp", StringUtils.join(resultFilter, " "), - StringUtils.join(order, ",")); + ObjectUtils.isNotEmpty(order) ? StringUtils.join(order, ",") : "null"); return filterSql; } } diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 1828eccdba..7f233d6fa5 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -6,6 +6,7 @@ import io.dataease.base.domain.*; import io.dataease.base.mapper.ChartViewMapper; import io.dataease.base.mapper.ext.ExtChartGroupMapper; import io.dataease.base.mapper.ext.ExtChartViewMapper; +import io.dataease.commons.constants.JdbcConstants; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.BeanUtils; import io.dataease.commons.utils.CommonBeanFactory; @@ -20,6 +21,7 @@ import io.dataease.datasource.service.DatasourceService; import io.dataease.dto.chart.*; import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.i18n.Translator; +import io.dataease.listener.util.CacheUtils; import io.dataease.provider.QueryProvider; import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableService; @@ -32,6 +34,7 @@ import javax.annotation.Resource; import java.math.BigDecimal; import java.math.RoundingMode; import java.util.*; +import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; /** @@ -53,6 +56,9 @@ public class ChartViewService { @Resource private ExtChartGroupMapper extChartGroupMapper; + //默认使用非公平 + private ReentrantLock lock = new ReentrantLock(); + public ChartViewWithBLOBs save(ChartViewWithBLOBs chartView) { checkName(chartView); long timestamp = System.currentTimeMillis(); @@ -65,6 +71,10 @@ public class ChartViewService { chartView.setUpdateTime(timestamp); chartViewMapper.insertSelective(chartView); } + Optional.ofNullable(chartView.getId()).ifPresent(id -> { + CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, id); + }); + return chartView; } @@ -188,6 +198,17 @@ public class ChartViewService { } } data = datasourceProvider.getData(datasourceRequest); + /** + * 直连不实用缓存 + String key = "provider_sql_"+datasourceRequest.getDatasource().getId() + "_" + datasourceRequest.getTable() + "_" +datasourceRequest.getQuery(); + Object cache; + if ((cache = CacheUtils.get(JdbcConstants.JDBC_PROVIDER_KEY, key)) == null) { + data = datasourceProvider.getData(datasourceRequest); + CacheUtils.put(JdbcConstants.JDBC_PROVIDER_KEY,key ,data, null, null); + }else { + data = (List) cache; + } + */ } else if (table.getMode() == 1) {// 抽取 // 连接doris，构建doris数据源查询 Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); @@ -202,7 +223,23 @@ public class ChartViewService { } else { datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, customFilter, extFilterList)); } - data = datasourceProvider.getData(datasourceRequest); + /*// 定时抽取使用缓存 + Object cache; + // 仪表板有参数不实用缓存 + if (CollectionUtils.isNotEmpty(requestList.getFilter())) { + data = datasourceProvider.getData(datasourceRequest); + } + // 仪表板无参数 且 未缓存过该视图 则查询后缓存 + else if ((cache = CacheUtils.get(JdbcConstants.VIEW_CACHE_KEY, id)) == null) { + lock.lock(); + data = datasourceProvider.getData(datasourceRequest); + CacheUtils.put(JdbcConstants.VIEW_CACHE_KEY, id, data, null, null); + } + // 仪表板有缓存 使用缓存 + else { + data = (List) cache; + }*/ + data = cacheViewData(datasourceProvider, datasourceRequest, id); } if (StringUtils.containsIgnoreCase(view.getType(), "pie") && data.size() > 1000) { data = data.subList(0, 1000); @@ -269,6 +306,35 @@ public class ChartViewService { return dto; } + /** + * 避免缓存击穿 + * 虽然流量不一定能够达到击穿的水平 + * @param datasourceProvider + * @param datasourceRequest + * @param viewId + * @return + * @throws Exception + */ + public List cacheViewData(DatasourceProvider datasourceProvider, DatasourceRequest datasourceRequest, String viewId) throws Exception{ + List result ; + Object cache = CacheUtils.get(JdbcConstants.VIEW_CACHE_KEY, viewId); + if (cache == null) { + if (lock.tryLock()) {// 获取锁成功 + result = datasourceProvider.getData(datasourceRequest); + if (result != null) { + CacheUtils.put(JdbcConstants.VIEW_CACHE_KEY, viewId, result, null, null); + } + lock.unlock(); + }else {//获取锁失败 + Thread.sleep(100);//避免CAS自旋频率过大 占用cpu资源过高 + result = cacheViewData(datasourceProvider, datasourceRequest, viewId); + } + }else { + result = (List)cache; + } + return result; + } + private void checkName(ChartViewWithBLOBs chartView) { // if (StringUtils.isEmpty(chartView.getId())) { // return; @@ -310,4 +376,10 @@ public class ChartViewService { public ChartViewWithBLOBs findOne(String id) { return chartViewMapper.selectByPrimaryKey(id); } + + public String chartCopy(String id) { + String newChartId = UUID.randomUUID().toString(); + extChartViewMapper.chartCopy(newChartId,id); + return newChartId; + } } diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 35c827ef67..d5a39f7b38 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -5,11 +5,14 @@ import io.dataease.base.domain.*; import io.dataease.base.mapper.DatasetTableMapper; import io.dataease.base.mapper.DatasetTableTaskMapper; import io.dataease.base.mapper.DatasourceMapper; +import io.dataease.base.mapper.ext.ExtChartViewMapper; +import io.dataease.commons.constants.JdbcConstants; import io.dataease.commons.constants.JobStatus; import io.dataease.commons.constants.ScheduleType; import io.dataease.commons.constants.UpdateType; import io.dataease.commons.utils.CommonBeanFactory; import io.dataease.commons.utils.DorisTableUtils; +import io.dataease.commons.utils.HttpClientUtil; import io.dataease.commons.utils.LogUtil; import io.dataease.datasource.constants.DatasourceTypes; import io.dataease.datasource.dto.*; @@ -19,6 +22,7 @@ import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.exception.DataEaseException; +import io.dataease.listener.util.CacheUtils; import io.dataease.provider.QueryProvider; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; @@ -27,6 +31,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFSheet; @@ -59,6 +64,7 @@ import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassDef; import org.pentaho.di.trans.steps.userdefinedjavaclass.UserDefinedJavaClassMeta; import org.pentaho.di.www.SlaveServerJobStatus; import org.quartz.JobExecutionContext; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; @@ -93,6 +99,9 @@ public class ExtractDataService { @Resource private DatasetTableTaskMapper datasetTableTaskMapper; + @Resource + private ExtChartViewMapper extChartViewMapper; + private static String lastUpdateTime = "${__last_update_time__}"; private static String currentUpdateTime = "${__current_update_time__}"; private static String separator = "|DE|"; @@ -274,7 +283,15 @@ public class ExtractDataService { } } break; - } + } + //侵入式清除下属视图缓存 + List viewIds = extChartViewMapper.allViewIds(datasetTableId); + if (CollectionUtils.isNotEmpty(viewIds)){ + viewIds.forEach(viewId -> { + CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, viewId); + }); + } + } private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed, Long execTime) { @@ -840,16 +857,19 @@ public class ExtractDataService { } public boolean isKettleRunning() { + try { + if (!InetAddress.getByName(carte).isReachable(1000)) { + return false; + } + }catch (Exception e){ + return false; + } + HttpGet getMethod = new HttpGet("http://" + carte + ":" + port); + HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder(); + clientBuilder.setConnectionTimeout(1); + clientBuilder.setCredentials(user, passwd); + CloseableHttpClient httpClient = clientBuilder.build(); try { - if (!InetAddress.getByName(carte).isReachable(1000)) { - return false; - } - HttpClient httpClient; - HttpGet getMethod = new HttpGet("http://" + carte + ":" + port); - HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder(); - clientBuilder.setConnectionTimeout(1); - clientBuilder.setCredentials(user, passwd); - httpClient = clientBuilder.build(); HttpResponse httpResponse = httpClient.execute(getMethod); int statusCode = httpResponse.getStatusLine().getStatusCode(); if (statusCode != -1 && statusCode < 400) { @@ -859,6 +879,12 @@ public class ExtractDataService { } } catch (Exception e) { return false; + }finally { + try { + httpClient.close(); + } catch (Exception e) { + LoggerFactory.getLogger(HttpClientUtil.class).error("HttpClient关闭连接失败", e); + } } } diff --git a/backend/src/main/resources/db/migration/V9__chart_view_copy.sql b/backend/src/main/resources/db/migration/V9__chart_view_copy.sql new file mode 100644 index 0000000000..0cbcb569c9 --- /dev/null +++ b/backend/src/main/resources/db/migration/V9__chart_view_copy.sql @@ -0,0 +1,25 @@ +DROP FUNCTION IF EXISTS `GET_CHART_VIEW_COPY_NAME`; +delimiter ;; +CREATE DEFINER=`root`@`%` FUNCTION `GET_CHART_VIEW_COPY_NAME`(chartId varchar(255)) RETURNS varchar(255) CHARSET utf8 + READS SQL DATA +BEGIN + +DECLARE chartName varchar(255); + +DECLARE pid varchar(255); + +DECLARE regexpInfo varchar(255); + +DECLARE chartNameCount INTEGER; + +select `name` ,`scene_id` into chartName, pid from chart_view where id =chartId; + +set regexpInfo = concat('^',chartName,'-copy','\\(([0-9])+\\)$'); + +select (count(1)+1) into chartNameCount from chart_view where name REGEXP regexpInfo; + +RETURN concat(chartName,'-copy(',chartNameCount,')'); + +END +;; +delimiter ; diff --git a/backend/src/main/resources/ehcache/ehcache.xml b/backend/src/main/resources/ehcache/ehcache.xml index a85f1cbf10..8f3bf770f0 100644 --- a/backend/src/main/resources/ehcache/ehcache.xml +++ b/backend/src/main/resources/ehcache/ehcache.xml @@ -82,5 +82,17 @@ + + \ No newline at end of file diff --git a/frontend/src/api/chart/chart.js b/frontend/src/api/chart/chart.js index ab528b0aaa..81b76ef1eb 100644 --- a/frontend/src/api/chart/chart.js +++ b/frontend/src/api/chart/chart.js @@ -27,3 +27,11 @@ export function getChartTree(data) { data }) } + +export function chartCopy(id) { + return request({ + url: '/chart/view/chartCopy/' + id, + method: 'post', + loading: true + }) +} diff --git a/frontend/src/components/canvas/components/Editor/ContextMenu.vue b/frontend/src/components/canvas/components/Editor/ContextMenu.vue index 85c189b244..bf2f0d42e0 100644 --- a/frontend/src/components/canvas/components/Editor/ContextMenu.vue +++ b/frontend/src/components/canvas/components/Editor/ContextMenu.vue @@ -4,7 +4,7 @@