From 8602db5e18f54ce655d7da0a4f491ef40b6a4ec6 Mon Sep 17 00:00:00 2001 From: junjie Date: Mon, 28 Jun 2021 16:43:25 +0800 Subject: [PATCH 01/14] =?UTF-8?q?fix:=20=E6=95=B0=E6=8D=AE=E9=9B=86?= =?UTF-8?q?=E7=BC=96=E8=BE=91=E6=97=B6=EF=BC=8C=E4=B8=8D=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=BF=9E=E6=8E=A5=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/views/dataset/add/AddSQL.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/dataset/add/AddSQL.vue b/frontend/src/views/dataset/add/AddSQL.vue index fe06f910db..3114ad4cc4 100644 --- a/frontend/src/views/dataset/add/AddSQL.vue +++ b/frontend/src/views/dataset/add/AddSQL.vue @@ -30,7 +30,7 @@ - + From a4d512bb181de6fe95781fbfc1680b4f2aa76527 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Mon, 28 Jun 2021 16:45:48 +0800 Subject: [PATCH 02/14] =?UTF-8?q?feat:=20=E5=8E=BB=E9=99=A4=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E5=90=8E=E5=8F=B010s=E8=B6=85=E6=97=B6=E9=99=90?= =?UTF-8?q?=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/utils/request.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/utils/request.js b/frontend/src/utils/request.js index c23aa96c9f..ca7598913a 100644 --- a/frontend/src/utils/request.js +++ b/frontend/src/utils/request.js @@ -16,7 +16,7 @@ const LinkTokenKey = Config.LinkTokenKey const service = axios.create({ baseURL: process.env.VUE_APP_BASE_API, // url = base url + request url // withCredentials: true, // send cookies when cross-domain requests - timeout: 10000 // request timeout + timeout: 0 // request timeout }) // request interceptor From d840266320317e5797b95d07010f9f659b44ce80 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Mon, 28 Jun 2021 16:47:16 +0800 Subject: [PATCH 03/14] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=20=E6=8A=BD?= =?UTF-8?q?=E5=8F=96=E6=A8=A1=E5=BC=8F=E8=A7=86=E5=9B=BE=20=E5=90=8E?= =?UTF-8?q?=E5=8F=B0=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../base/mapper/ext/ExtChartViewMapper.java | 7 +++++ .../commons/constants/JdbcConstants.java | 6 +++++ .../datasource/provider/JdbcProvider.java | 22 ++++++++++++--- .../io/dataease/listener/util/CacheUtils.java | 7 ++--- .../service/chart/ChartViewService.java | 27 ++++++++++++++++++- .../service/dataset/ExtractDataService.java | 16 ++++++++++- .../src/main/resources/ehcache/ehcache.xml | 12 +++++++++ 7 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java diff --git a/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java b/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java index fd823e2637..190bb92296 100644 --- a/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java +++ b/backend/src/main/java/io/dataease/base/mapper/ext/ExtChartViewMapper.java @@ -2,9 +2,16 @@ package io.dataease.base.mapper.ext; import io.dataease.controller.request.chart.ChartViewRequest; import io.dataease.dto.chart.ChartViewDTO; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; import java.util.List; +@Mapper public interface ExtChartViewMapper { List search(ChartViewRequest request); + + @Select("select id from chart_view where table_id = #{tableId}") + List allViewIds(@Param("tableId") String tableId); } diff --git a/backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java b/backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java new file mode 100644 index 0000000000..d617122096 --- /dev/null +++ b/backend/src/main/java/io/dataease/commons/constants/JdbcConstants.java @@ -0,0 +1,6 @@ +package io.dataease.commons.constants; + +public class JdbcConstants { + + public final static String VIEW_CACHE_KEY = "view_cache"; +} diff --git a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java index e2d759c838..17ed6ae35a 100644 --- a/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java +++ b/backend/src/main/java/io/dataease/datasource/provider/JdbcProvider.java @@ -10,7 +10,6 @@ import io.dataease.datasource.request.DatasourceRequest; import io.dataease.exception.DataEaseException; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; - import java.beans.PropertyVetoException; import java.sql.*; import java.util.*; @@ -22,14 +21,29 @@ public class JdbcProvider extends DatasourceProvider { private static int initPoolSize = 5; private static int maxConnections = 200; + /** + * 增加缓存机制 key 由 'provider_sql_' dsr.datasource.id dsr.table dsr.query共4部分组成,命中则使用缓存直接返回不再执行sql逻辑 + * @param dsr + * @return + * @throws Exception + */ + /** + * 这里使用声明式缓存不是很妥当 + * 改为chartViewService中使用编程式缓存 + @Cacheable( + value = JdbcConstants.JDBC_PROVIDER_KEY, + key = "'provider_sql_' + #dsr.datasource.id + '_' + #dsr.table + '_' + #dsr.query", + condition = "#dsr.pageSize == null || #dsr.pageSize == 0L" + ) + */ @Override - public List getData(DatasourceRequest datasourceRequest) throws Exception { + public List getData(DatasourceRequest dsr) throws Exception { List list = new LinkedList<>(); Connection connection = null; try { - connection = getConnectionFromPool(datasourceRequest); + connection = getConnectionFromPool(dsr); Statement stat = connection.createStatement(); - ResultSet rs = stat.executeQuery(datasourceRequest.getQuery()); + ResultSet rs = stat.executeQuery(dsr.getQuery()); list = fetchResult(rs); } catch (SQLException e) { DataEaseException.throwException(e); diff --git a/backend/src/main/java/io/dataease/listener/util/CacheUtils.java b/backend/src/main/java/io/dataease/listener/util/CacheUtils.java index 09e656315a..443a9dc3fd 100644 --- a/backend/src/main/java/io/dataease/listener/util/CacheUtils.java +++ b/backend/src/main/java/io/dataease/listener/util/CacheUtils.java @@ -24,18 +24,19 @@ public class CacheUtils { return element.getObjectValue(); } - private static void put(String cacheName, Object key, Object value, Integer ttl, Integer tti) { + public static void put(String cacheName, Object key, Object value, Integer ttl, Integer tti) { Element e = new Element(key, value); //不设置则使用xml配置 - if (ttl != null) + if (ttl != null) { e.setEternal(false); e.setTimeToLive(ttl); + } if (tti != null) e.setTimeToIdle(tti); cache(cacheName).put(e); } - private static boolean remove(String cacheName, Object key) { + public static boolean remove(String cacheName, Object key) { return cache(cacheName).remove(key); } diff --git a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java index 1828eccdba..4e59577082 100644 --- a/backend/src/main/java/io/dataease/service/chart/ChartViewService.java +++ b/backend/src/main/java/io/dataease/service/chart/ChartViewService.java @@ -6,6 +6,7 @@ import io.dataease.base.domain.*; import io.dataease.base.mapper.ChartViewMapper; import io.dataease.base.mapper.ext.ExtChartGroupMapper; import io.dataease.base.mapper.ext.ExtChartViewMapper; +import io.dataease.commons.constants.JdbcConstants; import io.dataease.commons.utils.AuthUtils; import io.dataease.commons.utils.BeanUtils; import io.dataease.commons.utils.CommonBeanFactory; @@ -20,6 +21,7 @@ import io.dataease.datasource.service.DatasourceService; import io.dataease.dto.chart.*; import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.i18n.Translator; +import io.dataease.listener.util.CacheUtils; import io.dataease.provider.QueryProvider; import io.dataease.service.dataset.DataSetTableFieldsService; import io.dataease.service.dataset.DataSetTableService; @@ -65,6 +67,10 @@ public class ChartViewService { chartView.setUpdateTime(timestamp); chartViewMapper.insertSelective(chartView); } + Optional.ofNullable(chartView.getId()).ifPresent(id -> { + CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, id); + }); + return chartView; } @@ -188,6 +194,17 @@ public class ChartViewService { } } data = datasourceProvider.getData(datasourceRequest); + /** + * 直连不实用缓存 + String key = "provider_sql_"+datasourceRequest.getDatasource().getId() + "_" + datasourceRequest.getTable() + "_" +datasourceRequest.getQuery(); + Object cache; + if ((cache = CacheUtils.get(JdbcConstants.JDBC_PROVIDER_KEY, key)) == null) { + data = datasourceProvider.getData(datasourceRequest); + CacheUtils.put(JdbcConstants.JDBC_PROVIDER_KEY,key ,data, null, null); + }else { + data = (List) cache; + } + */ } else if (table.getMode() == 1) {// 抽取 // 连接doris,构建doris数据源查询 Datasource ds = (Datasource) CommonBeanFactory.getBean("DorisDatasource"); @@ -202,7 +219,15 @@ public class ChartViewService { } else { datasourceRequest.setQuery(qp.getSQL(tableName, xAxis, yAxis, customFilter, extFilterList)); } - data = datasourceProvider.getData(datasourceRequest); + // String key = "provider_sql_"+datasourceRequest.getDatasource().getId() + "_" + datasourceRequest.getTable() + "_" +datasourceRequest.getQuery(); + // 定时抽取使用缓存 + Object cache; + if ((cache = CacheUtils.get(JdbcConstants.VIEW_CACHE_KEY, id)) == null) { + data = datasourceProvider.getData(datasourceRequest); + CacheUtils.put(JdbcConstants.VIEW_CACHE_KEY, id, data, null, null); + }else { + data = (List) cache; + } } if (StringUtils.containsIgnoreCase(view.getType(), "pie") && data.size() > 1000) { data = data.subList(0, 1000); diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 7faa2b5a73..807a0df2c5 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -5,6 +5,8 @@ import io.dataease.base.domain.*; import io.dataease.base.mapper.DatasetTableMapper; import io.dataease.base.mapper.DatasetTableTaskMapper; import io.dataease.base.mapper.DatasourceMapper; +import io.dataease.base.mapper.ext.ExtChartViewMapper; +import io.dataease.commons.constants.JdbcConstants; import io.dataease.commons.constants.JobStatus; import io.dataease.commons.constants.ScheduleType; import io.dataease.commons.constants.UpdateType; @@ -22,6 +24,7 @@ import io.dataease.datasource.provider.ProviderFactory; import io.dataease.datasource.request.DatasourceRequest; import io.dataease.dto.dataset.DataTableInfoDTO; import io.dataease.exception.DataEaseException; +import io.dataease.listener.util.CacheUtils; import io.dataease.provider.QueryProvider; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; @@ -96,6 +99,9 @@ public class ExtractDataService { @Resource private DatasetTableTaskMapper datasetTableTaskMapper; + @Resource + private ExtChartViewMapper extChartViewMapper; + private static String lastUpdateTime = "${__last_update_time__}"; private static String currentUpdateTime = "${__current_update_time__}"; private static String separator = "|DE|"; @@ -277,7 +283,15 @@ public class ExtractDataService { } } break; - } + } + //侵入式清除下属视图缓存 + List viewIds = extChartViewMapper.allViewIds(datasetTableId); + if (CollectionUtils.isNotEmpty(viewIds)){ + viewIds.forEach(viewId -> { + CacheUtils.remove(JdbcConstants.VIEW_CACHE_KEY, viewId); + }); + } + } private void updateTableStatus(String datasetTableId, DatasetTable datasetTable, JobStatus completed, Long execTime) { diff --git a/backend/src/main/resources/ehcache/ehcache.xml b/backend/src/main/resources/ehcache/ehcache.xml index a85f1cbf10..8f3bf770f0 100644 --- a/backend/src/main/resources/ehcache/ehcache.xml +++ b/backend/src/main/resources/ehcache/ehcache.xml @@ -82,5 +82,17 @@ + + \ No newline at end of file From f312eea03f430209b739832b27d834a24a2807bf Mon Sep 17 00:00:00 2001 From: taojinlong Date: Mon, 28 Jun 2021 17:25:16 +0800 Subject: [PATCH 04/14] =?UTF-8?q?fix:=20kettle=20=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/io/dataease/service/dataset/ExtractDataService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java index 7faa2b5a73..6addfafe4c 100644 --- a/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java +++ b/backend/src/main/java/io/dataease/service/dataset/ExtractDataService.java @@ -30,6 +30,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.poi.hssf.usermodel.HSSFSheet; import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFSheet; @@ -824,7 +825,7 @@ public class ExtractDataService { if (!InetAddress.getByName(carte).isReachable(1000)) { return false; } - HttpClient httpClient; + CloseableHttpClient httpClient; HttpGet getMethod = new HttpGet("http://" + carte + ":" + port); HttpClientManager.HttpClientBuilderFacade clientBuilder = HttpClientManager.getInstance().createBuilder(); clientBuilder.setConnectionTimeout(1); From f098f651cafada232aef1e8842d96dacd3945820 Mon Sep 17 00:00:00 2001 From: junjie Date: Mon, 28 Jun 2021 18:03:33 +0800 Subject: [PATCH 05/14] =?UTF-8?q?feat:=20=E8=BD=B4=E7=BA=BF=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E7=BA=BF=E6=9D=A1=E5=92=8C=E6=A0=87=E7=AD=BE=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/lang/en.js | 12 ++++- frontend/src/lang/tw.js | 12 ++++- frontend/src/lang/zh.js | 12 ++++- frontend/src/views/chart/chart/chart.js | 22 ++++++++ .../src/views/chart/chart/common/common.js | 2 + .../component-style/XAxisSelector.vue | 54 ++++++++++++++++++- .../component-style/YAxisSelector.vue | 54 ++++++++++++++++++- frontend/src/views/chart/view/ChartEdit.vue | 4 +- 8 files changed, 165 insertions(+), 7 deletions(-) diff --git a/frontend/src/lang/en.js b/frontend/src/lang/en.js index 8d1f1a3634..35a3013b5f 100644 --- a/frontend/src/lang/en.js +++ b/frontend/src/lang/en.js @@ -720,7 +720,17 @@ export default { chart_error_tips: 'Please contact admin ', title_cannot_empty: 'Title can not be empty', table_title_height: 'Table header height', - table_item_height: 'Table row height' + table_item_height: 'Table row height', + axis_show: 'Axis Show', + axis_color: 'Axis Color', + axis_width: 'Axis Width', + axis_type: 'Axis Type', + axis_type_solid: 'Solid', + axis_type_dashed: 'Dashed', + axis_type_dotted: 'Dotted', + axis_label_show: 'Label Show', + axis_label_color: 'Label Color', + axis_label_fontsize: 'Label Fontsize' }, dataset: { sheet_warn: 'There are multiple sheet pages, and the first one is extracted by default', diff --git a/frontend/src/lang/tw.js b/frontend/src/lang/tw.js index 3a6d20f23d..e9efa24aa4 100644 --- a/frontend/src/lang/tw.js +++ b/frontend/src/lang/tw.js @@ -762,7 +762,17 @@ export default { chart_error_tips: '如有疑問請聯系管理員', title_cannot_empty: '標題不能為空', table_title_height: '表頭行高', - table_item_height: '表格行高' + table_item_height: '表格行高', + axis_show: '軸線顯示', + axis_color: '軸線顏色', + axis_width: '軸線寬度', + axis_type: '軸線類型', + axis_type_solid: '實線', + axis_type_dashed: '虛線', + axis_type_dotted: '點', + axis_label_show: '標簽顯示', + axis_label_color: '標簽顏色', + axis_label_fontsize: '標簽大小' }, dataset: { sheet_warn: '有多個sheet頁面,默認抽取第一個', diff --git a/frontend/src/lang/zh.js b/frontend/src/lang/zh.js index 3c24007e28..20ce1cabe3 100644 --- a/frontend/src/lang/zh.js +++ b/frontend/src/lang/zh.js @@ -720,7 +720,17 @@ export default { chart_error_tips: '如有疑问请联系管理员', title_cannot_empty: '标题不能为空', table_title_height: '表头行高', - table_item_height: '表格行高' + table_item_height: '表格行高', + axis_show: '轴线显示', + axis_color: '轴线颜色', + axis_width: '轴线宽度', + axis_type: '轴线类型', + axis_type_solid: '实线', + axis_type_dashed: '虚线', + axis_type_dotted: '点', + axis_label_show: '标签显示', + axis_label_color: '标签颜色', + axis_label_fontsize: '标签大小' }, dataset: { sheet_warn: '有多个 Sheet 页,默认抽取第一个', diff --git a/frontend/src/views/chart/chart/chart.js b/frontend/src/views/chart/chart/chart.js index 96be52b959..8f87726180 100644 --- a/frontend/src/views/chart/chart/chart.js +++ b/frontend/src/views/chart/chart/chart.js @@ -81,8 +81,19 @@ export const DEFAULT_XAXIS_STYLE = { position: 'bottom', name: '', axisLabel: { + show: true, + color: '#333333', + fontSize: '12', rotate: 0, formatter: '{value}' + }, + splitLine: { + show: false, + lineStyle: { + color: '#cccccc', + width: 1, + style: 'solid' + } } } export const DEFAULT_YAXIS_STYLE = { @@ -90,8 +101,19 @@ export const DEFAULT_YAXIS_STYLE = { position: 'left', name: '', axisLabel: { + show: true, + color: '#333333', + fontSize: '12', rotate: 0, formatter: '{value}' + }, + splitLine: { + show: true, + lineStyle: { + color: '#cccccc', + width: 1, + style: 'solid' + } } } export const DEFAULT_BACKGROUND_COLOR = { diff --git a/frontend/src/views/chart/chart/common/common.js b/frontend/src/views/chart/chart/common/common.js index b47e0b9fe5..f4e1d15b01 100644 --- a/frontend/src/views/chart/chart/common/common.js +++ b/frontend/src/views/chart/chart/common/common.js @@ -26,12 +26,14 @@ export function componentStyle(chart_option, chart) { chart_option.xAxis.position = customStyle.xAxis.position chart_option.xAxis.name = customStyle.xAxis.name chart_option.xAxis.axisLabel = customStyle.xAxis.axisLabel + chart_option.xAxis.splitLine = customStyle.xAxis.splitLine } if (customStyle.yAxis && (chart.type.includes('bar') || chart.type.includes('line'))) { chart_option.yAxis.show = customStyle.yAxis.show chart_option.yAxis.position = customStyle.yAxis.position chart_option.yAxis.name = customStyle.yAxis.name chart_option.yAxis.axisLabel = customStyle.yAxis.axisLabel + chart_option.yAxis.splitLine = customStyle.yAxis.splitLine } if (customStyle.background) { chart_option.backgroundColor = hexColorToRGBA(customStyle.background.color, customStyle.background.alpha) diff --git a/frontend/src/views/chart/components/component-style/XAxisSelector.vue b/frontend/src/views/chart/components/component-style/XAxisSelector.vue index 637f2355c6..d3b0e6c7ec 100644 --- a/frontend/src/views/chart/components/component-style/XAxisSelector.vue +++ b/frontend/src/views/chart/components/component-style/XAxisSelector.vue @@ -24,6 +24,40 @@ + + + {{ $t('chart.axis_show') }} + + + + + + + + + + + {{ $t('chart.axis_type_solid') }} + {{ $t('chart.axis_type_dashed') }} + {{ $t('chart.axis_type_dotted') }} + + + + + + {{ $t('chart.axis_label_show') }} + + + + + + + + + + + + @@ -58,7 +92,8 @@ export default { data() { return { axisForm: JSON.parse(JSON.stringify(DEFAULT_XAXIS_STYLE)), - isSetting: false + isSetting: false, + fontSize: [] } }, watch: { @@ -74,14 +109,28 @@ export default { } if (customStyle.xAxis) { this.axisForm = customStyle.xAxis + if (!this.axisForm.splitLine) { + this.axisForm.splitLine = JSON.parse(JSON.stringify(DEFAULT_XAXIS_STYLE.splitLine)) + } } } } } }, mounted() { + this.init() }, methods: { + init() { + const arr = [] + for (let i = 10; i <= 40; i = i + 2) { + arr.push({ + name: i + '', + value: i + '' + }) + } + this.fontSize = arr + }, changeXAxisStyle() { if (!this.axisForm.show) { this.isSetting = false @@ -93,6 +142,9 @@ export default {