public/dataease
1
0
Fork 0
mirror of https://github.com/dataease/dataease.git synced 2026-05-23 13:58:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev-v2' into pr@dev-v2@fixDS

Browse Source
This commit is contained in:
taojinlong
2024-08-23 09:48:18 +08:00
committed by GitHub
parent a1b1b78629 e308daa600
commit e95f1c7fee
66 changed files with 2352 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/core-backend/src/main/java/io/dataease/MybatisPlusGenerator.java
View File

@@ -25,7 +25,7 @@ public class MybatisPlusGenerator {
/**
* 这是要生成代码的表名称
*/
private static final String TABLE_NAME = "visualization_report_filter";
private static final String TABLE_NAME = "visualization_outer_params_target_view_info";
/**
* 下面两个配置基本上不用动

17
core/core-backend/src/main/java/io/dataease/chart/charts/impl/numeric/IndicatorHandler.java
View File

@@ -10,6 +10,7 @@ import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@Component
public class IndicatorHandler extends YoyChartHandler {
@@ -36,23 +37,19 @@ public class IndicatorHandler extends YoyChartHandler {
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis);
var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
ChartViewFieldDTO chartViewFieldDTO = yAxis.get(0);
ChartFieldCompareDTO compareCalc = chartViewFieldDTO.getCompareCalc();
ChartViewFieldDTO yAxisChartViewFieldDTO = yAxis.get(0);
ChartFieldCompareDTO compareCalc = yAxisChartViewFieldDTO.getCompareCalc();
boolean isYoy = org.apache.commons.lang3.StringUtils.isNotEmpty(compareCalc.getType())
&& !org.apache.commons.lang3.StringUtils.equalsIgnoreCase(compareCalc.getType(), "none");
if (isYoy) {
xAxis.clear();
// 设置维度字段，从同环比中获取用户选择的字段
xAxis.addAll(allFields.stream().filter(i -> org.springframework.util.StringUtils.endsWithIgnoreCase(i.getId().toString(), yAxis.get(0).getCompareCalc().getField().toString())).toList());
xAxis.addAll(allFields.stream().filter(i -> org.springframework.util.StringUtils.endsWithIgnoreCase(i.getId().toString(), compareCalc.getField().toString())).toList());
xAxis.get(0).setSort("desc");
if (org.springframework.util.StringUtils.endsWithIgnoreCase("month_mom", compareCalc.getType())) {
xAxis.get(0).setDateStyle("y_M");
}
if (org.springframework.util.StringUtils.endsWithIgnoreCase("day_mom", compareCalc.getType())) {
if(Objects.isNull(compareCalc.getCustom())){
xAxis.get(0).setDateStyle("y_M_d");
}
if (org.springframework.util.StringUtils.endsWithIgnoreCase("year_mom", compareCalc.getType())) {
xAxis.get(0).setDateStyle("y");
}else{
xAxis.get(0).setDateStyle(compareCalc.getCustom().getTimeType());
}
}
formatResult.getAxisMap().put(ChartAxis.xAxis, xAxis);

86
core/core-backend/src/main/java/io/dataease/chart/charts/impl/table/TableHeatmapHandler.java Normal file
View File

@@ -0,0 +1,86 @@
package io.dataease.chart.charts.impl.table;
import io.dataease.chart.charts.impl.DefaultChartHandler;
import io.dataease.chart.utils.ChartDataBuild;
import io.dataease.engine.sql.SQLProvider;
import io.dataease.engine.trans.Dimension2SQLObj;
import io.dataease.engine.trans.Quota2SQLObj;
import io.dataease.engine.utils.Utils;
import io.dataease.extensions.datasource.dto.DatasourceRequest;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.model.SQLMeta;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.util.ChartDataUtil;
import io.dataease.extensions.view.util.FieldUtil;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@Component
public class TableHeatmapHandler extends DefaultChartHandler {
@Getter
private String type = "t-heatmap";
@Override
public AxisFormatResult formatAxis(ChartViewDTO view) {
var result = super.formatAxis(view);
var xAxis = new ArrayList<ChartViewFieldDTO>(view.getXAxis());
xAxis.addAll(view.getXAxisExt());
xAxis.addAll(view.getExtColor());
result.getAxisMap().put(ChartAxis.xAxis, xAxis);
return result;
}
public Map<String, Object> buildResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, List<String[]> data) {
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
Map<String, Object> result = ChartDataBuild.transChartData( xAxis, new ArrayList<>(), view, data, false);
return result;
}
@Override
public <T extends ChartCalcDataResult> T calcChartResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map<String, Object> sqlMap, SQLMeta sqlMeta, Provider provider) {
var dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
List<String> dsList = new ArrayList<>();
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap);
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis);
List<ChartViewFieldDTO> yFields = new ArrayList<>();
yFields.addAll(chartViewManege.transFieldDTO(Collections.singletonList(chartViewManege.createCountField(view.getTableId()))));
yAxis.addAll(yFields);
xAxis = xAxis.stream().filter(i-> !StringUtils.equalsIgnoreCase(i.getDataeaseName(),yAxis.get(0).getDataeaseName())).toList();
var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage);
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage);
String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.debug("calcite chart sql: " + querySql);
List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");
//自定义排序
data = ChartDataUtil.resultCustomSort(xAxis, data);
//数据重组逻辑可重载
var result = this.buildResult(view, formatResult, filterResult, data);
T calcResult = (T) new ChartCalcDataResult();
calcResult.setData(result);
calcResult.setContext(filterResult.getContext());
calcResult.setQuerySql(querySql);
calcResult.setOriginData(data);
return calcResult;
}
}

39
core/core-backend/src/main/java/io/dataease/chart/dao/auto/entity/CoreChartView.java
View File

@@ -137,7 +137,7 @@ public class CoreChartView implements Serializable {
private Long updateTime;
/**
* 缩略图
* 缩略图
*/
private String snapshot;
@@ -206,6 +206,16 @@ public class CoreChartView implements Serializable {
*/
private Boolean aggregate;
private String flowMapStartName;
private String flowMapEndName;
/**
* 颜色维度field
*/
private String extColor;
public Long getId() {
return id;
}
@@ -510,6 +520,30 @@ public class CoreChartView implements Serializable {
this.aggregate = aggregate;
}
public String getFlowMapStartName() {
return flowMapStartName;
}
public void setFlowMapStartName(String flowMapStartName) {
this.flowMapStartName = flowMapStartName;
}
public String getFlowMapEndName() {
return flowMapEndName;
}
public void setFlowMapEndName(String flowMapEndName) {
this.flowMapEndName = flowMapEndName;
}
public String getExtColor() {
return extColor;
}
public void setExtColor(String extColor) {
this.extColor = extColor;
}
@Override
public String toString() {
return "CoreChartView{" +
@@ -551,6 +585,9 @@ public class CoreChartView implements Serializable {
", copyFrom = " + copyFrom +
", copyId = " + copyId +
", aggregate = " + aggregate +
", flowMapStartName=" + flowMapStartName +
", flowMapEndName=" + flowMapEndName +
", extColor=" + extColor +
"}";
}
}

48
core/core-backend/src/main/java/io/dataease/chart/manage/ChartDataManage.java
View File

@@ -144,6 +144,7 @@ public class ChartDataManage {
// 过滤来自仪表板的条件
List<ChartExtFilterDTO> extFilterList = new ArrayList<>();
//组件过滤条件
List<SqlVariableDetails> sqlVariables = datasetGroupManage.getSqlParams(Collections.singletonList(view.getTableId()));
if (ObjectUtils.isNotEmpty(chartExtRequest.getFilter())) {
for (ChartExtFilterDTO request : chartExtRequest.getFilter()) {
// 解析多个fieldId,fieldId是一个逗号分隔的字符串
@@ -153,7 +154,6 @@ public class ChartDataManage {
}
boolean hasParameters = false;
List<SqlVariableDetails> sqlVariables = datasetGroupManage.getSqlParams(Collections.singletonList(view.getTableId()));
if (CollectionUtils.isNotEmpty(sqlVariables)) {
for (SqlVariableDetails parameter : Optional.ofNullable(request.getParameters()).orElse(new ArrayList<>())) {
String parameterId = StringUtils.endsWith(parameter.getId(), START_END_SEPARATOR) ? parameter.getId().split(START_END_SEPARATOR)[0] : parameter.getId();
@@ -234,18 +234,35 @@ public class ChartDataManage {
//联动过滤条件和外部参数过滤条件全部加上
if (ObjectUtils.isNotEmpty(filters)) {
for (ChartExtFilterDTO request : filters) {
DatasetTableFieldDTO datasetTableField = datasetTableFieldManage.selectById(Long.valueOf(request.getFieldId()));
request.setDatasetTableField(datasetTableField);
request.setFilterType(2);
// 相同数据集
if (Objects.equals(datasetTableField.getDatasetGroupId(), view.getTableId())) {
if (ObjectUtils.isNotEmpty(request.getViewIds())) {
if (request.getViewIds().contains(view.getId())) {
// 包含 DE 的为数据集参数
if(request.getFieldId().contains("DE")){
// 组装sql 参数原始数据
if (CollectionUtils.isNotEmpty(sqlVariables)) {
for(SqlVariableDetails sourceVariables : sqlVariables){
if(sourceVariables.getId().equals(request.getFieldId())){
if(CollectionUtils.isEmpty(request.getParameters())){
request.setParameters(new ArrayList<>());
}
request.getParameters().add(sourceVariables);
}
}
}
}else {
DatasetTableFieldDTO datasetTableField = datasetTableFieldManage.selectById(Long.valueOf(request.getFieldId()));
request.setDatasetTableField(datasetTableField);
request.setFilterType(2);
// 相同数据集
if (Objects.equals(datasetTableField.getDatasetGroupId(), view.getTableId())) {
if (ObjectUtils.isNotEmpty(request.getViewIds())) {
if (request.getViewIds().contains(view.getId())) {
extFilterList.add(request);
}
} else {
extFilterList.add(request);
}
} else {
extFilterList.add(request);
}
}
}
}
@@ -604,7 +621,8 @@ public class ChartDataManage {
if (StringUtils.equalsIgnoreCase(view.getType(), "table-pivot")
|| StringUtils.containsIgnoreCase(view.getType(), "group")
|| ("antv".equalsIgnoreCase(view.getRender()) && "line".equalsIgnoreCase(view.getType()))
|| StringUtils.equalsIgnoreCase(view.getType(), "flow-map")) {
|| StringUtils.equalsIgnoreCase(view.getType(), "flow-map")
|| StringUtils.equalsIgnoreCase(view.getType(), "t-heatmap")) {
xAxis.addAll(xAxisExt);
}
List<ChartViewFieldDTO> yAxis = new ArrayList<>(view.getYAxis());
@@ -754,16 +772,22 @@ public class ChartDataManage {
public void saveChartViewFromVisualization(String checkData, Long sceneId, Map<Long, ChartViewDTO> chartViewsInfo) {
if (!MapUtils.isEmpty(chartViewsInfo)) {
List<Long> disuseChartIdList = new ArrayList<>();
chartViewsInfo.forEach((key, chartViewDTO) -> {
if (checkData.indexOf(chartViewDTO.getId() + "") > -1) {
if (checkData.contains(chartViewDTO.getId() + "")) {
try {
chartViewDTO.setSceneId(sceneId);
chartViewManege.save(chartViewDTO);
} catch (Exception e) {
DEException.throwException(e);
}
} else {
disuseChartIdList.add(chartViewDTO.getId());
}
});
if (CollectionUtils.isNotEmpty(disuseChartIdList)) {
chartViewManege.disuse(disuseChartIdList);
}
}
}
}

11
core/core-backend/src/main/java/io/dataease/chart/manage/ChartViewManege.java
View File

@@ -26,6 +26,7 @@ import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.view.dto.*;
import io.dataease.extensions.view.filter.FilterTreeObj;
import io.dataease.i18n.Translator;
import io.dataease.license.config.XpackInteract;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.IDUtils;
import io.dataease.utils.JsonUtil;
@@ -98,6 +99,9 @@ public class ChartViewManege {
coreChartViewMapper.deleteById(id);
}
@XpackInteract(value = "chartViewManage")
public void disuse(List<Long> chartIdList) {}
@Transactional
public void deleteBySceneId(Long sceneId, List<Long> chartIds) {
QueryWrapper<CoreChartView> wrapper = new QueryWrapper<>();
@@ -252,6 +256,7 @@ public class ChartViewManege {
public ChartBaseVO chartBaseInfo(Long id) {
ChartBasePO po = extChartViewMapper.queryChart(id);
if (ObjectUtils.isEmpty(po)) return null;
ChartBaseVO vo = BeanUtils.copyBean(new ChartBaseVO(), po);
TypeReference<List<ChartViewFieldDTO>> tokenType = new TypeReference<>() {};
vo.setXAxis(JsonUtil.parseList(po.getXAxis(), tokenType));
@@ -324,6 +329,9 @@ public class ChartViewManege {
record.setDrillFields(objectMapper.writeValueAsString(dto.getDrillFields()));
record.setCustomFilter(objectMapper.writeValueAsString(dto.getCustomFilter()));
record.setViewFields(objectMapper.writeValueAsString(dto.getViewFields()));
record.setFlowMapStartName(objectMapper.writeValueAsString(dto.getFlowMapStartName()));
record.setFlowMapEndName(objectMapper.writeValueAsString(dto.getFlowMapEndName()));
record.setExtColor(objectMapper.writeValueAsString(dto.getExtColor()));
return record;
}
@@ -349,6 +357,9 @@ public class ChartViewManege {
dto.setDrillFields(JsonUtil.parseList(record.getDrillFields(), tokenType));
dto.setCustomFilter(JsonUtil.parseObject(record.getCustomFilter(), FilterTreeObj.class));
dto.setViewFields(JsonUtil.parseList(record.getViewFields(), tokenType));
dto.setFlowMapStartName(JsonUtil.parseList(record.getFlowMapStartName(), tokenType));
dto.setFlowMapEndName(JsonUtil.parseList(record.getFlowMapEndName(), tokenType));
dto.setExtColor(JsonUtil.parseList(record.getExtColor(), tokenType));
return dto;

10
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java
View File

@@ -112,7 +112,8 @@ public class DatasetDataManage {
sql = provider.transSqlDialect(sql, datasourceRequest.getDsList());
} else {
// parser sql params and replace default value
String originSql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(tableInfoDTO.getSql())), datasetTableDTO.getSqlVariableDetails(), false, false, null, false, datasourceRequest.getDsList(), pluginManage);
String originSql = provider.replaceComment(new String(Base64.getDecoder().decode(tableInfoDTO.getSql())));
originSql = SqlparserUtils.handleVariableDefaultValue(originSql, datasetTableDTO.getSqlVariableDetails(), false, false, null, false, datasourceRequest.getDsList(), pluginManage);
// add sql table schema
sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);
@@ -344,13 +345,16 @@ public class DatasetDataManage {
dsMap.put(datasourceSchemaDTO.getId(), datasourceSchemaDTO);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap);
Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());
// parser sql params and replace default value
String originSql = SqlparserUtils.handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(new String(Base64.getDecoder().decode(dto.getSql()))), dto.getSqlVariableDetails(), true, true, null, false, dsMap, pluginManage);
String originSql = provider.replaceComment(new String(Base64.getDecoder().decode(dto.getSql())));
originSql = SqlparserUtils.handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(originSql), dto.getSqlVariableDetails(), true, true, null, false, dsMap, pluginManage);
// sql 作为临时表，外层加上limit
String sql;
Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());
if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) {
// 先根据sql获取表字段
String sqlField = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);

36
core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java
View File

@@ -19,7 +19,9 @@ import io.dataease.extensions.datasource.dto.DatasetTableDTO;
import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO;
import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO;
import io.dataease.extensions.datasource.dto.DsTypeDTO;
import io.dataease.extensions.datasource.factory.ProviderFactory;
import io.dataease.extensions.datasource.model.SQLObj;
import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.extensions.datasource.vo.DatasourceConfiguration;
import io.dataease.extensions.datasource.vo.XpackPluginsDatasourceVO;
import io.dataease.extensions.view.dto.ChartExtFilterDTO;
@@ -64,25 +66,37 @@ public class DatasetSQLManage {
private List<SqlVariableDetails> filterParameters(ChartExtRequest chartExtRequest, Long datasetTableId) {
List<SqlVariableDetails> parameters = new ArrayList<>();
if (chartExtRequest != null && ObjectUtils.isNotEmpty(chartExtRequest.getOuterParamsFilters())) {
for (ChartExtFilterDTO filterDTO : chartExtRequest.getOuterParamsFilters()) {
if (CollectionUtils.isEmpty(filterDTO.getValue())) {
continue;
}
filterParametersAdaptor(parameters,filterDTO,datasetTableId);
}
}
if (chartExtRequest != null && ObjectUtils.isNotEmpty(chartExtRequest.getFilter())) {
for (ChartExtFilterDTO filterDTO : chartExtRequest.getFilter()) {
if (CollectionUtils.isEmpty(filterDTO.getValue())) {
continue;
}
if (ObjectUtils.isNotEmpty(filterDTO.getParameters())) {
for (SqlVariableDetails parameter : filterDTO.getParameters()) {
if (parameter.getDatasetTableId().equals(datasetTableId)) {
parameter.setValue(filterDTO.getValue());
parameter.setOperator(filterDTO.getOperator());
parameters.add(parameter);
}
}
}
filterParametersAdaptor(parameters,filterDTO,datasetTableId);
}
}
return parameters;
}
private void filterParametersAdaptor(List<SqlVariableDetails> parameters,ChartExtFilterDTO filterDTO,Long datasetTableId){
if (ObjectUtils.isNotEmpty(filterDTO.getParameters())) {
for (SqlVariableDetails parameter : filterDTO.getParameters()) {
if (parameter.getDatasetTableId().equals(datasetTableId)) {
parameter.setValue(filterDTO.getValue());
parameter.setOperator(filterDTO.getOperator());
parameters.add(parameter);
}
}
}
}
public Map<String, Object> getUnionSQLForEdit(DatasetGroupInfoDTO dataTableInfoDTO, ChartExtRequest chartExtRequest) throws Exception {
Map<Long, DatasourceSchemaDTO> dsMap = new LinkedHashMap<>();
List<UnionDTO> union = dataTableInfoDTO.getUnion();
@@ -430,8 +444,10 @@ public class DatasetSQLManage {
if (StringUtils.equalsIgnoreCase(currentDs.getType(), DatasetTableTypeConstants.DATASET_TABLE_DB)) {
tableObj = SQLObj.builder().tableSchema(tableSchema).tableName(infoDTO.getTable()).tableAlias(tableAlias).build();
} else if (StringUtils.equalsIgnoreCase(currentDs.getType(), DatasetTableTypeConstants.DATASET_TABLE_SQL)) {
Provider provider = ProviderFactory.getProvider(dsMap.entrySet().iterator().next().getValue().getType());
// parser sql params and replace default value
String sql = SqlparserUtils.handleVariableDefaultValue(new String(Base64.getDecoder().decode(infoDTO.getSql())), currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters, isCross, dsMap, pluginManage);
String sql = provider.replaceComment(new String(Base64.getDecoder().decode(infoDTO.getSql())));
sql = SqlparserUtils.handleVariableDefaultValue(sql, currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters, isCross, dsMap, pluginManage);
// add table schema
if (isCross) {
sql = SqlUtils.addSchema(sql, tableSchema);

45
core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java
View File

@@ -382,7 +382,7 @@ public class CalciteProvider extends Provider {
// schema
ResultSet resultSet = null;
try (ConnectionObj con = getConnection(datasourceRequest.getDatasource()); PreparedStatement statement = getPreparedStatement(con.getConnection(), datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery())) {
try (ConnectionObj con = getConnection(datasourceRequest.getDatasource()); Statement statement = getPreparedStatement(con.getConnection(), datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) {
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
@@ -390,12 +390,13 @@ public class CalciteProvider extends Provider {
if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) {
LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery());
for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) {
statement.setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
}
resultSet = ((PreparedStatement) statement).executeQuery();
} else {
resultSet = statement.executeQuery(datasourceRequest.getQuery());
}
resultSet = statement.executeQuery();
fieldList = getField(resultSet, datasourceRequest);
dataList = getData(resultSet, datasourceRequest);
} catch (SQLException e) {
@@ -426,7 +427,7 @@ public class CalciteProvider extends Provider {
// schema
ResultSet resultSet = null;
try (ConnectionObj con = getConnection(datasourceRequest.getDatasource()); PreparedStatement statement = getPreparedStatement(con.getConnection(), datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery())) {
try (ConnectionObj con = getConnection(datasourceRequest.getDatasource()); Statement statement = getPreparedStatement(con.getConnection(), datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) {
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
@@ -434,13 +435,14 @@ public class CalciteProvider extends Provider {
if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) {
LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery());
for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) {
statement.setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
}
((PreparedStatement) statement).execute();
} else {
statement.execute(datasourceRequest.getQuery());
}
statement.execute();
} catch (SQLException e) {
DEException.throwException("SQL ERROR: " + e.getMessage());
} catch (Exception e) {
@@ -465,7 +467,7 @@ public class CalciteProvider extends Provider {
// schema
ResultSet resultSet = null;
try (ConnectionObj con = getConnection(datasourceRequest.getDatasource()); PreparedStatement statement = getPreparedStatement(con.getConnection(), datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery())) {
try (ConnectionObj con = getConnection(datasourceRequest.getDatasource()); Statement statement = getPreparedStatement(con.getConnection(), datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) {
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
@@ -473,13 +475,14 @@ public class CalciteProvider extends Provider {
if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) {
LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery());
for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) {
statement.setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
}
return ((PreparedStatement) statement).executeUpdate();
} else {
return statement.executeUpdate(datasourceRequest.getQuery());
}
return statement.executeUpdate();
} catch (SQLException e) {
DEException.throwException("SQL ERROR: " + e.getMessage());
} catch (Exception e) {
@@ -1158,16 +1161,22 @@ public class CalciteProvider extends Provider {
return stat;
}
public PreparedStatement getPreparedStatement(Connection connection, int queryTimeout, String sql) throws Exception {
public Statement getPreparedStatement(Connection connection, int queryTimeout, String sql, List<TableFieldWithValue> values) throws Exception {
if (connection == null) {
throw new Exception("Failed to get connection!");
}
PreparedStatement stat = connection.prepareStatement(sql);
try {
stat.setQueryTimeout(queryTimeout);
} catch (Exception e) {
if (CollectionUtils.isNotEmpty(values)) {
PreparedStatement stat = null;
try {
stat = connection.prepareStatement(sql);
stat.setQueryTimeout(queryTimeout);
} catch (Exception e) {
DEException.throwException(e.getMessage());
}
return stat;
} else {
return getStatement(connection, queryTimeout);
}
return stat;
}
protected boolean isDefaultClassLoader(String customDriver) {

20
core/core-backend/src/main/java/io/dataease/visualization/dao/auto/entity/VisualizationOuterParamsTargetViewInfo.java
View File

@@ -1,5 +1,6 @@
package io.dataease.visualization.dao.auto.entity;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.io.Serializable;
@@ -9,7 +10,7 @@ import java.io.Serializable;
* </p>
*
* @author fit2cloud
* @since 2024-03-08
* @since 2024-08-21
*/
@TableName("visualization_outer_params_target_view_info")
public class VisualizationOuterParamsTargetViewInfo implements Serializable {
@@ -19,6 +20,7 @@ public class VisualizationOuterParamsTargetViewInfo implements Serializable {
/**
* 主键
*/
@TableId("target_id")
private String targetId;
/**
@@ -27,10 +29,15 @@ public class VisualizationOuterParamsTargetViewInfo implements Serializable {
private String paramsInfoId;
/**
* 联动视图ID
* 联动视图ID/联动过滤项ID
*/
private String targetViewId;
/**
* 联动数据集id/联动过滤组件id
*/
private String targetDsId;
/**
* 联动字段ID
*/
@@ -70,6 +77,14 @@ public class VisualizationOuterParamsTargetViewInfo implements Serializable {
this.targetViewId = targetViewId;
}
public String getTargetDsId() {
return targetDsId;
}
public void setTargetDsId(String targetDsId) {
this.targetDsId = targetDsId;
}
public String getTargetFieldId() {
return targetFieldId;
}
@@ -100,6 +115,7 @@ public class VisualizationOuterParamsTargetViewInfo implements Serializable {
"targetId = " + targetId +
", paramsInfoId = " + paramsInfoId +
", targetViewId = " + targetViewId +
", targetDsId = " + targetDsId +
", targetFieldId = " + targetFieldId +
", copyFrom = " + copyFrom +
", copyId = " + copyId +

2
core/core-backend/src/main/java/io/dataease/visualization/dao/auto/mapper/VisualizationOuterParamsTargetViewInfoMapper.java
View File

@@ -10,7 +10,7 @@ import org.apache.ibatis.annotations.Mapper;
* </p>
*
* @author fit2cloud
* @since 2024-03-08
* @since 2024-08-21
*/
@Mapper
public interface VisualizationOuterParamsTargetViewInfoMapper extends BaseMapper<VisualizationOuterParamsTargetViewInfo> {

3
core/core-backend/src/main/java/io/dataease/visualization/dao/ext/mapper/ExtVisualizationOuterParamsMapper.java
View File

@@ -1,6 +1,7 @@
package io.dataease.visualization.dao.ext.mapper;
import io.dataease.api.dataset.vo.CoreDatasetGroupVO;
import io.dataease.api.visualization.dto.VisualizationOuterParamsDTO;
import io.dataease.api.visualization.dto.VisualizationOuterParamsInfoDTO;
import org.apache.ibatis.annotations.Mapper;
@@ -20,4 +21,6 @@ public interface ExtVisualizationOuterParamsMapper {
void deleteOuterParamsWithVisualizationId(@Param("visualizationId") String visualizationId);
List<VisualizationOuterParamsInfoDTO> getVisualizationOuterParamsInfo(@Param("visualizationId") String visualizationId);
List<CoreDatasetGroupVO> queryDsWithVisualizationId(@Param("visualizationId") String visualizationId);
}

54
core/core-backend/src/main/java/io/dataease/visualization/server/VisualizationOuterParamsService.java
View File

@@ -1,10 +1,20 @@
package io.dataease.visualization.server;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.ibm.icu.impl.coll.CollationLoader;
import io.dataease.api.dataset.vo.CoreDatasetGroupVO;
import io.dataease.api.dataset.vo.CoreDatasetTableFieldVO;
import io.dataease.api.visualization.VisualizationOuterParamsApi;
import io.dataease.api.visualization.dto.VisualizationOuterParamsDTO;
import io.dataease.api.visualization.dto.VisualizationOuterParamsInfoDTO;
import io.dataease.api.visualization.response.VisualizationOuterParamsBaseResponse;
import io.dataease.dataset.dao.auto.entity.CoreDatasetTable;
import io.dataease.dataset.dao.auto.mapper.CoreDatasetTableMapper;
import io.dataease.engine.constant.DeTypeConstants;
import io.dataease.extensions.view.dto.SqlVariableDetails;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.JsonUtil;
import io.dataease.visualization.dao.auto.entity.VisualizationOuterParams;
import io.dataease.visualization.dao.auto.entity.VisualizationOuterParamsInfo;
import io.dataease.visualization.dao.auto.entity.VisualizationOuterParamsTargetViewInfo;
@@ -13,14 +23,13 @@ import io.dataease.visualization.dao.auto.mapper.VisualizationOuterParamsMapper;
import io.dataease.visualization.dao.auto.mapper.VisualizationOuterParamsTargetViewInfoMapper;
import io.dataease.visualization.dao.ext.mapper.ExtVisualizationOuterParamsMapper;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.StringUtils;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.*;
import java.util.stream.Collectors;
/**
@@ -41,12 +50,21 @@ public class VisualizationOuterParamsService implements VisualizationOuterParams
@Resource
private VisualizationOuterParamsTargetViewInfoMapper outerParamsTargetViewInfoMapper;
@Resource
private CoreDatasetTableMapper coreDatasetTableMapper;
@Override
public VisualizationOuterParamsDTO queryWithVisualizationId(String visualizationId) {
VisualizationOuterParamsDTO visualizationOuterParamsDTO = extOuterParamsMapper.queryWithVisualizationId(visualizationId);
return visualizationOuterParamsDTO;
}
@Override
public VisualizationOuterParamsDTO queryWithVisualizationIdDS(String dvId) {
return null;
}
@Override
public void updateOuterParamsSet(VisualizationOuterParamsDTO outerParamsDTO) {
String visualizationId = outerParamsDTO.getVisualizationId();
@@ -85,4 +103,32 @@ public class VisualizationOuterParamsService implements VisualizationOuterParams
List<VisualizationOuterParamsInfoDTO> result = extOuterParamsMapper.getVisualizationOuterParamsInfo(visualizationId);
return new VisualizationOuterParamsBaseResponse(Optional.ofNullable(result).orElse(new ArrayList<>()).stream().collect(Collectors.toMap(VisualizationOuterParamsInfoDTO::getSourceInfo, VisualizationOuterParamsInfoDTO::getTargetInfoList)));
}
@Override
public List<CoreDatasetGroupVO> queryDsWithVisualizationId(String visualizationId) {
List<CoreDatasetGroupVO> result = extOuterParamsMapper.queryDsWithVisualizationId(visualizationId);
if(!CollectionUtils.isEmpty(result)){
result.forEach(coreDatasetGroupVO -> {
List<CoreDatasetTableFieldVO> fields = coreDatasetGroupVO.getDatasetFields();
QueryWrapper<CoreDatasetTable> wrapper = new QueryWrapper<>();
wrapper.eq("dataset_group_id", coreDatasetGroupVO.getId());
List<CoreDatasetTable> tableResult = coreDatasetTableMapper.selectList(wrapper);
if(!CollectionUtils.isEmpty(tableResult)){
tableResult.forEach(coreDatasetTable -> {
String sqlVarDetail = coreDatasetTable.getSqlVariableDetails();
if(StringUtils.isNotEmpty(sqlVarDetail)){
TypeReference<List<SqlVariableDetails>> listTypeReference = new TypeReference<List<SqlVariableDetails>>() {
};
List<SqlVariableDetails> defaultsSqlVariableDetails = JsonUtil.parseList(sqlVarDetail, listTypeReference);
defaultsSqlVariableDetails.forEach(sqlVariableDetails -> {
String varFieldId = coreDatasetTable.getId()+"|DE|"+sqlVariableDetails.getVariableName();
fields.add(new CoreDatasetTableFieldVO(varFieldId,sqlVariableDetails.getVariableName(), DeTypeConstants.DE_STRING));
});
}
});
}
});
}
return result;
}
}

16
core/core-backend/src/main/resources/db/desktop/V2.10__ddl.sql
View File

@@ -6,3 +6,19 @@ alter table `core_dataset_table_field` add params text null comment '计算字
alter table `core_datasource`
add `enable_data_fill` tinyint default 0 null comment '启用数据填报功能';
ALTER TABLE `visualization_outer_params_target_view_info`
MODIFY COLUMN `target_view_id` varchar(50) NULL DEFAULT NULL COMMENT '联动视图ID/联动过滤项ID' ,
ADD COLUMN `target_ds_id` varchar(50) NULL COMMENT '联动数据集id/联动过滤组件id' ;
alter table `core_chart_view`
add flow_map_start_name longtext comment '流向地图起点名称field';
alter table `core_chart_view`
add flow_map_end_name longtext comment '流向地图终点名称field';
alter table `core_chart_view`
add ext_color longtext comment '颜色维度field';
update visualization_outer_params_target_view_info tvi INNER JOIN core_chart_view ccv on tvi.target_view_id = ccv.id
set tvi.target_ds_id = ccv.table_id

13
core/core-backend/src/main/resources/db/migration/V2.10__ddl.sql
View File

@@ -57,3 +57,16 @@ CREATE TABLE `xpack_threshold_instance`
);
ALTER TABLE `visualization_outer_params_target_view_info`
MODIFY COLUMN `target_view_id` varchar(50) NULL DEFAULT NULL COMMENT '联动视图ID/联动过滤项ID' ,
ADD COLUMN `target_ds_id` varchar(50) NULL COMMENT '联动数据集id/联动过滤组件id' ;
alter table `core_chart_view`
add flow_map_start_name longtext comment '流向地图起点名称field';
alter table `core_chart_view`
add flow_map_end_name longtext comment '流向地图终点名称field';
alter table `core_chart_view`
add ext_color longtext comment '颜色维度field';
update visualization_outer_params_target_view_info tvi INNER JOIN core_chart_view ccv on tvi.target_view_id = ccv.id
set tvi.target_ds_id = ccv.table_id

4
core/core-backend/src/main/resources/mybatis/ExtDataVisualizationMapper.xml
View File

@@ -160,7 +160,9 @@
FROM data_visualization_info
where data_visualization_info.delete_flag = 0
and data_visualization_info.id = #{dvId}
and data_visualization_info.type = #{dvType}
<if test="dvType">
and data_visualization_info.type = #{dvType}
</if>
</select>
<select id="findRecent" resultType="io.dataease.visualization.dao.ext.po.VisualizationResourcePO">

42
core/core-backend/src/main/resources/mybatis/ExtVisualizationOuterParamsMapper.xml
View File

@@ -28,9 +28,21 @@
</collection>
</resultMap>
<resultMap id="BaseDsResultMapDTO" type="io.dataease.api.dataset.vo.CoreDatasetGroupVO">
<collection property="datasetFields" ofType="io.dataease.api.dataset.vo.CoreDatasetTableFieldVO"
column="{dataset_group_id=id}"
select="getDsFieldInfo">
</collection>
<collection property="datasetViews" ofType="io.dataease.api.chart.vo.ChartBaseVO"
column="{dataset_group_id=id,visualizationId = visualizationId}"
select="getViewInfo">
</collection>
</resultMap>
<resultMap id="OuterParamsInfoMap" type="io.dataease.api.visualization.dto.VisualizationOuterParamsInfoDTO" extends="BaseResultMapParamsInfo">
<collection property="targetViewInfoList" ofType="io.dataease.visualization.dao.auto.entity.VisualizationOuterParamsTargetViewInfo">
<result column="target_view_id" jdbcType="VARCHAR" property="targetViewId"/>
<result column="target_ds_id" jdbcType="VARCHAR" property="targetDsId"/>
<result column="target_field_id" jdbcType="VARCHAR" property="targetFieldId"/>
</collection>
</resultMap>
@@ -49,6 +61,7 @@
popi.param_name,
ifnull( popi.checked, 0 ) AS checked,
poptvi.target_view_id,
poptvi.target_ds_id,
poptvi.target_field_id
FROM
visualization_outer_params pop
@@ -114,5 +127,34 @@
and popi.checked=1
</select>
<select id="queryDsWithVisualizationId" resultMap="BaseDsResultMapDTO">
SELECT DISTINCT
cdg.*,#{visualizationId} as visualizationId
FROM
core_dataset_group cdg
INNER JOIN core_chart_view ccv ON cdg.id = ccv.table_id and ccv.type != 'VQuery'
INNER JOIN data_visualization_info dvi ON ccv.scene_id = dvi.id
WHERE
ccv.scene_id = #{visualizationId}
AND dvi.id = #{visualizationId}
AND LOCATE(ccv.id, dvi.component_data)
</select>
<select id="getDsFieldInfo" resultType="io.dataease.api.dataset.vo.CoreDatasetTableFieldVO">
select cdtf.*,cdtf.id as attachId from core_dataset_table_field cdtf where cdtf.dataset_group_id = #{dataset_group_id}
</select>
<select id="getViewInfo" resultType="io.dataease.api.chart.vo.ChartBaseVO">
SELECT DISTINCT
ccv.id as chartId,ccv.title as chartName,ccv.type as chartType
FROM
core_chart_view ccv
INNER JOIN data_visualization_info dvi ON ccv.scene_id = dvi.id
WHERE
ccv.table_id = #{dataset_group_id}
AND ccv.type != 'VQuery'
AND dvi.id = #{visualizationId}
AND LOCATE(ccv.id, dvi.component_data)
</select>
</mapper>