From ee1870a172b9a615c4a67c08fb9cf53e4a52376c Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Mon, 8 Sep 2025 17:25:29 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20SQLBot=20=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=20=E8=A1=8C=E5=88=97=E6=9D=83=E9=99=90?= =?UTF-8?q?=E6=9C=BA=E5=88=B6=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataset/manage/DatasetSQLBotManage.java | 43 ++++++++++--- .../dataset/manage/DatasetSQLManage.java | 47 ++++++++------ .../java/io/dataease/utils/BeanUtils.java | 61 ++++++++++++++++++- .../datasource/provider/Provider.java | 28 +++++++-- 4 files changed, 149 insertions(+), 30 deletions(-) diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java index e834f58245..db40c48dcf 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java @@ -31,8 +31,10 @@ import io.dataease.engine.trans.WhereTree2Str; import io.dataease.engine.utils.Utils; import io.dataease.exception.DEException; import io.dataease.extensions.datasource.api.PluginManageApi; +import io.dataease.extensions.datasource.dto.CalParam; import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO; import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO; +import io.dataease.extensions.datasource.dto.FieldGroupDTO; import io.dataease.extensions.datasource.factory.ProviderFactory; import io.dataease.extensions.datasource.model.SQLMeta; import io.dataease.extensions.datasource.provider.Provider; @@ -270,7 +272,11 @@ public class DatasetSQLBotManage { } } - private void rebuildTable(SQLBotAssistanTable table, List columnPermissionsDTOS, List rowPermissionsTree) { + TypeReference> groupTokenType = new TypeReference<>() { + }; + TypeReference> typeToken = new TypeReference<>() { + }; + private void rebuildTable(SQLBotAssistanTable table, List columnPermissionsDTOS, List rowPermissionsTree, Map dsRowData) { Map rowData = table.getRowData(); CoreDatasetGroup coreDatasetGroup = BeanUtils.mapToBean(rowData, CoreDatasetGroup.class); @@ -287,6 +293,12 @@ public class DatasetSQLBotManage { List dsFields = sqlbotFields.stream().map(field -> { Map fieldRowData = field.getRowData(); DatasetTableFieldDTO fieldDTO = BeanUtils.mapToBean(fieldRowData, DatasetTableFieldDTO.class); + if (ObjectUtils.isNotEmpty(fieldRowData.get("group_list"))) { + fieldDTO.setGroupList(JsonUtil.parseList(fieldRowData.get("group_list").toString(), groupTokenType)); + } + if (ObjectUtils.isNotEmpty(fieldRowData.get("params"))) { + fieldDTO.setParams(JsonUtil.parseList(fieldRowData.get("params").toString(), typeToken)); + } fieldDTO.setFieldShortName(fieldDTO.getDataeaseName()); return fieldDTO; }).collect(Collectors.toList()); @@ -301,8 +313,16 @@ public class DatasetSQLBotManage { } Map sqlMap = null; + CoreDatasource coreDatasource = null; + String dsType = dsRowData.get("type").toString(); + Configuration config = null; + if (dsType.contains(DatasourceConfiguration.DatasourceType.Excel.name()) || dsType.contains(DatasourceConfiguration.DatasourceType.API.name())) { + coreDatasource = deEngine; + } else { + coreDatasource = BeanUtils.mapToBean(dsRowData, CoreDatasource.class); + } try { - sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null); + sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null, coreDatasource); } catch (Exception e) { throw new RuntimeException(e); } @@ -357,7 +377,7 @@ public class DatasetSQLBotManage { Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), fields, false, dsMap, Utils.getParams(fields), null, pluginManage); String querySQL; querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, false); - querySQL = provider.rebuildSQL(querySQL, sqlMeta, false, dsMap); + querySQL = provider.rebuildSQL(querySQL, sqlMeta, false, dsMap, true); table.setSql(querySQL); } @@ -371,13 +391,18 @@ public class DatasetSQLBotManage { return; } vos.forEach(vo -> { + Map dsRowData = vo.getRowData(); List tables = vo.getTables(); tables.forEach(table -> { Long datasetGroupId = table.getDatasetGroupId(); List columnPermissionsDTOS = ObjectUtils.isEmpty(colPermissionMap) ? null : colPermissionMap.get(datasetGroupId); List rowPermissionsTreeDTOS = ObjectUtils.isEmpty(rowPermissionMap) ? null : rowPermissionMap.get(datasetGroupId); if (table.isNeedTransform() || ObjectUtils.isNotEmpty(columnPermissionsDTOS) || ObjectUtils.isNotEmpty(rowPermissionsTreeDTOS)) { - rebuildTable(table, columnPermissionsDTOS, rowPermissionsTreeDTOS); + try { + rebuildTable(table, columnPermissionsDTOS, rowPermissionsTreeDTOS, dsRowData); + } catch (Exception e) { + LogUtil.error(e); + } } }); }); @@ -416,16 +441,17 @@ public class DatasetSQLBotManage { dsHost = environment.getProperty("dataease.dataease-servers", String.class); } String dsType = row.get("cd_type").toString(); + String config_json = null; Configuration config = null; if (dsType.contains(DatasourceConfiguration.DatasourceType.Excel.name()) || dsType.contains(DatasourceConfiguration.DatasourceType.API.name())) { - String config_json = EncryptUtils.aesDecrypt(deEngine.getConfiguration()).toString(); + config_json = EncryptUtils.aesDecrypt(deEngine.getConfiguration()).toString(); config = JsonUtil.parseObject(config_json, Configuration.class); if (StringUtils.isNotBlank(dsHost) && ObjectUtils.isNotEmpty(config)) { config.setHost(dsHost); } dsType = deEngine.getType(); } else { - String config_json = EncryptUtils.aesDecrypt(dsConfig.toString()).toString(); + config_json = EncryptUtils.aesDecrypt(dsConfig.toString()).toString(); config = JsonUtil.parseObject(config_json, Configuration.class); } DataSQLBotAssistantVO vo = new DataSQLBotAssistantVO(); @@ -439,7 +465,10 @@ public class DatasetSQLBotManage { vo.setSchema(config.getSchema()); vo.setUser(config.getUsername()); vo.setPassword(config.getPassword()); - vo.setRowData(buildRowData(row, 0)); + row.put("cd_configuration", config_json); + Map rowData = buildRowData(row, 0); + rowData.put("id", Long.parseLong(row.get("cd_id").toString())); + vo.setRowData(rowData); if (encryptEnabled) { aesVO(vo); } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java index 39219f4824..24de2c7264 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java @@ -124,6 +124,10 @@ public class DatasetSQLManage { } public Map getUnionSQLForEdit(DatasetGroupInfoDTO dataTableInfoDTO, ChartExtRequest chartExtRequest) throws Exception { + return getUnionSQLForEdit(dataTableInfoDTO, chartExtRequest, null); + } + + public Map getUnionSQLForEdit(DatasetGroupInfoDTO dataTableInfoDTO, ChartExtRequest chartExtRequest, CoreDatasource coreDatasource) throws Exception { Map dsMap = new LinkedHashMap<>(); List union = dataTableInfoDTO.getUnion(); // 所有选中的字段，即select后的查询字段 @@ -146,7 +150,7 @@ public class DatasetSQLManage { if (dsMap.containsKey(datasetTable.getDatasourceId())) { schema = dsMap.get(datasetTable.getDatasourceId()).getSchemaAlias(); } else { - schema = putObj2Map(dsMap, datasetTable, isCross); + schema = putObj2Map(dsMap, datasetTable, isCross, coreDatasource); } SQLObj table = getUnionTable(datasetTable, tableInfo, schema, i, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross, dsMap); if (i == 0) { @@ -491,24 +495,29 @@ public class DatasetSQLManage { } public String putObj2Map(Map dsMap, DatasetTableDTO ds, boolean isCross) { + return putObj2Map(dsMap, ds, isCross, null); + } + public String putObj2Map(Map dsMap, DatasetTableDTO ds, boolean isCross, CoreDatasource coreDatasource) { // 通过datasource id校验数据源权限 - BusiPerCheckDTO dto = new BusiPerCheckDTO(); - dto.setId(ds.getDatasourceId()); - dto.setAuthEnum(AuthEnum.READ); - boolean checked = corePermissionManage.checkAuth(dto); - if (!checked) { - DEException.throwException(Translator.get("i18n_no_datasource_permission")); + if (ObjectUtils.isEmpty(coreDatasource)) { + BusiPerCheckDTO dto = new BusiPerCheckDTO(); + dto.setId(ds.getDatasourceId()); + dto.setAuthEnum(AuthEnum.READ); + boolean checked = corePermissionManage.checkAuth(dto); + if (!checked) { + DEException.throwException(Translator.get("i18n_no_datasource_permission")); + } } - - String schemaAlias; if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.DB) || StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.SQL)) { - CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(ds.getDatasourceId()); - if (coreDatasource == null) { - DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + ds.getDatasourceId()); - } - if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { - coreDatasource = engineManage.getDeEngine(); + if (ObjectUtils.isEmpty(coreDatasource)) { + coreDatasource = dataSourceManage.getCoreDatasource(ds.getDatasourceId()); + if (coreDatasource == null) { + DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + ds.getDatasourceId()); + } + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + coreDatasource = engineManage.getDeEngine(); + } } Map map = JsonUtil.parseObject(coreDatasource.getConfiguration(), Map.class); @@ -525,7 +534,9 @@ public class DatasetSQLManage { dsMap.put(coreDatasource.getId(), datasourceSchemaDTO); } } else if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.Es)) { - CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(ds.getDatasourceId()); + if (ObjectUtils.isEmpty(coreDatasource)) { + coreDatasource = dataSourceManage.getCoreDatasource(ds.getDatasourceId()); + } schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId()); if (!dsMap.containsKey(coreDatasource.getId())) { DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); @@ -534,7 +545,9 @@ public class DatasetSQLManage { dsMap.put(coreDatasource.getId(), datasourceSchemaDTO); } } else { - CoreDatasource coreDatasource = engineManage.getDeEngine(); + if (ObjectUtils.isEmpty(coreDatasource)) { + coreDatasource = engineManage.getDeEngine(); + } schemaAlias = String.format(SQLConstants.SCHEMA, coreDatasource.getId()); if (!dsMap.containsKey(coreDatasource.getId())) { DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); diff --git a/sdk/common/src/main/java/io/dataease/utils/BeanUtils.java b/sdk/common/src/main/java/io/dataease/utils/BeanUtils.java index a06bc70a27..9cf93a1020 100644 --- a/sdk/common/src/main/java/io/dataease/utils/BeanUtils.java +++ b/sdk/common/src/main/java/io/dataease/utils/BeanUtils.java @@ -108,16 +108,75 @@ public class BeanUtils { // 查找Map中对应的key（支持驼峰和下划线） Object value = findValueInMap(map, propertyName); + + // 只有当值存在且有写入方法时才设置属性 if (value != null && descriptor.getWriteMethod() != null) { - descriptor.getWriteMethod().invoke(bean, value); + try { + // 可选：添加类型转换逻辑来处理类型不匹配的情况 + Object convertedValue = convertTypeIfNeeded(value, descriptor.getPropertyType()); + descriptor.getWriteMethod().invoke(bean, convertedValue); + } catch (IllegalArgumentException e) { + // 类型不匹配时跳过该属性，而不是抛出异常 + System.err.println("类型不匹配跳过属性: " + propertyName + ", 期望类型: " + + descriptor.getPropertyType() + ", 实际类型: " + value.getClass()); + } } } return bean; } catch (Exception e) { + e.printStackTrace(); throw new RuntimeException("Map转Bean失败", e); } } + private static Object convertTypeIfNeeded(Object value, Class targetType) { + if (value == null) { + return null; + } + + // 如果类型匹配，直接返回 + if (targetType.isInstance(value)) { + return value; + } + + // 添加常见的类型转换逻辑 + try { + if (targetType == String.class) { + return value.toString(); + } else if (targetType == Integer.class || targetType == int.class) { + if (value instanceof Number) { + return ((Number) value).intValue(); + } else { + return Integer.parseInt(value.toString()); + } + } else if (targetType == Long.class || targetType == long.class) { + if (value instanceof Number) { + return ((Number) value).longValue(); + } else { + return Long.parseLong(value.toString()); + } + } else if (targetType == Double.class || targetType == double.class) { + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } else { + return Double.parseDouble(value.toString()); + } + } else if (targetType == Boolean.class || targetType == boolean.class) { + if (value instanceof Boolean) { + return value; + } else { + return Boolean.parseBoolean(value.toString()); + } + } + // 可以继续添加其他类型的转换... + } catch (Exception e) { + // 转换失败时返回原值，让后续逻辑处理 + System.err.println("类型转换失败: " + value + " to " + targetType); + } + + return value; // 无法转换时返回原值 + } + private static Object findValueInMap(Map map, String propertyName) { Object value = map.get(propertyName); if (value != null) return value; diff --git a/sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/provider/Provider.java b/sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/provider/Provider.java index 35bbacf96a..44269f79af 100644 --- a/sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/provider/Provider.java +++ b/sdk/extensions/extensions-datasource/src/main/java/io/dataease/extensions/datasource/provider/Provider.java @@ -123,12 +123,15 @@ public abstract class Provider { } public String rebuildSQL(String sql, SQLMeta sqlMeta, boolean crossDs, Map dsMap) { + return rebuildSQL(sql, sqlMeta, crossDs, dsMap, false); + } + public String rebuildSQL(String sql, SQLMeta sqlMeta, boolean crossDs, Map dsMap, boolean forSqlbot) { logger.debug("calcite sql: " + sql); if (crossDs) { return sql; } - String s = transSqlDialect(sql, dsMap); + String s = transSqlDialect(sql, dsMap, forSqlbot); String tableDialect = sqlMeta.getTableDialect(); s = replaceTablePlaceHolder(s, tableDialect); s = replaceCalcFieldPlaceHolder(s, sqlMeta); @@ -136,11 +139,18 @@ public abstract class Provider { } public String transSqlDialect(String sql, Map dsMap) throws DEException { + return transSqlDialect(sql, dsMap, false); + } + public String transSqlDialect(String sql, Map dsMap, boolean forSqlbot) throws DEException { DatasourceSchemaDTO value = dsMap.entrySet().iterator().next().getValue(); - try (ConnectionObj connection = getConnection(value)) { - // 获取数据库version - if (connection != null) { - value.setDsVersion(connection.getConnection().getMetaData().getDatabaseMajorVersion()); + ConnectionObj connection = null; + try { + if (!forSqlbot) { + connection = getConnection(value); + // 获取数据库version + if (connection != null) { + value.setDsVersion(connection.getConnection().getMetaData().getDatabaseMajorVersion()); + } } SqlParser parser = SqlParser.create(sql, SqlParser.Config.DEFAULT.withLex(Lex.JAVA)); SqlNode sqlNode = parser.parseStmt(); @@ -151,6 +161,14 @@ public abstract class Provider { return dialect; } catch (Exception e) { DEException.throwException(e.getMessage()); + } finally { + if (connection != null) { + try { + connection.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } } return null; }