From 4e482fc8606837b433b74bf283bb871e94d80304 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Thu, 4 Sep 2025 11:19:49 +0800 Subject: [PATCH] =?UTF-8?q?perf:=20SQLBot=20=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=20=E4=BD=BF=E7=94=A8=20AES=20=E5=8A=A0?= =?UTF-8?q?=E5=AF=86=E5=8F=AF=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ext/mapper/DataSetAssistantMapper.java | 10 ++- .../dataset/manage/DatasetSQLBotManage.java | 79 ++++++++++++++++--- .../dataset/server/DatasetSQLBotServer.java | 4 +- .../api/dataset/DataAssistantApi.java | 3 +- .../api/dataset/vo/SQLBotAssistanTable.java | 4 + .../api/dataset/vo/SQLBotAssistantField.java | 6 ++ 6 files changed, 87 insertions(+), 19 deletions(-) diff --git a/core/core-backend/src/main/java/io/dataease/dataset/dao/ext/mapper/DataSetAssistantMapper.java b/core/core-backend/src/main/java/io/dataease/dataset/dao/ext/mapper/DataSetAssistantMapper.java index d1b493f6d8..ce0c475a26 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/dao/ext/mapper/DataSetAssistantMapper.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/dao/ext/mapper/DataSetAssistantMapper.java @@ -1,5 +1,6 @@ package io.dataease.dataset.dao.ext.mapper; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -32,9 +33,10 @@ public interface DataSetAssistantMapper { left join `core_dataset_table_field` cdtf on cdtf.dataset_table_id = cdt.id left join `core_dataset_group` cdg on cdg.id = cdt.dataset_group_id where cdg.is_cross != 1 and (cd.STATUS IS NULL OR cd.STATUS != 'Error') + ${ew.customSqlSegment} """ ) - List> queryAll(); + List> queryAll(@Param("ew") QueryWrapper queryWrapper); @Select(""" WITH user_ds_permissions AS ( @@ -69,8 +71,9 @@ public interface DataSetAssistantMapper { INNER JOIN `core_dataset_table_field` cdtf ON cdtf.dataset_table_id = cdt.id where not exists( select 1 from user_ds_permissions ds_p where cd.id = ds_p.resource_id ) and not exists( select 1 from user_dg_permissions dg_p where cdg.id = dg_p.resource_id ) + ${ew.customSqlSegment} """) - List> queryCommunity(); + List> queryCommunity(@Param("ew") QueryWrapper queryWrapper); @@ -144,9 +147,10 @@ public interface DataSetAssistantMapper { INNER JOIN `core_dataset_table_field` cdtf ON cdtf.dataset_table_id = cdt.id INNER JOIN user_ds_permissions ds_p ON cd.id = ds_p.resource_id INNER JOIN user_dg_permissions dg_p ON cdg.id = dg_p.resource_id + ${ew.customSqlSegment} """) - List> queryEnterprise(@Param("oid") Long oid, @Param("uid") Long uid, @Param("orgAdmin") Boolean orgAdmin); + List> queryEnterprise(@Param("oid") Long oid, @Param("uid") Long uid, @Param("orgAdmin") Boolean orgAdmin, @Param("ew") QueryWrapper queryWrapper); } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java index 646c115e96..bb3fd768f8 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java @@ -1,5 +1,6 @@ package io.dataease.dataset.manage; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import io.dataease.api.dataset.union.DatasetTableInfoDTO; import io.dataease.api.dataset.vo.DataSQLBotAssistantVO; import io.dataease.api.dataset.vo.SQLBotAssistanTable; @@ -14,22 +15,24 @@ import io.dataease.api.permissions.role.dto.RoleRequest; import io.dataease.api.permissions.role.vo.RoleVO; import io.dataease.auth.bo.TokenUserBO; import io.dataease.commons.utils.EncryptUtils; +import io.dataease.constant.ColumnPermissionConstants; import io.dataease.dataset.dao.ext.mapper.DataSetAssistantMapper; import io.dataease.datasource.dao.auto.entity.CoreDatasource; import io.dataease.datasource.manage.EngineManage; import io.dataease.extensions.datasource.vo.Configuration; import io.dataease.extensions.datasource.vo.DatasourceConfiguration; +import io.dataease.extensions.view.dto.ColumnPermissionItem; +import io.dataease.extensions.view.dto.ColumnPermissions; import io.dataease.home.manage.DeIndexManage; import io.dataease.utils.*; import jakarta.annotation.Resource; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; -import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; @@ -48,8 +51,18 @@ public class DatasetSQLBotManage { @Resource private Environment environment; - private String aesKey = "y5txe1mRmS_JpOrUzFzHEu-kIQn3lf7l"; - private String aesIv = "sqlbot_em_aes_iv"; + private CoreDatasource deEngine; + + @Resource + private PermissionManage permissionManage; + + @Value("${dataease.sqlbot.encrypt:false}") + private boolean encryptEnabled; + + @Value("${dataease.sqlbot.aes-key:y5txe1mRmS_JpOrUzFzHEu-kIQn3lf7l}") + private String aesKey; + @Value("${dataease.sqlbot.aes-iv:sqlbot_em_aes_iv}") + private String aesIv; private String aesEncrypt(String text) { String iv = aesIv; @@ -115,7 +128,7 @@ public class DatasetSQLBotManage { } - public List getDatasourceList(List ids) { + public List getDatasourceList(Long dsId, Long datasetId) { TokenUserBO user = Objects.requireNonNull(AuthUtils.getUser()); Long oid = user.getDefaultOid(); Long uid = user.getUserId(); @@ -126,16 +139,23 @@ public class DatasetSQLBotManage { List> list = null; boolean isAdmin = uid == 1; boolean withColsOrRowsPermission = false; + QueryWrapper queryWrapper = new QueryWrapper<>(); + if (ObjectUtils.isNotEmpty(datasetId)) { + queryWrapper.eq("cdg.id", datasetId); + } + if (ObjectUtils.isNotEmpty(dsId)) { + queryWrapper.eq("cd.id", dsId); + } if (ObjectUtils.isEmpty(model)) { if (!isAdmin) { return null; } - list = dataSetAssistantMapper.queryAll(); + list = dataSetAssistantMapper.queryAll(queryWrapper); } else if (!model) { if (!isAdmin) { return null; } - list = dataSetAssistantMapper.queryCommunity(); + list = dataSetAssistantMapper.queryCommunity(queryWrapper); } else { boolean isRootRole = isAdmin; if (!isAdmin) { @@ -145,11 +165,11 @@ public class DatasetSQLBotManage { isRootRole = roleVOS.stream().anyMatch(RoleVO::isRoot); roleIds = roleVOS.stream().map(RoleVO::getId).toList(); - colPermissionMap = getColPermission(uid, roleIds); + /*colPermissionMap = getColPermission(uid, roleIds); rowPermissionMap = getRowPermission(uid, roleIds); - withColsOrRowsPermission = MapUtils.isNotEmpty(colPermissionMap) || MapUtils.isNotEmpty(rowPermissionMap); + withColsOrRowsPermission = MapUtils.isNotEmpty(colPermissionMap) || MapUtils.isNotEmpty(rowPermissionMap);*/ } - list = dataSetAssistantMapper.queryEnterprise(oid, uid, isRootRole); + list = dataSetAssistantMapper.queryEnterprise(oid, uid, isRootRole, queryWrapper); } @@ -157,6 +177,7 @@ public class DatasetSQLBotManage { Map dsFlagMap = new HashMap<>(); Map tableFlagMap = new HashMap<>(); Map fieldFlagMap = new HashMap<>(); + deEngine = engineManage.getDeEngine(); for (Map row : list) { // build ds String datasourceId = row.get("datasource_id").toString(); @@ -208,13 +229,41 @@ public class DatasetSQLBotManage { vos.forEach(vo -> { List tables = vo.getTables(); tables.forEach(table -> { + Long datasetGroupId = table.getDatasetGroupId(); + List dataSetColumnPermissionsDTOS = colPermissionMap.get(datasetGroupId); + + List columnPermissionItems = new ArrayList<>(); + + for (DataSetColumnPermissionsDTO dataSetColumnPermissionsDTO : dataSetColumnPermissionsDTOS) { + ColumnPermissions columnPermissions = JsonUtil.parseObject(dataSetColumnPermissionsDTO.getPermissions(), ColumnPermissions.class); + if (!columnPermissions.getEnable()) { + continue; + } + if (StringUtils.equalsAnyIgnoreCase(dataSetColumnPermissionsDTO.getAuthTargetType(), "user", "role")) { + columnPermissionItems.addAll(columnPermissions.getColumns().stream().filter(columnPermissionItem -> columnPermissionItem.getSelected()).collect(Collectors.toList())); + } + } + List filterFields = table.getFields().stream().filter(field -> { + List fieldColumnPermissionItems = columnPermissionItems.stream().filter(columnPermissionItem -> columnPermissionItem.getId().equals(field.getFieldId())).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(fieldColumnPermissionItems)) { + return true; + } + return fieldColumnPermissionItems.stream().map(ColumnPermissionItem::getOpt).toList().contains(ColumnPermissionConstants.Desensitization); + }).collect(Collectors.toList()); + table.setFields(filterFields); }); }); - return null; + return vos; } private SQLBotAssistantField buildField(Map row) { SQLBotAssistantField field = new SQLBotAssistantField(); + if (ObjectUtils.isNotEmpty(row.get("field_id"))) { + field.setFieldId(Long.parseLong(row.get("field_id").toString())); + } + if (ObjectUtils.isNotEmpty(row.get("dataease_name"))) { + field.setDataeaseName(row.get("dataease_name").toString()); + } field.setName(row.get("origin_name").toString()); field.setType(row.get("field_type").toString()); field.setComment(row.get("field_show_name").toString()); @@ -234,7 +283,6 @@ public class DatasetSQLBotManage { String dsType = row.get("ds_type").toString(); Configuration config = null; if (dsType.contains(DatasourceConfiguration.DatasourceType.Excel.name()) || dsType.contains(DatasourceConfiguration.DatasourceType.API.name())) { - CoreDatasource deEngine = engineManage.getDeEngine(); String config_json = EncryptUtils.aesDecrypt(deEngine.getConfiguration()).toString(); config = JsonUtil.parseObject(config_json, Configuration.class); if (StringUtils.isNotBlank(dsHost) && ObjectUtils.isNotEmpty(config)) { @@ -256,7 +304,9 @@ public class DatasetSQLBotManage { vo.setSchema(config.getSchema()); vo.setUser(config.getUsername()); vo.setPassword(config.getPassword()); - aesVO(vo); + if (encryptEnabled) { + aesVO(vo); + } return vo; } @@ -282,6 +332,9 @@ public class DatasetSQLBotManage { SQLBotAssistanTable table = new SQLBotAssistanTable(); table.setName(row.get("table_name").toString()); table.setComment(row.get("dataset_name").toString()); + if (ObjectUtils.isNotEmpty(row.get("dataset_group_id"))) { + table.setDatasetGroupId(Long.parseLong(row.get("dataset_group_id").toString())); + } Object infoObj = null; if (ObjectUtils.isNotEmpty(infoObj = row.get("info"))) { diff --git a/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetSQLBotServer.java b/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetSQLBotServer.java index 293276aaea..e3ef3b481a 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetSQLBotServer.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/server/DatasetSQLBotServer.java @@ -16,7 +16,7 @@ public class DatasetSQLBotServer implements DataAssistantApi { @Resource private DatasetSQLBotManage datasetSQLBotManage; @Override - public List getDatasourceList() { - return datasetSQLBotManage.getDatasourceList(null); + public List getDatasourceList(Long dsId, Long datasetId) { + return datasetSQLBotManage.getDatasourceList(dsId, datasetId); } } diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DataAssistantApi.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DataAssistantApi.java index 0862c8e077..fb171424b3 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DataAssistantApi.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/DataAssistantApi.java @@ -2,10 +2,11 @@ package io.dataease.api.dataset; import io.dataease.api.dataset.vo.DataSQLBotAssistantVO; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; import java.util.List; public interface DataAssistantApi { @GetMapping("/datasource") - List getDatasourceList(); + List getDatasourceList(@RequestParam(required = false) Long dsId, @RequestParam(required = false) Long datasetId); } diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistanTable.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistanTable.java index 002eac4aa9..d9414ed4fa 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistanTable.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistanTable.java @@ -1,5 +1,6 @@ package io.dataease.api.dataset.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import java.io.Serializable; @@ -14,4 +15,7 @@ public class SQLBotAssistanTable implements Serializable { private String sql; private List fields = new ArrayList<>(); + @JsonIgnore + private Long datasetGroupId; + } diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistantField.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistantField.java index 6877c28c41..970c9f0e4c 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistantField.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistantField.java @@ -1,5 +1,6 @@ package io.dataease.api.dataset.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import java.io.Serializable; @@ -9,4 +10,9 @@ public class SQLBotAssistantField implements Serializable { private String name; private String comment; private String type; + + @JsonIgnore + private Long fieldId; + @JsonIgnore + private String dataeaseName; }