diff --git a/core/core-backend/src/main/java/io/dataease/dataset/dao/ext/mapper/DataSetAssistantMapper.java b/core/core-backend/src/main/java/io/dataease/dataset/dao/ext/mapper/DataSetAssistantMapper.java index ce0c475a26..5f8dfe9d13 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/dao/ext/mapper/DataSetAssistantMapper.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/dao/ext/mapper/DataSetAssistantMapper.java @@ -14,24 +14,49 @@ public interface DataSetAssistantMapper { @Select( """ select - cdt.id, cdt.datasource_id, cdt.table_name, cdt.info, - cdg.name as dataset_name, - cd.name as ds_name, - cd.description as ds_desc, - cd.type as ds_type, - cd.configuration as ds_config, + cd.id as cd_id, + cd.name as cd_name, + cd.description as cd_description, + cd.type as cd_type, + cd.configuration as cd_configuration, - cdtf.id as field_id, - cdtf.origin_name, - cdtf.name as field_show_name, - cdtf.description as field_desc, - cdtf.dataease_name, - cdtf.type as field_type + cdg.id as cdg_id, + cdg.name as cdg_name, + cdg.type as cdg_type, + cdg.mode as cdg_model, + cdg.info as cdg_info, + cdg.union_sql as cdg_union_sql, + cdg.is_cross as cdg_is_cross, - from `core_dataset_table` cdt - left join `core_datasource` cd on cdt.datasource_id = cd.id - left join `core_dataset_table_field` cdtf on cdtf.dataset_table_id = cdt.id - left join `core_dataset_group` cdg on cdg.id = cdt.dataset_group_id + cdt.id as cdt_id, + cdt.table_name as cdt_table_name, + cdt.type as cdt_type, + cdt.info as cdt_info, + cdt.sql_variable_details as cdt_sql_variable_details, + + cdtf.id as cdtf_id, + cdtf.origin_name as cdtf_origin_name, + cdtf.name as cdtf_name, + cdtf.description as cdtf_description, + cdtf.dataease_name as cdtf_dataease_name, + cdtf.field_short_name as cdtf_field_short_name, + cdtf.group_list as cdtf_group_list, + cdtf.other_group as cdtf_other_group, + cdtf.group_type as cdtf_group_type, + cdtf.type as cdtf_type, + cdtf.de_type as cdtf_de_type, + cdtf.de_extract_type as cdtf_de_extract_type, + cdtf.ext_field as cdtf_ext_field, + cdtf.checked as cdtf_checked, + cdtf.accuracy as cdtf_accuracy, + cdtf.date_format as cdtf_date_format, + cdtf.date_format_type as cdtf_date_format_type, + cdtf.params as cdtf_params + + from `core_dataset_group` cdg + left join `core_dataset_table` cdt on cdg.id = cdt.dataset_group_id + left join `core_dataset_table_field` cdtf on cdtf.dataset_group_id = cdg.id + inner join `core_datasource` cd on cdt.datasource_id = cd.id where cdg.is_cross != 1 and (cd.STATUS IS NULL OR cd.STATUS != 'Error') ${ew.customSqlSegment} """ @@ -52,23 +77,49 @@ public interface DataSetAssistantMapper { ) temp ) SELECT - cdt.id, cdt.datasource_id, cdt.table_name, cdt.info, - cdg.name as dataset_name, - cd.name as ds_name, - cd.description as ds_desc, - cd.type as ds_type, - cd.configuration as ds_config, - cdtf.id as field_id, - cdtf.origin_name, - cdtf.name as field_show_name, - cdtf.description as field_desc, - cdtf.dataease_name, - cdtf.type as field_type + cd.id as cd_id, + cd.name as cd_name, + cd.description as cd_description, + cd.type as cd_type, + cd.configuration as cd_configuration, + + cdg.id as cdg_id, + cdg.name as cdg_name, + cdg.type as cdg_type, + cdg.mode as cdg_model, + cdg.info as cdg_info, + cdg.union_sql as cdg_union_sql, + cdg.is_cross as cdg_is_cross, + + cdt.id as cdt_id, + cdt.table_name as cdt_table_name, + cdt.type as cdt_type, + cdt.info as cdt_info, + cdt.sql_variable_details as cdt_sql_variable_details, + + cdtf.id as cdtf_id, + cdtf.origin_name as cdtf_origin_name, + cdtf.name as cdtf_name, + cdtf.description as cdtf_description, + cdtf.dataease_name as cdtf_dataease_name, + cdtf.field_short_name as cdtf_field_short_name, + cdtf.group_list as cdtf_group_list, + cdtf.other_group as cdtf_other_group, + cdtf.group_type as cdtf_group_type, + cdtf.type as cdtf_type, + cdtf.de_type as cdtf_de_type, + cdtf.de_extract_type as cdtf_de_extract_type, + cdtf.ext_field as cdtf_ext_field, + cdtf.checked as cdtf_checked, + cdtf.accuracy as cdtf_accuracy, + cdtf.date_format as cdtf_date_format, + cdtf.date_format_type as cdtf_date_format_type, + cdtf.params as cdtf_params FROM `core_dataset_table` cdt INNER JOIN `core_datasource` cd ON cdt.datasource_id = cd.id and (cd.STATUS IS NULL OR cd.STATUS != 'Error') INNER JOIN `core_dataset_group` cdg ON cdg.id = cdt.dataset_group_id AND cdg.is_cross != 1 - INNER JOIN `core_dataset_table_field` cdtf ON cdtf.dataset_table_id = cdt.id + INNER JOIN `core_dataset_table_field` cdtf ON cdtf.dataset_group_id = cdg.id where not exists( select 1 from user_ds_permissions ds_p where cd.id = ds_p.resource_id ) and not exists( select 1 from user_dg_permissions dg_p where cdg.id = dg_p.resource_id ) ${ew.customSqlSegment} @@ -118,33 +169,50 @@ public interface DataSetAssistantMapper { ) SELECT - cdt.id, cdt.datasource_id, cdt.table_name, cdt.info, - cdg.id as dataset_group_id, - cdg.name as dataset_name, - cd.name as ds_name, - cd.description as ds_desc, - cd.type as ds_type, - cd.configuration as ds_config, - cdtf.id as field_id, - cdtf.origin_name, - cdtf.name as field_show_name, - cdtf.description as field_desc, - cdtf.dataease_name, - cdtf.field_short_name, - cdtf.group_type, - cdtf.de_type, - cdtf.de_extract_type, - cdtf.ext_field, - cdtf.date_format, - cdtf.date_format_type, - cdtf.params, - cdtf.type as field_type + cd.id as cd_id, + cd.name as cd_name, + cd.description as cd_description, + cd.type as cd_type, + cd.configuration as cd_configuration, + + cdg.id as cdg_id, + cdg.name as cdg_name, + cdg.type as cdg_type, + cdg.mode as cdg_model, + cdg.info as cdg_info, + cdg.union_sql as cdg_union_sql, + cdg.is_cross as cdg_is_cross, + + cdt.id as cdt_id, + cdt.table_name as cdt_table_name, + cdt.type as cdt_type, + cdt.info as cdt_info, + cdt.sql_variable_details as cdt_sql_variable_details, + + cdtf.id as cdtf_id, + cdtf.origin_name as cdtf_origin_name, + cdtf.name as cdtf_name, + cdtf.description as cdtf_description, + cdtf.dataease_name as cdtf_dataease_name, + cdtf.field_short_name as cdtf_field_short_name, + cdtf.group_list as cdtf_group_list, + cdtf.other_group as cdtf_other_group, + cdtf.group_type as cdtf_group_type, + cdtf.type as cdtf_type, + cdtf.de_type as cdtf_de_type, + cdtf.de_extract_type as cdtf_de_extract_type, + cdtf.ext_field as cdtf_ext_field, + cdtf.checked as cdtf_checked, + cdtf.accuracy as cdtf_accuracy, + cdtf.date_format as cdtf_date_format, + cdtf.date_format_type as cdtf_date_format_type, + cdtf.params as cdtf_params FROM `core_dataset_table` cdt INNER JOIN `core_datasource` cd ON cdt.datasource_id = cd.id AND (cd.STATUS IS NULL OR cd.STATUS != 'Error') INNER JOIN `core_dataset_group` cdg ON cdg.id = cdt.dataset_group_id AND cdg.is_cross != 1 - INNER JOIN `core_dataset_table_field` cdtf ON cdtf.dataset_table_id = cdt.id + INNER JOIN `core_dataset_table_field` cdtf ON cdtf.dataset_group_id = cdg.id INNER JOIN user_ds_permissions ds_p ON cd.id = ds_p.resource_id INNER JOIN user_dg_permissions dg_p ON cdg.id = dg_p.resource_id ${ew.customSqlSegment} diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java index bb3fd768f8..e834f58245 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLBotManage.java @@ -1,41 +1,64 @@ package io.dataease.dataset.manage; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.fasterxml.jackson.core.type.TypeReference; +import io.dataease.api.dataset.union.DatasetGroupInfoDTO; import io.dataease.api.dataset.union.DatasetTableInfoDTO; +import io.dataease.api.dataset.union.UnionDTO; import io.dataease.api.dataset.vo.DataSQLBotAssistantVO; import io.dataease.api.dataset.vo.SQLBotAssistanTable; import io.dataease.api.dataset.vo.SQLBotAssistantField; import io.dataease.api.permissions.dataset.api.ColumnPermissionsApi; -import io.dataease.api.permissions.dataset.api.RowPermissionsApi; import io.dataease.api.permissions.dataset.dto.DataSetColumnPermissionsDTO; import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO; -import io.dataease.api.permissions.dataset.dto.DatasetRowPermissionsTreeRequest; import io.dataease.api.permissions.role.api.RoleApi; import io.dataease.api.permissions.role.dto.RoleRequest; import io.dataease.api.permissions.role.vo.RoleVO; import io.dataease.auth.bo.TokenUserBO; import io.dataease.commons.utils.EncryptUtils; import io.dataease.constant.ColumnPermissionConstants; +import io.dataease.dataset.dao.auto.entity.CoreDatasetGroup; import io.dataease.dataset.dao.ext.mapper.DataSetAssistantMapper; +import io.dataease.dataset.utils.TableUtils; import io.dataease.datasource.dao.auto.entity.CoreDatasource; import io.dataease.datasource.manage.EngineManage; +import io.dataease.engine.constant.ExtFieldConstant; +import io.dataease.engine.sql.SQLProvider; +import io.dataease.engine.trans.Field2SQLObj; +import io.dataease.engine.trans.Order2SQLObj; +import io.dataease.engine.trans.Table2SQLObj; +import io.dataease.engine.trans.WhereTree2Str; +import io.dataease.engine.utils.Utils; +import io.dataease.exception.DEException; +import io.dataease.extensions.datasource.api.PluginManageApi; +import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO; +import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO; +import io.dataease.extensions.datasource.factory.ProviderFactory; +import io.dataease.extensions.datasource.model.SQLMeta; +import io.dataease.extensions.datasource.provider.Provider; import io.dataease.extensions.datasource.vo.Configuration; import io.dataease.extensions.datasource.vo.DatasourceConfiguration; import io.dataease.extensions.view.dto.ColumnPermissionItem; import io.dataease.extensions.view.dto.ColumnPermissions; +import io.dataease.extensions.view.dto.DatasetRowPermissionsTreeItem; +import io.dataease.extensions.view.dto.DatasetRowPermissionsTreeObj; import io.dataease.home.manage.DeIndexManage; +import io.dataease.i18n.Translator; import io.dataease.utils.*; import jakarta.annotation.Resource; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; import java.util.*; +import java.util.function.Function; import java.util.stream.Collectors; + @Component public class DatasetSQLBotManage { @@ -56,6 +79,12 @@ public class DatasetSQLBotManage { @Resource private PermissionManage permissionManage; + @Resource + private DatasetSQLManage datasetSQLManage; + + @Autowired(required = false) + private PluginManageApi pluginManage; + @Value("${dataease.sqlbot.encrypt:false}") private boolean encryptEnabled; @@ -102,29 +131,8 @@ public class DatasetSQLBotManage { } private Map> getRowPermission(Long uid, List roleIds) { - RowPermissionsApi rowPermissionsApi = CommonBeanFactory.getBean(RowPermissionsApi.class); - Objects.requireNonNull(rowPermissionsApi); - - DatasetRowPermissionsTreeRequest request = new DatasetRowPermissionsTreeRequest(); - request.setEnable(true); - - request.setAuthTargetId(uid); - request.setAuthTargetType("user"); - List permissionsTreeDTOS = rowPermissionsApi.list(request); - - if (ObjectUtils.isNotEmpty(roleIds)) { - request.setAuthTargetId(null); - request.setAuthTargetIds(roleIds); - request.setAuthTargetType("role"); - List rolePermissionDTOS = rowPermissionsApi.list(request); - if (CollectionUtils.isNotEmpty(rolePermissionDTOS)) { - permissionsTreeDTOS.addAll(rolePermissionDTOS); - } - } - if (CollectionUtils.isEmpty(permissionsTreeDTOS)) { - return null; - } - return permissionsTreeDTOS.stream().collect(Collectors.groupingBy(DataSetRowPermissionsTreeDTO::getDatasetId)); + List datasetRowPermissions = permissionManage.getRowPermissionsTree(null, uid); + return datasetRowPermissions.stream().collect(Collectors.groupingBy(DataSetRowPermissionsTreeDTO::getDatasetId)); } @@ -138,7 +146,6 @@ public class DatasetSQLBotManage { Boolean model = deIndexManage.xpackModel(); List> list = null; boolean isAdmin = uid == 1; - boolean withColsOrRowsPermission = false; QueryWrapper queryWrapper = new QueryWrapper<>(); if (ObjectUtils.isNotEmpty(datasetId)) { queryWrapper.eq("cdg.id", datasetId); @@ -165,9 +172,8 @@ public class DatasetSQLBotManage { isRootRole = roleVOS.stream().anyMatch(RoleVO::isRoot); roleIds = roleVOS.stream().map(RoleVO::getId).toList(); - /*colPermissionMap = getColPermission(uid, roleIds); + colPermissionMap = getColPermission(uid, roleIds); rowPermissionMap = getRowPermission(uid, roleIds); - withColsOrRowsPermission = MapUtils.isNotEmpty(colPermissionMap) || MapUtils.isNotEmpty(rowPermissionMap);*/ } list = dataSetAssistantMapper.queryEnterprise(oid, uid, isRootRole, queryWrapper); } @@ -180,7 +186,7 @@ public class DatasetSQLBotManage { deEngine = engineManage.getDeEngine(); for (Map row : list) { // build ds - String datasourceId = row.get("datasource_id").toString(); + String datasourceId = row.get("cd_id").toString(); DataSQLBotAssistantVO vo = dsFlagMap.get(datasourceId); if (ObjectUtils.isEmpty(vo)) { vo = buildDs(row); @@ -190,7 +196,7 @@ public class DatasetSQLBotManage { result.add(vo); } // build table - String tableId = row.get("id").toString(); + String tableId = row.get("cdg_id").toString(); SQLBotAssistanTable table = tableFlagMap.get(tableId); if (ObjectUtils.isEmpty(table)) { table = buildTable(row); @@ -200,7 +206,7 @@ public class DatasetSQLBotManage { vo.getTables().add(table); } // build field - String fieldId = row.get("field_id").toString(); + String fieldId = row.get("cdtf_id").toString(); SQLBotAssistantField field = fieldFlagMap.get(fieldId); if (ObjectUtils.isEmpty(field)) { field = buildField(row); @@ -208,71 +214,200 @@ public class DatasetSQLBotManage { continue; fieldFlagMap.put(fieldId, field); table.getFields().add(field); + if (field.isNeedTransform()) { + table.setNeedTransform(true); + } } } - /*if (withColsOrRowsPermission) { - result = filterPermissions(result, list, colPermissionMap, rowPermissionMap); - }*/ + filterPermissions(result, list, colPermissionMap, rowPermissionMap); LogUtil.info("sqlbot ds api result: {}", result); return result; } - private List filterPermissions( + public void buildFieldName(Map sqlMap, List fields) { + // 获取内层union sql和字段 + List unionFields = (List) sqlMap.get("field"); + for (DatasetTableFieldDTO datasetTableFieldDTO : fields) { + if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_NORMAL)) { + for (DatasetTableFieldDTO fieldDTO : unionFields) { + if (Objects.equals(datasetTableFieldDTO.getDatasetTableId(), fieldDTO.getDatasetTableId()) + && Objects.equals(datasetTableFieldDTO.getOriginName(), fieldDTO.getOriginName())) { + datasetTableFieldDTO.setDataeaseName(fieldDTO.getDataeaseName()); + datasetTableFieldDTO.setFieldShortName(fieldDTO.getFieldShortName()); + } + } + } + if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_CALC)) { + String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); + datasetTableFieldDTO.setDataeaseName(dataeaseName); + datasetTableFieldDTO.setFieldShortName(dataeaseName); + datasetTableFieldDTO.setDeExtractType(datasetTableFieldDTO.getDeType()); + } + if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_GROUP)) { + String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); + datasetTableFieldDTO.setDataeaseName(dataeaseName); + datasetTableFieldDTO.setFieldShortName(dataeaseName); + datasetTableFieldDTO.setDeExtractType(0); + datasetTableFieldDTO.setDeType(0); + datasetTableFieldDTO.setGroupType("d"); + } + } + } + + public void getField(DatasetRowPermissionsTreeObj tree, Map fieldMap) { + if (ObjectUtils.isNotEmpty(tree)) { + if (ObjectUtils.isNotEmpty(tree.getItems())) { + for (DatasetRowPermissionsTreeItem item : tree.getItems()) { + if (ObjectUtils.isNotEmpty(item)) { + if (StringUtils.equalsIgnoreCase(item.getType(), "item") || ObjectUtils.isEmpty(item.getSubTree())) { + item.setField(fieldMap.get(item.getFieldId())); + } else if (StringUtils.equalsIgnoreCase(item.getType(), "tree") || (ObjectUtils.isNotEmpty(item.getSubTree()) && StringUtils.isNotEmpty(item.getSubTree().getLogic()))) { + getField(item.getSubTree(), fieldMap); + } + } + } + } + } + } + + private void rebuildTable(SQLBotAssistanTable table, List columnPermissionsDTOS, List rowPermissionsTree) { + Map rowData = table.getRowData(); + CoreDatasetGroup coreDatasetGroup = BeanUtils.mapToBean(rowData, CoreDatasetGroup.class); + + DatasetGroupInfoDTO datasetGroupInfoDTO = new DatasetGroupInfoDTO(); + BeanUtils.copyBean(datasetGroupInfoDTO, coreDatasetGroup); + + datasetGroupInfoDTO.setUnionSql(null); + List unionDTOList = JsonUtil.parseList(coreDatasetGroup.getInfo(), new TypeReference<>() { + }); + datasetGroupInfoDTO.setUnion(unionDTOList); + + List sqlbotFields = table.getFields(); + + List dsFields = sqlbotFields.stream().map(field -> { + Map fieldRowData = field.getRowData(); + DatasetTableFieldDTO fieldDTO = BeanUtils.mapToBean(fieldRowData, DatasetTableFieldDTO.class); + fieldDTO.setFieldShortName(fieldDTO.getDataeaseName()); + return fieldDTO; + }).collect(Collectors.toList()); + + datasetGroupInfoDTO.setAllFields(dsFields); + Map fieldMap = dsFields.stream().collect(Collectors.toMap(DatasetTableFieldDTO::getId, Function.identity())); + if (CollectionUtils.isNotEmpty(rowPermissionsTree)) { + rowPermissionsTree.forEach(treeDTO -> { + DatasetRowPermissionsTreeObj tree = treeDTO.getTree(); + getField(tree, fieldMap); + }); + } + + Map sqlMap = null; + try { + sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null); + } catch (Exception e) { + throw new RuntimeException(e); + } + String sql = (String) sqlMap.get("sql"); + + // 获取allFields + List fields = datasetGroupInfoDTO.getAllFields(); + if (ObjectUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_no_fields")); + } + + List originFields = new ArrayList<>(fields); + Map desensitizationList = new HashMap<>(); + if (CollectionUtils.isNotEmpty(columnPermissionsDTOS)) { + List columnPermissionItems = new ArrayList<>(); + for (DataSetColumnPermissionsDTO dataSetColumnPermissionsDTO : columnPermissionsDTOS) { + ColumnPermissions columnPermissions = JsonUtil.parseObject(dataSetColumnPermissionsDTO.getPermissions(), ColumnPermissions.class); + if (!columnPermissions.getEnable()) { + continue; + } + if (StringUtils.equalsAnyIgnoreCase(dataSetColumnPermissionsDTO.getAuthTargetType(), "user", "role")) { + columnPermissionItems.addAll(columnPermissions.getColumns().stream().filter(columnPermissionItem -> columnPermissionItem.getSelected()).collect(Collectors.toList())); + } + } + fields = fields.stream().filter(field -> { + List fieldColumnPermissionItems = columnPermissionItems.stream().filter(columnPermissionItem -> columnPermissionItem.getId().equals(field.getId())).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(fieldColumnPermissionItems)) { + return true; + } + return fieldColumnPermissionItems.stream().map(ColumnPermissionItem::getOpt).toList().contains(ColumnPermissionConstants.Desensitization); + }).collect(Collectors.toList()); + // fields = permissionManage.filterColumnPermissions(fields, desensitizationList, datasetGroupInfoDTO.getId(), null); + if (ObjectUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_no_column_permission")); + } + } + buildFieldName(sqlMap, originFields); + Map dsMap = (Map) sqlMap.get("dsMap"); + List dsList = new ArrayList<>(); + for (Map.Entry next : dsMap.entrySet()) { + dsList.add(next.getValue().getType()); + } + boolean needOrder = Utils.isNeedOrder(dsList); + sql = Utils.replaceSchemaAlias(sql, dsMap); + Provider provider = ProviderFactory.getProvider(dsList.getFirst()); + + // build query sql + SQLMeta sqlMeta = new SQLMeta(); + Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", false); + Field2SQLObj.field2sqlObj(sqlMeta, fields, fields, false, dsMap, Utils.getParams(fields), null, pluginManage, true); + WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields, false, dsMap, Utils.getParams(fields), null, pluginManage); + Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), fields, false, dsMap, Utils.getParams(fields), null, pluginManage); + String querySQL; + querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, false); + querySQL = provider.rebuildSQL(querySQL, sqlMeta, false, dsMap); + table.setSql(querySQL); + } + + private void filterPermissions( List vos, List> list, Map> colPermissionMap, Map> rowPermissionMap ) { if (CollectionUtils.isEmpty(vos)) { - return vos; + return; } vos.forEach(vo -> { List tables = vo.getTables(); tables.forEach(table -> { Long datasetGroupId = table.getDatasetGroupId(); - List dataSetColumnPermissionsDTOS = colPermissionMap.get(datasetGroupId); - - List columnPermissionItems = new ArrayList<>(); - - for (DataSetColumnPermissionsDTO dataSetColumnPermissionsDTO : dataSetColumnPermissionsDTOS) { - ColumnPermissions columnPermissions = JsonUtil.parseObject(dataSetColumnPermissionsDTO.getPermissions(), ColumnPermissions.class); - if (!columnPermissions.getEnable()) { - continue; - } - if (StringUtils.equalsAnyIgnoreCase(dataSetColumnPermissionsDTO.getAuthTargetType(), "user", "role")) { - columnPermissionItems.addAll(columnPermissions.getColumns().stream().filter(columnPermissionItem -> columnPermissionItem.getSelected()).collect(Collectors.toList())); - } + List columnPermissionsDTOS = ObjectUtils.isEmpty(colPermissionMap) ? null : colPermissionMap.get(datasetGroupId); + List rowPermissionsTreeDTOS = ObjectUtils.isEmpty(rowPermissionMap) ? null : rowPermissionMap.get(datasetGroupId); + if (table.isNeedTransform() || ObjectUtils.isNotEmpty(columnPermissionsDTOS) || ObjectUtils.isNotEmpty(rowPermissionsTreeDTOS)) { + rebuildTable(table, columnPermissionsDTOS, rowPermissionsTreeDTOS); } - List filterFields = table.getFields().stream().filter(field -> { - List fieldColumnPermissionItems = columnPermissionItems.stream().filter(columnPermissionItem -> columnPermissionItem.getId().equals(field.getFieldId())).collect(Collectors.toList()); - if (CollectionUtils.isEmpty(fieldColumnPermissionItems)) { - return true; - } - return fieldColumnPermissionItems.stream().map(ColumnPermissionItem::getOpt).toList().contains(ColumnPermissionConstants.Desensitization); - }).collect(Collectors.toList()); - table.setFields(filterFields); }); }); - return vos; } private SQLBotAssistantField buildField(Map row) { SQLBotAssistantField field = new SQLBotAssistantField(); - if (ObjectUtils.isNotEmpty(row.get("field_id"))) { - field.setFieldId(Long.parseLong(row.get("field_id").toString())); + if (ObjectUtils.isNotEmpty(row.get("cdtf_id"))) { + field.setFieldId(Long.parseLong(row.get("cdtf_id").toString())); } - if (ObjectUtils.isNotEmpty(row.get("dataease_name"))) { - field.setDataeaseName(row.get("dataease_name").toString()); + if (ObjectUtils.isNotEmpty(row.get("cdtf_dataease_name"))) { + field.setDataeaseName(row.get("cdtf_dataease_name").toString()); } - field.setName(row.get("origin_name").toString()); - field.setType(row.get("field_type").toString()); - field.setComment(row.get("field_show_name").toString()); + field.setName(row.get("cdtf_origin_name").toString()); + field.setType(row.get("cdtf_type").toString()); + field.setComment(row.get("cdtf_name").toString()); + if (ObjectUtils.isNotEmpty(row.get("cdtf_ext_field")) && !row.get("cdtf_ext_field").equals(0)) { + field.setNeedTransform(true); + } + Map fieldRowData = buildRowData(row, 3); + fieldRowData.put("datasource_id", Long.parseLong(row.get("cd_id").toString())); + fieldRowData.put("dataset_group_id", row.get("cdg_id")); + fieldRowData.put("dataset_table_id", row.get("cdt_id")); + field.setRowData(fieldRowData); return field; } private DataSQLBotAssistantVO buildDs(Map row) { - Object dsConfig = row.get("ds_config"); + Object dsConfig = row.get("cd_configuration"); if (ObjectUtils.isEmpty(dsConfig) || StringUtils.isBlank(dsConfig.toString())) { return null; } @@ -280,7 +415,7 @@ public class DatasetSQLBotManage { if (StringUtils.isBlank(dsHost)) { dsHost = environment.getProperty("dataease.dataease-servers", String.class); } - String dsType = row.get("ds_type").toString(); + String dsType = row.get("cd_type").toString(); Configuration config = null; if (dsType.contains(DatasourceConfiguration.DatasourceType.Excel.name()) || dsType.contains(DatasourceConfiguration.DatasourceType.API.name())) { String config_json = EncryptUtils.aesDecrypt(deEngine.getConfiguration()).toString(); @@ -298,18 +433,31 @@ public class DatasetSQLBotManage { vo.setExtraParams(config.getExtraParams()); vo.setHost(config.getHost()); vo.setPort(config.getPort()); - vo.setName(row.get("ds_name").toString()); - vo.setComment(ObjectUtils.isEmpty(row.get("ds_desc")) ? vo.getName() : row.get("ds_desc").toString()); + vo.setName(row.get("cd_name").toString()); + vo.setComment(ObjectUtils.isEmpty(row.get("cd_description")) ? vo.getName() : row.get("cd_description").toString()); vo.setType(dsType); vo.setSchema(config.getSchema()); vo.setUser(config.getUsername()); vo.setPassword(config.getPassword()); + vo.setRowData(buildRowData(row, 0)); if (encryptEnabled) { aesVO(vo); } return vo; } + private Map buildRowData(Map row, int level) { + String[] levels = {"cd_", "cdg_", "cdt_", "cdtf_"}; + String alias = levels[level]; + Map filteredMap = new HashMap<>(); + row.forEach((key, value) -> { + if (key.startsWith(alias)) { + filteredMap.put(key.substring(alias.length()), value); + } + }); + return filteredMap; + } + private void aesVO(DataSQLBotAssistantVO vo) { if (StringUtils.isNotBlank(vo.getHost())) { vo.setHost(aesEncrypt(vo.getHost())); @@ -330,21 +478,25 @@ public class DatasetSQLBotManage { private SQLBotAssistanTable buildTable(Map row) { SQLBotAssistanTable table = new SQLBotAssistanTable(); - table.setName(row.get("table_name").toString()); - table.setComment(row.get("dataset_name").toString()); - if (ObjectUtils.isNotEmpty(row.get("dataset_group_id"))) { - table.setDatasetGroupId(Long.parseLong(row.get("dataset_group_id").toString())); - } + table.setName(row.get("cdg_name").toString()); + table.setComment(row.get("cdg_name").toString()); + table.setDatasetGroupId(Long.parseLong(row.get("cdg_id").toString())); Object infoObj = null; - if (ObjectUtils.isNotEmpty(infoObj = row.get("info"))) { + if (ObjectUtils.isNotEmpty(infoObj = row.get("cdt_info"))) { String info = infoObj.toString(); DatasetTableInfoDTO tableInfoDTO = JsonUtil.parseObject(info, DatasetTableInfoDTO.class); if (StringUtils.isNotBlank(tableInfoDTO.getSql())) { String sql = new String(Base64.getDecoder().decode(tableInfoDTO.getSql())); table.setSql(sql); } + if (StringUtils.isNotBlank(tableInfoDTO.getTable())) { + table.setName(tableInfoDTO.getTable()); + } } + Map tableRowData = buildRowData(row, 1); + tableRowData.put("datasource_id", Long.parseLong(row.get("cd_id").toString())); + table.setRowData(tableRowData); return table; } diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/PermissionManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/PermissionManage.java index cf62ff9541..4182a481a6 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/PermissionManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/PermissionManage.java @@ -138,8 +138,10 @@ public class PermissionManage { // 获取当前数据集下，当前用户、角色、组织所有的行权限（非白名单，非禁用） List records = rowPermissionsTree(datasetId, user); // 构建权限tree中的field，如果field不存在，置为null - for (DataSetRowPermissionsTreeDTO record : records) { - getField(record.getTree()); + if (ObjectUtils.isNotEmpty(datasetId)) { + for (DataSetRowPermissionsTreeDTO record : records) { + getField(record.getTree()); + } } return records; } @@ -154,7 +156,9 @@ public class PermissionManage { UserFormVO userEntity = getRowPermissionsApi().getUserById(userId); List roleIds = userEntity.getRoleIds().stream().map(x -> Long.valueOf(x)).collect(Collectors.toList()); DatasetRowPermissionsTreeRequest dataSetRowPermissionsDTO = new DatasetRowPermissionsTreeRequest(); - dataSetRowPermissionsDTO.setDatasetId(datasetId); + if (ObjectUtils.isNotEmpty(datasetId)) { + dataSetRowPermissionsDTO.setDatasetId(datasetId); + } dataSetRowPermissionsDTO.setEnable(true); if (ObjectUtils.isNotEmpty(userId)) { diff --git a/core/core-backend/src/main/java/io/dataease/engine/trans/Field2SQLObj.java b/core/core-backend/src/main/java/io/dataease/engine/trans/Field2SQLObj.java index 4c2544c109..8f76f7110f 100644 --- a/core/core-backend/src/main/java/io/dataease/engine/trans/Field2SQLObj.java +++ b/core/core-backend/src/main/java/io/dataease/engine/trans/Field2SQLObj.java @@ -1,8 +1,8 @@ package io.dataease.engine.trans; import io.dataease.constant.DeTypeConstants; -import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.constant.SQLConstants; +import io.dataease.engine.constant.ExtFieldConstant; import io.dataease.engine.func.FunctionConstant; import io.dataease.engine.utils.Utils; import io.dataease.extensions.datasource.api.PluginManageApi; @@ -17,12 +17,14 @@ import org.apache.commons.lang3.StringUtils; import java.util.*; +import static io.dataease.constant.SQLConstants.FIELD_DOT_FIX; + /** * @Author Junjun */ public class Field2SQLObj { - public static void field2sqlObj(SQLMeta meta, List fields, List originFields, boolean isCross, Map dsMap, List fieldParam, List chartParam, PluginManageApi pluginManage) { + public static void field2sqlObj(SQLMeta meta, List fields, List originFields, boolean isCross, Map dsMap, List fieldParam, List chartParam, PluginManageApi pluginManage, boolean forSqlbot) { SQLObj tableObj = meta.getTable(); if (ObjectUtils.isEmpty(tableObj)) { return; @@ -79,6 +81,13 @@ public class Field2SQLObj { } } String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); + if (forSqlbot) { + fieldAlias = x.getOriginName(); + if (ObjectUtils.isNotEmpty(x.getExtField()) && !x.getExtField().equals(ExtFieldConstant.EXT_NORMAL) && StringUtils.isNotBlank(x.getName())) { + fieldAlias = x.getName(); + } + fieldAlias = String.format(FIELD_DOT_FIX, fieldAlias); + } // 处理横轴字段 xFields.add(getXFields(x, originField, fieldAlias, isCross)); } @@ -87,6 +96,10 @@ public class Field2SQLObj { meta.setXFieldsDialect(fieldsDialect); } + public static void field2sqlObj(SQLMeta meta, List fields, List originFields, boolean isCross, Map dsMap, List fieldParam, List chartParam, PluginManageApi pluginManage) { + field2sqlObj(meta, fields, originFields, isCross, dsMap, fieldParam, chartParam, pluginManage, false); + } + public static SQLObj getXFields(DatasetTableFieldDTO f, String originField, String fieldAlias, boolean isCross) { String fieldName = ""; if (originField != null) { diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/DataSQLBotAssistantVO.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/DataSQLBotAssistantVO.java index 0e796090e1..4b9c4ad1ca 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/DataSQLBotAssistantVO.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/DataSQLBotAssistantVO.java @@ -1,10 +1,12 @@ package io.dataease.api.dataset.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Map; @Data public class DataSQLBotAssistantVO implements Serializable { @@ -31,4 +33,7 @@ public class DataSQLBotAssistantVO implements Serializable { private List tables = new ArrayList<>(); + @JsonIgnore + private Map rowData; + } diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistanTable.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistanTable.java index d9414ed4fa..29256457ea 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistanTable.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistanTable.java @@ -6,6 +6,7 @@ import lombok.Data; import java.io.Serializable; import java.util.ArrayList; import java.util.List; +import java.util.Map; @Data public class SQLBotAssistanTable implements Serializable { @@ -17,5 +18,11 @@ public class SQLBotAssistanTable implements Serializable { @JsonIgnore private Long datasetGroupId; + @JsonIgnore + private boolean needTransform; + @JsonIgnore + private boolean needPermission; + @JsonIgnore + private Map rowData; } diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistantField.java b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistantField.java index 970c9f0e4c..1c1bd22371 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistantField.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/dataset/vo/SQLBotAssistantField.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import java.io.Serializable; +import java.util.Map; @Data public class SQLBotAssistantField implements Serializable { @@ -15,4 +16,11 @@ public class SQLBotAssistantField implements Serializable { private Long fieldId; @JsonIgnore private String dataeaseName; + + @JsonIgnore + private boolean needTransform; + @JsonIgnore + private boolean needPermission; + @JsonIgnore + private Map rowData; } diff --git a/sdk/common/src/main/java/io/dataease/utils/BeanUtils.java b/sdk/common/src/main/java/io/dataease/utils/BeanUtils.java index bd513040ee..a06bc70a27 100644 --- a/sdk/common/src/main/java/io/dataease/utils/BeanUtils.java +++ b/sdk/common/src/main/java/io/dataease/utils/BeanUtils.java @@ -2,10 +2,12 @@ package io.dataease.utils; import org.apache.commons.lang3.StringUtils; +import java.beans.PropertyDescriptor; import java.lang.reflect.Field; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; +import java.util.Map; public class BeanUtils { @@ -77,4 +79,64 @@ public class BeanUtils { } return fieldNames; } + + private static String underscoreToCamel(String underscore) { + StringBuilder result = new StringBuilder(); + String[] parts = underscore.split("_"); + for (int i = 0; i < parts.length; i++) { + String part = parts[i]; + if (part.isEmpty()) continue; + if (i == 0) { + result.append(part); + } else { + result.append(Character.toUpperCase(part.charAt(0))) + .append(part.substring(1).toLowerCase()); + } + } + return result.toString(); + } + + + public static T mapToBean(Map map, Class clazz) { + try { + T bean = clazz.getDeclaredConstructor().newInstance(); + PropertyDescriptor[] descriptors = org.springframework.beans.BeanUtils.getPropertyDescriptors(clazz); + + for (PropertyDescriptor descriptor : descriptors) { + String propertyName = descriptor.getName(); + if ("class".equals(propertyName)) continue; + + // 查找Map中对应的key（支持驼峰和下划线） + Object value = findValueInMap(map, propertyName); + if (value != null && descriptor.getWriteMethod() != null) { + descriptor.getWriteMethod().invoke(bean, value); + } + } + return bean; + } catch (Exception e) { + throw new RuntimeException("Map转Bean失败", e); + } + } + + private static Object findValueInMap(Map map, String propertyName) { + Object value = map.get(propertyName); + if (value != null) return value; + + String underscore = camelToUnderscore(propertyName); + value = map.get(underscore); + if (value != null) return value; + + for (Map.Entry entry : map.entrySet()) { + String camelKey = underscoreToCamel(entry.getKey()); + if (propertyName.equals(camelKey)) { + return entry.getValue(); + } + } + + return null; + } + + private static String camelToUnderscore(String camel) { + return camel.replaceAll("([a-z])([A-Z])", "$1_$2").toLowerCase(); + } }