From 58384931c0d70a7f80c9a4794bfac6d96b37e329 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Wed, 14 May 2025 19:11:25 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E3=80=90=E6=93=8D=E4=BD=9C=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E3=80=91=E6=96=B0=E5=A2=9E=E6=93=8D=E4=BD=9C=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E4=B8=BA=E3=80=90=E5=AF=BC=E5=87=BA=E3=80=91=E5=92=8C?= =?UTF-8?q?=E3=80=90=E4=B8=8B=E8=BD=BD=E3=80=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../chart/server/ChartDataServer.java | 1 - .../dao/auto/entity/CoreExportTask.java | 6 +- .../manage/ExportCenterDownLoadManage.java | 593 ++++++++++++++++++ .../manage/ExportCenterManage.java | 584 ++--------------- .../server/ExportCenterServer.java | 6 + .../server/DataVisualizationServer.java | 4 +- core/core-frontend/src/api/dataset.ts | 6 + .../visualized/data/dataset/ExportExcel.vue | 15 +- .../api/exportCenter/ExportCenterApi.java | 4 + .../main/java/io/dataease/constant/LogOT.java | 3 +- .../main/java/io/dataease/constant/LogST.java | 3 +- 11 files changed, 674 insertions(+), 551 deletions(-) create mode 100644 core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterDownLoadManage.java diff --git a/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java b/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java index f3996c7eac..780eccf0af 100644 --- a/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java +++ b/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java @@ -309,7 +309,6 @@ public class ChartDataServer implements ChartDataApi { } } else { exportCenterManage.addTask(request.getViewId(), "chart", request); - return; } } diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/dao/auto/entity/CoreExportTask.java b/core/core-backend/src/main/java/io/dataease/exportCenter/dao/auto/entity/CoreExportTask.java index 0ae560d6e0..9788eb02db 100644 --- a/core/core-backend/src/main/java/io/dataease/exportCenter/dao/auto/entity/CoreExportTask.java +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/dao/auto/entity/CoreExportTask.java @@ -26,7 +26,7 @@ public class CoreExportTask implements Serializable { private String fileSizeUnit; - private String exportFrom; + private Long exportFrom; private String exportStatus; @@ -88,11 +88,11 @@ public class CoreExportTask implements Serializable { this.fileSizeUnit = fileSizeUnit; } - public String getExportFrom() { + public Long getExportFrom() { return exportFrom; } - public void setExportFrom(String exportFrom) { + public void setExportFrom(Long exportFrom) { this.exportFrom = exportFrom; } diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterDownLoadManage.java b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterDownLoadManage.java new file mode 100644 index 0000000000..986eb7a9f0 --- /dev/null +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterDownLoadManage.java @@ -0,0 +1,593 @@ +package io.dataease.exportCenter.manage; + + +import com.fasterxml.jackson.core.type.TypeReference; +import io.dataease.api.chart.dto.ViewDetailField; +import io.dataease.api.chart.request.ChartExcelRequest; +import io.dataease.api.chart.request.ChartExcelRequestInner; +import io.dataease.api.dataset.dto.DataSetExportRequest; +import io.dataease.api.dataset.union.DatasetGroupInfoDTO; +import io.dataease.api.dataset.union.UnionDTO; +import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO; +import io.dataease.api.permissions.user.vo.UserFormVO; +import io.dataease.api.xpack.dataFilling.DataFillingApi; +import io.dataease.api.xpack.dataFilling.dto.DataFillFormTableDataRequest; +import io.dataease.auth.bo.TokenUserBO; +import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper; +import io.dataease.chart.server.ChartDataServer; +import io.dataease.commons.utils.ExcelWatermarkUtils; +import io.dataease.constant.DeTypeConstants; +import io.dataease.constant.LogOT; +import io.dataease.constant.LogST; +import io.dataease.dataset.dao.auto.entity.CoreDatasetGroup; +import io.dataease.dataset.dao.auto.mapper.CoreDatasetGroupMapper; +import io.dataease.dataset.manage.*; +import io.dataease.datasource.utils.DatasourceUtils; +import io.dataease.engine.sql.SQLProvider; +import io.dataease.engine.trans.Field2SQLObj; +import io.dataease.engine.trans.Order2SQLObj; +import io.dataease.engine.trans.Table2SQLObj; +import io.dataease.engine.trans.WhereTree2Str; +import io.dataease.engine.utils.Utils; +import io.dataease.exception.DEException; +import io.dataease.exportCenter.dao.auto.entity.CoreExportTask; +import io.dataease.exportCenter.dao.auto.mapper.CoreExportTaskMapper; +import io.dataease.exportCenter.util.ExportCenterUtils; +import io.dataease.extensions.datasource.api.PluginManageApi; +import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO; +import io.dataease.extensions.datasource.dto.DatasourceRequest; +import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO; +import io.dataease.extensions.datasource.factory.ProviderFactory; +import io.dataease.extensions.datasource.model.SQLMeta; +import io.dataease.extensions.datasource.provider.Provider; +import io.dataease.extensions.view.dto.ChartViewDTO; +import io.dataease.extensions.view.dto.ColumnPermissionItem; +import io.dataease.extensions.view.dto.DatasetRowPermissionsTreeObj; +import io.dataease.i18n.Translator; +import io.dataease.license.utils.LicenseUtil; +import io.dataease.log.DeLog; +import io.dataease.model.ExportTaskDTO; +import io.dataease.utils.*; +import io.dataease.visualization.dao.auto.entity.VisualizationWatermark; +import io.dataease.visualization.dao.auto.mapper.VisualizationWatermarkMapper; +import io.dataease.visualization.dao.ext.mapper.ExtDataVisualizationMapper; +import io.dataease.visualization.dto.WatermarkContentDTO; +import io.dataease.websocket.WsMessage; +import io.dataease.websocket.WsService; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.Resource; +import jakarta.servlet.http.HttpServletResponse; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import java.io.*; +import java.net.URLEncoder; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.charset.StandardCharsets; +import java.text.DecimalFormat; +import java.util.*; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Component +@Transactional(rollbackFor = Exception.class) +public class ExportCenterDownLoadManage { + @Resource + private CoreExportTaskMapper exportTaskMapper; + @Resource + private CoreChartViewMapper coreChartViewMapper; + @Resource + private PermissionManage permissionManage; + @Autowired + private WsService wsService; + @Autowired(required = false) + private PluginManageApi pluginManage; + @Value("${dataease.export.core.size:10}") + private int core; + @Value("${dataease.export.max.size:10}") + private int max; + + @Value("${dataease.path.exportData:/opt/dataease2.0/data/exportData/}") + private String exportData_path; + @Resource + private VisualizationWatermarkMapper watermarkMapper; + @Resource + private ExtDataVisualizationMapper visualizationMapper; + @Value("${dataease.export.page.size:50000}") + private Integer extractPageSize; + static private List STATUS = Arrays.asList("SUCCESS", "FAILED", "PENDING", "IN_PROGRESS", "ALL"); + private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; + private int keepAliveSeconds = 600; + private Map Running_Task = new HashMap<>(); + @Resource + private ChartDataServer chartDataServer; + @Resource + private CoreDatasetGroupMapper coreDatasetGroupMapper; + @Resource + private DatasetSQLManage datasetSQLManage; + @Resource + private DatasetTableFieldManage datasetTableFieldManage; + @Resource + private DatasetDataManage datasetDataManage; + private final Long sheetLimit = 1000000L; + @Autowired(required = false) + private DataFillingApi dataFillingApi = null; + + private DataFillingApi getDataFillingApi() { + return dataFillingApi; + } + + @PostConstruct + public void init() { + scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(core); + scheduledThreadPoolExecutor.setKeepAliveTime(keepAliveSeconds, TimeUnit.SECONDS); + scheduledThreadPoolExecutor.setMaximumPoolSize(max); + } + + @Scheduled(fixedRate = 5000) + public void checkRunningTask() { + Iterator> iterator = Running_Task.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + if (entry.getValue().isDone()) { + iterator.remove(); + try { + CoreExportTask exportTask = exportTaskMapper.selectById(entry.getKey()); + ExportTaskDTO exportTaskDTO = new ExportTaskDTO(); + BeanUtils.copyBean(exportTaskDTO, exportTask); + setExportFromName(exportTaskDTO); + WsMessage message = new WsMessage(exportTask.getUserId(), "/task-export-topic", exportTaskDTO); + wsService.releaseMessage(message); + } catch (Exception e) { + + } + } + } + } + + private void setExportFromName(ExportTaskDTO exportTaskDTO) { + if (exportTaskDTO.getExportFromType().equalsIgnoreCase("chart")) { + exportTaskDTO.setExportFromName(coreChartViewMapper.selectById(exportTaskDTO.getExportFrom()).getTitle()); + } + if (exportTaskDTO.getExportFromType().equalsIgnoreCase("dataset")) { + exportTaskDTO.setExportFromName(coreDatasetGroupMapper.selectById(exportTaskDTO.getExportFrom()).getName()); + } + if (exportTaskDTO.getExportFromType().equalsIgnoreCase("data_filling")) { + exportTaskDTO.setExportFromName(getDataFillingApi().get(Long.parseLong(exportTaskDTO.getExportFrom())).getName()); + } + } + + @DeLog(id = "#p0.exportFrom", ot = LogOT.EXPORT, st = LogST.DRIVER_FILE) + public void startDataFillingTask(CoreExportTask exportTask, HashMap request) { + if (ObjectUtils.isEmpty(getDataFillingApi())) { + return; + } + String dataPath = exportData_path + exportTask.getId(); + File directory = new File(dataPath); + boolean isCreated = directory.mkdir(); + TokenUserBO tokenUserBO = AuthUtils.getUser(); + Future future = scheduledThreadPoolExecutor.submit(() -> { + AuthUtils.setUser(tokenUserBO); + try { + exportTask.setExportStatus("IN_PROGRESS"); + exportTaskMapper.updateById(exportTask); + getDataFillingApi().writeExcel(dataPath + "/" + exportTask.getId() + ".xlsx", new DataFillFormTableDataRequest().setId(exportTask.getExportFrom()).setWithoutLogs(true), exportTask.getUserId(), Long.parseLong(request.get("org").toString())); + exportTask.setExportProgress("100"); + exportTask.setExportStatus("SUCCESS"); + + setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); + } catch (Exception e) { + exportTask.setMsg(e.getMessage()); + LogUtil.error("Failed to export data", e); + exportTask.setExportStatus("FAILED"); + } finally { + exportTaskMapper.updateById(exportTask); + } + }); + Running_Task.put(exportTask.getId(), future); + } + + @DeLog(id = "#p0.exportFrom", ot = LogOT.EXPORT, st = LogST.DATASET) + public void startDatasetTask(CoreExportTask exportTask, DataSetExportRequest request) { + String dataPath = exportData_path + exportTask.getId(); + File directory = new File(dataPath); + boolean isCreated = directory.mkdir(); + + TokenUserBO tokenUserBO = AuthUtils.getUser(); + Future future = scheduledThreadPoolExecutor.submit(() -> { + LicenseUtil.validate(); + AuthUtils.setUser(tokenUserBO); + try { + exportTask.setExportStatus("IN_PROGRESS"); + exportTaskMapper.updateById(exportTask); + CoreDatasetGroup coreDatasetGroup = coreDatasetGroupMapper.selectById(exportTask.getExportFrom()); + if (coreDatasetGroup == null) { + throw new Exception("Not found dataset group: " + exportTask.getExportFrom()); + } + DatasetGroupInfoDTO dto = new DatasetGroupInfoDTO(); + BeanUtils.copyBean(dto, coreDatasetGroup); + dto.setUnionSql(null); + List unionDTOList = JsonUtil.parseList(coreDatasetGroup.getInfo(), new TypeReference<>() { + }); + dto.setUnion(unionDTOList); + List dsFields = datasetTableFieldManage.selectByDatasetGroupId(Long.valueOf(exportTask.getExportFrom())); + List allFields = dsFields.stream().map(ele -> { + DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO(); + BeanUtils.copyBean(datasetTableFieldDTO, ele); + datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName()); + return datasetTableFieldDTO; + }).collect(Collectors.toList()); + + Map sqlMap = datasetSQLManage.getUnionSQLForEdit(dto, null); + String sql = (String) sqlMap.get("sql"); + if (ObjectUtils.isEmpty(allFields)) { + DEException.throwException(Translator.get("i18n_no_fields")); + } + Map desensitizationList = new HashMap<>(); + allFields = permissionManage.filterColumnPermissions(allFields, desensitizationList, dto.getId(), null); + if (ObjectUtils.isEmpty(allFields)) { + DEException.throwException(Translator.get("i18n_no_column_permission")); + } + dto.setAllFields(allFields); + datasetDataManage.buildFieldName(sqlMap, allFields); + Map dsMap = (Map) sqlMap.get("dsMap"); + DatasourceUtils.checkDsStatus(dsMap); + List dsList = new ArrayList<>(); + for (Map.Entry next : dsMap.entrySet()) { + dsList.add(next.getValue().getType()); + } + boolean needOrder = Utils.isNeedOrder(dsList); + boolean crossDs = dto.getIsCross(); + if (!crossDs) { + if (datasetDataManage.notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) { + DEException.throwException(Translator.get("i18n_not_full")); + } + sql = Utils.replaceSchemaAlias(sql, dsMap); + } + List rowPermissionsTree = new ArrayList<>(); + TokenUserBO user = AuthUtils.getUser(); + if (user != null) { + rowPermissionsTree = permissionManage.getRowPermissionsTree(dto.getId(), user.getUserId()); + } + if (StringUtils.isNotEmpty(request.getExpressionTree())) { + DatasetRowPermissionsTreeObj datasetRowPermissionsTreeObj = JsonUtil.parseObject(request.getExpressionTree(), DatasetRowPermissionsTreeObj.class); + permissionManage.getField(datasetRowPermissionsTreeObj); + DataSetRowPermissionsTreeDTO dataSetRowPermissionsTreeDTO = new DataSetRowPermissionsTreeDTO(); + dataSetRowPermissionsTreeDTO.setTree(datasetRowPermissionsTreeObj); + dataSetRowPermissionsTreeDTO.setExportData(true); + rowPermissionsTree.add(dataSetRowPermissionsTreeDTO); + } + + Provider provider; + if (crossDs) { + provider = ProviderFactory.getDefaultProvider(); + } else { + provider = ProviderFactory.getProvider(dsList.getFirst()); + } + SQLMeta sqlMeta = new SQLMeta(); + Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); + Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); + WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); + Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); + String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap); + Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null); + Long curLimit = ExportCenterUtils.getExportLimit("dataset"); + totalCount = totalCount > curLimit ? curLimit : totalCount; + + Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0); + Workbook wb = new SXSSFWorkbook(); + CellStyle cellStyle = wb.createCellStyle(); + Font font = wb.createFont(); + font.setFontHeightInPoints((short) 12); + font.setBold(true); + cellStyle.setFont(font); + cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); + cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + + for (Long s = 1L; s < sheetCount + 1; s++) { + Long sheetSize; + if (s.equals(sheetCount)) { + sheetSize = totalCount - (s - 1) * sheetLimit; + } else { + sheetSize = sheetLimit; + } + Long pageSize = (sheetSize / extractPageSize) + (sheetSize % extractPageSize > 0 ? 1 : 0); + Sheet detailsSheet = null; + List> details = new ArrayList<>(); + for (Long p = 0L; p < pageSize; p++) { + int beforeCount = (int) ((s - 1) * sheetLimit); + String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, beforeCount + p.intValue() * extractPageSize, extractPageSize); + if (pageSize == 1) { + querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, sheetSize.intValue()); + } + querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setQuery(querySQL); + datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(coreDatasetGroup.getIsCross()); + Map previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList, false); + List> data = (List>) previewData.get("data"); + if (p.equals(0L)) { + detailsSheet = wb.createSheet("数据" + s); + List header = new ArrayList<>(); + for (DatasetTableFieldDTO field : allFields) { + header.add(field.getName()); + } + details.add(header); + for (Map obj : data) { + List row = new ArrayList<>(); + for (DatasetTableFieldDTO field : allFields) { + String string = (String) obj.get(field.getDataeaseName()); + row.add(string); + } + details.add(row); + } + if (CollectionUtils.isNotEmpty(details)) { + for (int i = 0; i < details.size(); i++) { + Row row = detailsSheet.createRow(i); + List rowData = details.get(i); + if (rowData != null) { + for (int j = 0; j < rowData.size(); j++) { + Cell cell = row.createCell(j); + if (i == 0) { + cell.setCellValue(rowData.get(j)); + cell.setCellStyle(cellStyle); + detailsSheet.setColumnWidth(j, 255 * 20); + } else { + if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) { + try { + cell.setCellValue(Double.valueOf(rowData.get(j))); + } catch (Exception e) { + cell.setCellValue(rowData.get(j)); + } + } else { + cell.setCellValue(rowData.get(j)); + } + } + } + } + } + } + } else { + details.clear(); + for (Map obj : data) { + List row = new ArrayList<>(); + for (DatasetTableFieldDTO field : allFields) { + String string = (String) obj.get(field.getDataeaseName()); + row.add(string); + } + details.add(row); + } + int lastNum = detailsSheet.getLastRowNum(); + for (int i = 0; i < details.size(); i++) { + Row row = detailsSheet.createRow(i + lastNum + 1); + List rowData = details.get(i); + if (rowData != null) { + for (int j = 0; j < rowData.size(); j++) { + Cell cell = row.createCell(j); + cell.setCellValue(rowData.get(j)); + } + } + } + } + exportTask.setExportStatus("IN_PROGRESS"); + double exportRogress2 = (double) ((double) s - 1) / ((double) sheetCount); + double exportRogress = (double) ((double) (p + 1) / (double) pageSize) * ((double) 1 / sheetCount); + DecimalFormat df = new DecimalFormat("#.##"); + String formattedResult = df.format((exportRogress + exportRogress2) * 100); + exportTask.setExportProgress(formattedResult); + exportTaskMapper.updateById(exportTask); + } + } + this.addWatermarkTools(wb); + FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx"); + wb.write(fileOutputStream); + fileOutputStream.flush(); + fileOutputStream.close(); + wb.close(); + exportTask.setExportProgress("100"); + exportTask.setExportStatus("SUCCESS"); + setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); + + } catch (Exception e) { + LogUtil.error("Failed to export data", e); + exportTask.setMsg(e.getMessage()); + exportTask.setExportStatus("FAILED"); + } finally { + exportTaskMapper.updateById(exportTask); + } + }); + Running_Task.put(exportTask.getId(), future); + } + + @DeLog(id = "#p0.exportFrom", ot = LogOT.EXPORT, st = LogST.VIEW) + public void startViewTask(CoreExportTask exportTask, ChartExcelRequest request) { + String dataPath = exportData_path + exportTask.getId(); + File directory = new File(dataPath); + boolean isCreated = directory.mkdir(); + TokenUserBO tokenUserBO = AuthUtils.getUser(); + Future future = scheduledThreadPoolExecutor.submit(() -> { + LicenseUtil.validate(); + AuthUtils.setUser(tokenUserBO); + try { + exportTask.setExportStatus("IN_PROGRESS"); + exportTaskMapper.updateById(exportTask); + Workbook wb = new SXSSFWorkbook(); + CellStyle cellStyle = wb.createCellStyle(); + Font font = wb.createFont(); + font.setFontHeightInPoints((short) 12); + font.setBold(true); + cellStyle.setFont(font); + cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); + cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + + List details = new ArrayList<>(); + Sheet detailsSheet; + Integer sheetIndex = 1; + if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info") || request.getViewInfo().getType().equalsIgnoreCase("table-normal")) { + request.getViewInfo().getChartExtRequest().setPageSize(Long.valueOf(extractPageSize)); + ChartViewDTO chartViewDTO = chartDataServer.findExcelData(request); + for (long i = 1; i < chartViewDTO.getTotalPage() + 1; i++) { + request.getViewInfo().getChartExtRequest().setGoPage(i); + chartDataServer.findExcelData(request); + details.addAll(request.getDetails()); + if (((details.size() + extractPageSize) > sheetLimit) || i == chartViewDTO.getTotalPage()) { + detailsSheet = wb.createSheet("数据" + sheetIndex); + Integer[] excelTypes = request.getExcelTypes(); + details.add(0, request.getHeader()); + ViewDetailField[] detailFields = request.getDetailFields(); + Object[] header = request.getHeader(); + ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb); + sheetIndex++; + details.clear(); + exportTask.setExportStatus("IN_PROGRESS"); + double exportRogress = (double) (i / (chartViewDTO.getTotalPage() + 1)); + DecimalFormat df = new DecimalFormat("#.##"); + String formattedResult = df.format((exportRogress) * 100); + exportTask.setExportProgress(formattedResult); + exportTaskMapper.updateById(exportTask); + } + } + } else { + downloadNotTableInfoData(request, wb); + } + this.addWatermarkTools(wb); + + try (FileOutputStream outputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx")) { + wb.write(outputStream); + outputStream.flush(); + } + wb.close(); + exportTask.setExportProgress("100"); + exportTask.setExportStatus("SUCCESS"); + setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); + } catch (Exception e) { + exportTask.setMsg(e.getMessage()); + LogUtil.error("Failed to export data", e); + exportTask.setExportStatus("FAILED"); + } finally { + exportTaskMapper.updateById(exportTask); + } + }); + Running_Task.put(exportTask.getId(), future); + } + + private void downloadNotTableInfoData(ChartExcelRequest request, Workbook wb) { + chartDataServer.findExcelData(request); + //给单元格设置样式 + CellStyle cellStyle = wb.createCellStyle(); + Font font = wb.createFont(); + //设置字体大小 + font.setFontHeightInPoints((short) 12); + //设置字体加粗 + font.setBold(true); + //给字体设置样式 + cellStyle.setFont(font); + //设置单元格背景颜色 + cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); + //设置单元格填充样式(使用纯色背景颜色填充) + cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + if (CollectionUtils.isEmpty(request.getMultiInfo())) { + if (request.getViewInfo().getType().equalsIgnoreCase("chart-mix-dual-line")) { + } else { + List details = request.getDetails(); + Integer[] excelTypes = request.getExcelTypes(); + details.add(0, request.getHeader()); + ViewDetailField[] detailFields = request.getDetailFields(); + Object[] header = request.getHeader(); + Sheet detailsSheet = wb.createSheet("数据"); + if (request.getViewInfo().getType().equalsIgnoreCase("table-normal")) { + ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb); + } else { + ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null); + } + } + } else { + //多个sheet + for (int i = 0; i < request.getMultiInfo().size(); i++) { + ChartExcelRequestInner requestInner = request.getMultiInfo().get(i); + + List details = requestInner.getDetails(); + Integer[] excelTypes = requestInner.getExcelTypes(); + details.add(0, requestInner.getHeader()); + ViewDetailField[] detailFields = requestInner.getDetailFields(); + Object[] header = requestInner.getHeader(); + //明细sheet + Sheet detailsSheet = wb.createSheet("数据 " + (i + 1)); + ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null); + } + } + } + + private void setFileSize(String filePath, CoreExportTask exportTask) { + File file = new File(filePath); + long length = file.length(); + String unit = "Mb"; + Double size = 0.0; + if ((double) length / 1024 / 1024 > 1) { + if ((double) length / 1024 / 1024 / 1024 > 1) { + unit = "Gb"; + size = Double.valueOf(String.format("%.2f", (double) length / 1024 / 1024 / 1024)); + } else { + size = Double.valueOf(String.format("%.2f", (double) length / 1024 / 1024)); + } + + } else { + unit = "Kb"; + size = Double.valueOf(String.format("%.2f", (double) length / 1024)); + } + exportTask.setFileSize(size); + exportTask.setFileSizeUnit(unit); + } + + public void addWatermarkTools(Workbook wb) { + VisualizationWatermark watermark = watermarkMapper.selectById("system_default"); + WatermarkContentDTO watermarkContent = JsonUtil.parseObject(watermark.getSettingContent(), WatermarkContentDTO.class); + if (watermarkContent.getEnable() && watermarkContent.getExcelEnable()) { + UserFormVO userInfo = visualizationMapper.queryInnerUserInfo(AuthUtils.getUser().getUserId()); + // 在主逻辑中添加水印 + int watermarkPictureIdx = ExcelWatermarkUtils.addWatermarkImage(wb, watermarkContent, userInfo); // 生成水印图片并获取 ID + for (Sheet sheet : wb) { + ExcelWatermarkUtils.addWatermarkToSheet(sheet, watermarkPictureIdx); // 为每个 Sheet 添加水印 + } + } + } + + public void download(CoreExportTask exportTask, HttpServletResponse response) throws Exception { + response.setContentType("application/octet-stream"); + String encodedFileName = URLEncoder.encode(exportTask.getFileName(), StandardCharsets.UTF_8); + response.setHeader("Content-Disposition", "attachment; filename=\"" + encodedFileName + "\"; filename*=utf-8''" + encodedFileName); + String filePath; + + if (exportTask.getExportTime() < 1730277243491L) { + filePath = exportData_path + exportTask.getId() + "/" + exportTask.getFileName(); + } else { + filePath = exportData_path + exportTask.getId() + "/" + exportTask.getId() + ".xlsx"; + } + + try ( + FileInputStream fileInputStream = new FileInputStream(filePath); + OutputStream outputStream = response.getOutputStream() + ) { + FileChannel fileChannel = fileInputStream.getChannel(); + WritableByteChannel outputChannel = Channels.newChannel(outputStream); + fileChannel.transferTo(0, fileChannel.size(), outputChannel); + response.flushBuffer(); + } catch (IOException e) { + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + } +} + diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java index d87777d790..6f5213f2fa 100644 --- a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java @@ -1,53 +1,23 @@ package io.dataease.exportCenter.manage; - import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import com.fasterxml.jackson.core.type.TypeReference; -import io.dataease.api.chart.dto.ViewDetailField; import io.dataease.api.chart.request.ChartExcelRequest; -import io.dataease.api.chart.request.ChartExcelRequestInner; import io.dataease.api.dataset.dto.DataSetExportRequest; -import io.dataease.api.dataset.union.DatasetGroupInfoDTO; -import io.dataease.api.dataset.union.UnionDTO; import io.dataease.api.export.BaseExportApi; -import io.dataease.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO; import io.dataease.api.xpack.dataFilling.DataFillingApi; -import io.dataease.api.xpack.dataFilling.dto.DataFillFormTableDataRequest; -import io.dataease.auth.bo.TokenUserBO; -import io.dataease.chart.dao.auto.mapper.CoreChartViewMapper; -import io.dataease.chart.server.ChartDataServer; import io.dataease.commons.utils.ExcelWatermarkUtils; -import io.dataease.dataset.dao.auto.entity.CoreDatasetGroup; -import io.dataease.dataset.dao.auto.mapper.CoreDatasetGroupMapper; +import io.dataease.constant.LogOT; +import io.dataease.constant.LogST; import io.dataease.dataset.manage.*; -import io.dataease.datasource.utils.DatasourceUtils; -import io.dataease.constant.DeTypeConstants; -import io.dataease.engine.sql.SQLProvider; -import io.dataease.engine.trans.Field2SQLObj; -import io.dataease.engine.trans.Order2SQLObj; -import io.dataease.engine.trans.Table2SQLObj; -import io.dataease.engine.trans.WhereTree2Str; -import io.dataease.engine.utils.Utils; import io.dataease.exception.DEException; import io.dataease.exportCenter.dao.auto.entity.CoreExportTask; import io.dataease.exportCenter.dao.auto.mapper.CoreExportTaskMapper; import io.dataease.exportCenter.dao.ext.mapper.ExportTaskExtMapper; -import io.dataease.exportCenter.util.ExportCenterUtils; -import io.dataease.extensions.datasource.api.PluginManageApi; -import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO; -import io.dataease.extensions.datasource.dto.DatasourceRequest; -import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO; -import io.dataease.extensions.datasource.factory.ProviderFactory; -import io.dataease.extensions.datasource.model.SQLMeta; -import io.dataease.extensions.datasource.provider.Provider; -import io.dataease.extensions.view.dto.*; -import io.dataease.i18n.Translator; import io.dataease.license.config.XpackInteract; -import io.dataease.license.utils.LicenseUtil; +import io.dataease.log.DeLog; import io.dataease.model.ExportTaskDTO; -import io.dataease.system.manage.CoreUserManage; import io.dataease.system.manage.SysParameterManage; import io.dataease.utils.*; import io.dataease.visualization.dao.auto.entity.VisualizationWatermark; @@ -55,15 +25,11 @@ import io.dataease.visualization.dao.auto.mapper.VisualizationWatermarkMapper; import io.dataease.visualization.dao.ext.mapper.ExtDataVisualizationMapper; import io.dataease.visualization.server.DataVisualizationServer; import io.dataease.websocket.WsMessage; -import io.dataease.websocket.WsService; -import jakarta.annotation.PostConstruct; import jakarta.annotation.Resource; import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ObjectUtils; +import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.apache.poi.ss.usermodel.*; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; @@ -72,16 +38,9 @@ import org.springframework.transaction.annotation.Transactional; import io.dataease.visualization.dto.WatermarkContentDTO; import io.dataease.api.permissions.user.vo.UserFormVO; -import java.io.*; import java.net.InetAddress; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.text.DecimalFormat; import java.util.*; import java.util.concurrent.Future; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; @Component @Transactional(rollbackFor = Exception.class) @@ -95,13 +54,7 @@ public class ExportCenterManage implements BaseExportApi { @Resource DataVisualizationServer dataVisualizationServer; @Resource - private CoreChartViewMapper coreChartViewMapper; - @Resource - private PermissionManage permissionManage; - @Autowired - private WsService wsService; - @Autowired(required = false) - private PluginManageApi pluginManage; + private ExportCenterDownLoadManage exportCenterDownLoadManage; @Resource private SysParameterManage sysParameterManage; @Value("${dataease.export.core.size:10}") @@ -109,38 +62,14 @@ public class ExportCenterManage implements BaseExportApi { @Value("${dataease.export.max.size:10}") private int max; - - private final static String DATA_URL_TITLE = "data:image/jpeg;base64,"; @Value("${dataease.path.exportData:/opt/dataease2.0/data/exportData/}") private String exportData_path; @Resource private VisualizationWatermarkMapper watermarkMapper; @Resource private ExtDataVisualizationMapper visualizationMapper; - - public Integer getExtractPageSize() { - return extractPageSize; - } - - @Value("${dataease.export.page.size:50000}") - private Integer extractPageSize; static private List STATUS = Arrays.asList("SUCCESS", "FAILED", "PENDING", "IN_PROGRESS", "ALL"); - private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; - private int keepAliveSeconds = 600; private Map Running_Task = new HashMap<>(); - @Resource - private ChartDataServer chartDataServer; - @Resource - private CoreDatasetGroupMapper coreDatasetGroupMapper; - @Resource - private CoreUserManage coreUserManage; - @Resource - private DatasetSQLManage datasetSQLManage; - @Resource - private DatasetTableFieldManage datasetTableFieldManage; - @Resource - private DatasetDataManage datasetDataManage; - private final Long sheetLimit = 1000000L; @Autowired(required = false) private DataFillingApi dataFillingApi = null; @@ -149,56 +78,12 @@ public class ExportCenterManage implements BaseExportApi { } - @PostConstruct - public void init() { - scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(core); - scheduledThreadPoolExecutor.setKeepAliveTime(keepAliveSeconds, TimeUnit.SECONDS); - scheduledThreadPoolExecutor.setMaximumPoolSize(max); - } - - @Scheduled(fixedRate = 5000) - public void checkRunningTask() { - Iterator> iterator = Running_Task.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - if (entry.getValue().isDone()) { - iterator.remove(); - try { - CoreExportTask exportTask = exportTaskMapper.selectById(entry.getKey()); - ExportTaskDTO exportTaskDTO = new ExportTaskDTO(); - BeanUtils.copyBean(exportTaskDTO, exportTask); - setExportFromName(exportTaskDTO); - WsMessage message = new WsMessage(exportTask.getUserId(), "/task-export-topic", exportTaskDTO); - wsService.releaseMessage(message); - } catch (Exception e) { - - } - } - } - } - - public void download(String id, HttpServletResponse response) throws Exception { + if (!downLoadInfos.stream().anyMatch(downLoadInfo -> downLoadInfo.getId().equals(id))) { + DEException.throwException("任务不存在"); + } CoreExportTask exportTask = exportTaskMapper.selectById(id); - OutputStream outputStream = response.getOutputStream(); - response.setContentType("application/vnd.ms-excel"); - - response.setHeader("Content-disposition", "attachment;filename=" + URLEncoder.encode(exportTask.getFileName(), StandardCharsets.UTF_8)); - InputStream fileInputStream; - if (exportTask.getExportTime() < 1730277243491L) { - fileInputStream = new FileInputStream(exportData_path + id + "/" + exportTask.getFileName()); - } else { - fileInputStream = new FileInputStream(exportData_path + id + "/" + id + ".xlsx"); - } - byte[] buffer = new byte[4096]; - int bytesRead; - while ((bytesRead = fileInputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - outputStream.flush(); - outputStream.close(); - fileInputStream.close(); - response.flushBuffer(); + exportCenterDownLoadManage.download(exportTask, response); } public void delete(String id) { @@ -256,15 +141,15 @@ public class ExportCenterManage implements BaseExportApi { FileUtils.deleteDirectoryRecursively(exportData_path + id); if (exportTask.getExportFromType().equalsIgnoreCase("chart")) { ChartExcelRequest request = JsonUtil.parseObject(exportTask.getParams(), ChartExcelRequest.class); - startViewTask(exportTask, request); + exportCenterDownLoadManage.startViewTask(exportTask, request); } if (exportTask.getExportFromType().equalsIgnoreCase("dataset")) { DataSetExportRequest request = JsonUtil.parseObject(exportTask.getParams(), DataSetExportRequest.class); - startDatasetTask(exportTask, request); + exportCenterDownLoadManage.startDatasetTask(exportTask, request); } if (exportTask.getExportFromType().equalsIgnoreCase("data_filling")) { HashMap request = JsonUtil.parseObject(exportTask.getParams(), HashMap.class); - startDataFillingTask(exportTask, request); + exportCenterDownLoadManage.startDataFillingTask(exportTask, request); } } @@ -294,7 +179,6 @@ public class ExportCenterManage implements BaseExportApi { return pager; } - public Map exportTasks() { Map result = new HashMap<>(); QueryWrapper queryWrapper = new QueryWrapper<>(); @@ -351,18 +235,6 @@ public class ExportCenterManage implements BaseExportApi { } } - private void setExportFromName(ExportTaskDTO exportTaskDTO) { - if (exportTaskDTO.getExportFromType().equalsIgnoreCase("chart")) { - exportTaskDTO.setExportFromName(coreChartViewMapper.selectById(exportTaskDTO.getExportFrom()).getTitle()); - } - if (exportTaskDTO.getExportFromType().equalsIgnoreCase("dataset")) { - exportTaskDTO.setExportFromName(coreDatasetGroupMapper.selectById(exportTaskDTO.getExportFrom()).getName()); - } - if (exportTaskDTO.getExportFromType().equalsIgnoreCase("data_filling")) { - exportTaskDTO.setExportFromName(getDataFillingApi().get(Long.parseLong(exportTaskDTO.getExportFrom())).getName()); - } - } - private String hostName() { String hostname = null; try { @@ -376,9 +248,9 @@ public class ExportCenterManage implements BaseExportApi { public void addTask(String exportFrom, String exportFromType, ChartExcelRequest request) { CoreExportTask exportTask = new CoreExportTask(); - exportTask.setId(UUID.randomUUID().toString()); + exportTask.setId(IDUtils.snowID().toString()); exportTask.setUserId(AuthUtils.getUser().getUserId()); - exportTask.setExportFrom(exportFrom); + exportTask.setExportFrom(Long.valueOf(exportFrom)); exportTask.setExportFromType(exportFromType); exportTask.setExportStatus("PENDING"); exportTask.setFileName(request.getViewName() + ".xlsx"); @@ -387,15 +259,15 @@ public class ExportCenterManage implements BaseExportApi { exportTask.setParams(JsonUtil.toJSONString(request).toString()); exportTask.setExportMachineName(hostName()); exportTaskMapper.insert(exportTask); - startViewTask(exportTask, request); + exportCenterDownLoadManage.startViewTask(exportTask, request); } public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request) throws Exception { datasetGroupManage.getDatasetGroupInfoDTO(exportFrom, null); CoreExportTask exportTask = new CoreExportTask(); - exportTask.setId(UUID.randomUUID().toString()); + exportTask.setId(IDUtils.snowID().toString()); exportTask.setUserId(AuthUtils.getUser().getUserId()); - exportTask.setExportFrom(String.valueOf(exportFrom)); + exportTask.setExportFrom(exportFrom); exportTask.setExportFromType(exportFromType); exportTask.setExportStatus("PENDING"); exportTask.setFileName(request.getFilename() + ".xlsx"); @@ -404,16 +276,16 @@ public class ExportCenterManage implements BaseExportApi { exportTask.setParams(JsonUtil.toJSONString(request).toString()); exportTask.setExportMachineName(hostName()); exportTaskMapper.insert(exportTask); - startDatasetTask(exportTask, request); + exportCenterDownLoadManage.startDatasetTask(exportTask, request); } @Override public void addTask(String exportFromId, String exportFromType, HashMap request, Long userId, Long org) { CoreExportTask exportTask = new CoreExportTask(); request.put("org", org); - exportTask.setId(UUID.randomUUID().toString()); + exportTask.setId(IDUtils.snowID().toString()); exportTask.setUserId(userId); - exportTask.setExportFrom(exportFromId); + exportTask.setExportFrom(Long.valueOf(exportFromId)); exportTask.setExportFromType(exportFromType); exportTask.setExportStatus("PENDING"); exportTask.setFileName(request.get("name") + ".xlsx"); @@ -423,400 +295,10 @@ public class ExportCenterManage implements BaseExportApi { exportTask.setExportMachineName(hostName()); exportTaskMapper.insert(exportTask); if (StringUtils.equals(exportFromType, "data_filling")) { - startDataFillingTask(exportTask, request); + exportCenterDownLoadManage.startDataFillingTask(exportTask, request); } } - private void startDataFillingTask(CoreExportTask exportTask, HashMap request) { - - if (ObjectUtils.isEmpty(getDataFillingApi())) { - return; - } - - String dataPath = exportData_path + exportTask.getId(); - File directory = new File(dataPath); - boolean isCreated = directory.mkdir(); - TokenUserBO tokenUserBO = AuthUtils.getUser(); - Future future = scheduledThreadPoolExecutor.submit(() -> { - AuthUtils.setUser(tokenUserBO); - try { - exportTask.setExportStatus("IN_PROGRESS"); - exportTaskMapper.updateById(exportTask); - - getDataFillingApi().writeExcel(dataPath + "/" + exportTask.getId() + ".xlsx", new DataFillFormTableDataRequest().setId(Long.parseLong(exportTask.getExportFrom())).setWithoutLogs(true), exportTask.getUserId(), Long.parseLong(request.get("org").toString())); - - - exportTask.setExportProgress("100"); - exportTask.setExportStatus("SUCCESS"); - - setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); - } catch (Exception e) { - exportTask.setMsg(e.getMessage()); - LogUtil.error("Failed to export data", e); - exportTask.setExportStatus("FAILED"); - } finally { - exportTaskMapper.updateById(exportTask); - } - }); - Running_Task.put(exportTask.getId(), future); - } - - - private void startDatasetTask(CoreExportTask exportTask, DataSetExportRequest request) { - String dataPath = exportData_path + exportTask.getId(); - File directory = new File(dataPath); - boolean isCreated = directory.mkdir(); - - TokenUserBO tokenUserBO = AuthUtils.getUser(); - Future future = scheduledThreadPoolExecutor.submit(() -> { - LicenseUtil.validate(); - AuthUtils.setUser(tokenUserBO); - try { - exportTask.setExportStatus("IN_PROGRESS"); - exportTaskMapper.updateById(exportTask); - CoreDatasetGroup coreDatasetGroup = coreDatasetGroupMapper.selectById(exportTask.getExportFrom()); - if (coreDatasetGroup == null) { - throw new Exception("Not found dataset group: " + exportTask.getExportFrom()); - } - DatasetGroupInfoDTO dto = new DatasetGroupInfoDTO(); - BeanUtils.copyBean(dto, coreDatasetGroup); - dto.setUnionSql(null); - List unionDTOList = JsonUtil.parseList(coreDatasetGroup.getInfo(), new TypeReference<>() { - }); - dto.setUnion(unionDTOList); - List dsFields = datasetTableFieldManage.selectByDatasetGroupId(Long.valueOf(exportTask.getExportFrom())); - List allFields = dsFields.stream().map(ele -> { - DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO(); - BeanUtils.copyBean(datasetTableFieldDTO, ele); - datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName()); - return datasetTableFieldDTO; - }).collect(Collectors.toList()); - - Map sqlMap = datasetSQLManage.getUnionSQLForEdit(dto, null); - String sql = (String) sqlMap.get("sql"); - if (ObjectUtils.isEmpty(allFields)) { - DEException.throwException(Translator.get("i18n_no_fields")); - } - Map desensitizationList = new HashMap<>(); - allFields = permissionManage.filterColumnPermissions(allFields, desensitizationList, dto.getId(), null); - if (ObjectUtils.isEmpty(allFields)) { - DEException.throwException(Translator.get("i18n_no_column_permission")); - } - dto.setAllFields(allFields); - datasetDataManage.buildFieldName(sqlMap, allFields); - Map dsMap = (Map) sqlMap.get("dsMap"); - DatasourceUtils.checkDsStatus(dsMap); - List dsList = new ArrayList<>(); - for (Map.Entry next : dsMap.entrySet()) { - dsList.add(next.getValue().getType()); - } - boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = dto.getIsCross(); - if (!crossDs) { - if (datasetDataManage.notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) { - DEException.throwException(Translator.get("i18n_not_full")); - } - sql = Utils.replaceSchemaAlias(sql, dsMap); - } - List rowPermissionsTree = new ArrayList<>(); - TokenUserBO user = AuthUtils.getUser(); - if (user != null) { - rowPermissionsTree = permissionManage.getRowPermissionsTree(dto.getId(), user.getUserId()); - } - if (StringUtils.isNotEmpty(request.getExpressionTree())) { - DatasetRowPermissionsTreeObj datasetRowPermissionsTreeObj = JsonUtil.parseObject(request.getExpressionTree(), DatasetRowPermissionsTreeObj.class); - permissionManage.getField(datasetRowPermissionsTreeObj); - DataSetRowPermissionsTreeDTO dataSetRowPermissionsTreeDTO = new DataSetRowPermissionsTreeDTO(); - dataSetRowPermissionsTreeDTO.setTree(datasetRowPermissionsTreeObj); - dataSetRowPermissionsTreeDTO.setExportData(true); - rowPermissionsTree.add(dataSetRowPermissionsTreeDTO); - } - - Provider provider; - if (crossDs) { - provider = ProviderFactory.getDefaultProvider(); - } else { - provider = ProviderFactory.getProvider(dsList.getFirst()); - } - SQLMeta sqlMeta = new SQLMeta(); - Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); - Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); - WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); - Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); - String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap); - Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null); - Long curLimit = ExportCenterUtils.getExportLimit("dataset"); - totalCount = totalCount > curLimit ? curLimit : totalCount; - - Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0); - Workbook wb = new SXSSFWorkbook(); - CellStyle cellStyle = wb.createCellStyle(); - Font font = wb.createFont(); - font.setFontHeightInPoints((short) 12); - font.setBold(true); - cellStyle.setFont(font); - cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - - for (Long s = 1L; s < sheetCount + 1; s++) { - Long sheetSize; - if (s.equals(sheetCount)) { - sheetSize = totalCount - (s - 1) * sheetLimit; - } else { - sheetSize = sheetLimit; - } - Long pageSize = (sheetSize / extractPageSize) + (sheetSize % extractPageSize > 0 ? 1 : 0); - Sheet detailsSheet = null; - List> details = new ArrayList<>(); - for (Long p = 0L; p < pageSize; p++) { - int beforeCount = (int) ((s - 1) * sheetLimit); - String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, beforeCount + p.intValue() * extractPageSize, extractPageSize); - if (pageSize == 1) { - querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, sheetSize.intValue()); - } - querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); - DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setQuery(querySQL); - datasourceRequest.setDsList(dsMap); - datasourceRequest.setIsCross(coreDatasetGroup.getIsCross()); - Map previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList, false); - List> data = (List>) previewData.get("data"); - if (p.equals(0L)) { - detailsSheet = wb.createSheet("数据" + s); - List header = new ArrayList<>(); - for (DatasetTableFieldDTO field : allFields) { - header.add(field.getName()); - } - details.add(header); - for (Map obj : data) { - List row = new ArrayList<>(); - for (DatasetTableFieldDTO field : allFields) { - String string = (String) obj.get(field.getDataeaseName()); - row.add(string); - } - details.add(row); - } - if (CollectionUtils.isNotEmpty(details)) { - for (int i = 0; i < details.size(); i++) { - Row row = detailsSheet.createRow(i); - List rowData = details.get(i); - if (rowData != null) { - for (int j = 0; j < rowData.size(); j++) { - Cell cell = row.createCell(j); - if (i == 0) { - cell.setCellValue(rowData.get(j)); - cell.setCellStyle(cellStyle); - detailsSheet.setColumnWidth(j, 255 * 20); - } else { - if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) { - try { - cell.setCellValue(Double.valueOf(rowData.get(j))); - } catch (Exception e) { - cell.setCellValue(rowData.get(j)); - } - } else { - cell.setCellValue(rowData.get(j)); - } - } - } - } - } - } - } else { - details.clear(); - for (Map obj : data) { - List row = new ArrayList<>(); - for (DatasetTableFieldDTO field : allFields) { - String string = (String) obj.get(field.getDataeaseName()); - row.add(string); - } - details.add(row); - } - int lastNum = detailsSheet.getLastRowNum(); - for (int i = 0; i < details.size(); i++) { - Row row = detailsSheet.createRow(i + lastNum + 1); - List rowData = details.get(i); - if (rowData != null) { - for (int j = 0; j < rowData.size(); j++) { - Cell cell = row.createCell(j); - cell.setCellValue(rowData.get(j)); - } - } - } - } - exportTask.setExportStatus("IN_PROGRESS"); - double exportRogress2 = (double) ((double) s - 1) / ((double) sheetCount); - double exportRogress = (double) ((double) (p + 1) / (double) pageSize) * ((double) 1 / sheetCount); - DecimalFormat df = new DecimalFormat("#.##"); - String formattedResult = df.format((exportRogress + exportRogress2) * 100); - exportTask.setExportProgress(formattedResult); - exportTaskMapper.updateById(exportTask); - } - } - this.addWatermarkTools(wb); - FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx"); - wb.write(fileOutputStream); - fileOutputStream.flush(); - fileOutputStream.close(); - wb.close(); - exportTask.setExportProgress("100"); - exportTask.setExportStatus("SUCCESS"); - setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); - - } catch (Exception e) { - LogUtil.error("Failed to export data", e); - exportTask.setMsg(e.getMessage()); - exportTask.setExportStatus("FAILED"); - } finally { - exportTaskMapper.updateById(exportTask); - } - }); - Running_Task.put(exportTask.getId(), future); - } - - - private void startViewTask(CoreExportTask exportTask, ChartExcelRequest request) { - String dataPath = exportData_path + exportTask.getId(); - File directory = new File(dataPath); - boolean isCreated = directory.mkdir(); - TokenUserBO tokenUserBO = AuthUtils.getUser(); - Future future = scheduledThreadPoolExecutor.submit(() -> { - LicenseUtil.validate(); - AuthUtils.setUser(tokenUserBO); - try { - exportTask.setExportStatus("IN_PROGRESS"); - exportTaskMapper.updateById(exportTask); - Workbook wb = new SXSSFWorkbook(); - CellStyle cellStyle = wb.createCellStyle(); - Font font = wb.createFont(); - font.setFontHeightInPoints((short) 12); - font.setBold(true); - cellStyle.setFont(font); - cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - - List details = new ArrayList<>(); - Sheet detailsSheet; - Integer sheetIndex = 1; - if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info") || request.getViewInfo().getType().equalsIgnoreCase("table-normal")) { - request.getViewInfo().getChartExtRequest().setPageSize(Long.valueOf(extractPageSize)); - ChartViewDTO chartViewDTO = chartDataServer.findExcelData(request); - for (long i = 1; i < chartViewDTO.getTotalPage() + 1; i++) { - request.getViewInfo().getChartExtRequest().setGoPage(i); - chartDataServer.findExcelData(request); - details.addAll(request.getDetails()); - if (((details.size() + extractPageSize) > sheetLimit) || i == chartViewDTO.getTotalPage()) { - detailsSheet = wb.createSheet("数据" + sheetIndex); - Integer[] excelTypes = request.getExcelTypes(); - details.add(0, request.getHeader()); - ViewDetailField[] detailFields = request.getDetailFields(); - Object[] header = request.getHeader(); - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb); - sheetIndex++; - details.clear(); - exportTask.setExportStatus("IN_PROGRESS"); - double exportRogress = (double) (i / (chartViewDTO.getTotalPage() + 1)); - DecimalFormat df = new DecimalFormat("#.##"); - String formattedResult = df.format((exportRogress) * 100); - exportTask.setExportProgress(formattedResult); - exportTaskMapper.updateById(exportTask); - } - } - } else { - downloadNotTableInfoData(request, wb); - } - this.addWatermarkTools(wb); - - try (FileOutputStream outputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx")) { - wb.write(outputStream); - outputStream.flush(); - } - wb.close(); - exportTask.setExportProgress("100"); - exportTask.setExportStatus("SUCCESS"); - setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); - } catch (Exception e) { - exportTask.setMsg(e.getMessage()); - LogUtil.error("Failed to export data", e); - exportTask.setExportStatus("FAILED"); - } finally { - exportTaskMapper.updateById(exportTask); - } - }); - Running_Task.put(exportTask.getId(), future); - } - - private void downloadNotTableInfoData(ChartExcelRequest request, Workbook wb) { - chartDataServer.findExcelData(request); - //给单元格设置样式 - CellStyle cellStyle = wb.createCellStyle(); - Font font = wb.createFont(); - //设置字体大小 - font.setFontHeightInPoints((short) 12); - //设置字体加粗 - font.setBold(true); - //给字体设置样式 - cellStyle.setFont(font); - //设置单元格背景颜色 - cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - //设置单元格填充样式(使用纯色背景颜色填充) - cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - if (CollectionUtils.isEmpty(request.getMultiInfo())) { - if (request.getViewInfo().getType().equalsIgnoreCase("chart-mix-dual-line")) { - } else { - List details = request.getDetails(); - Integer[] excelTypes = request.getExcelTypes(); - details.add(0, request.getHeader()); - ViewDetailField[] detailFields = request.getDetailFields(); - Object[] header = request.getHeader(); - Sheet detailsSheet = wb.createSheet("数据"); - if (request.getViewInfo().getType().equalsIgnoreCase("table-normal")) { - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb); - } else { - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null); - } - } - } else { - //多个sheet - for (int i = 0; i < request.getMultiInfo().size(); i++) { - ChartExcelRequestInner requestInner = request.getMultiInfo().get(i); - - List details = requestInner.getDetails(); - Integer[] excelTypes = requestInner.getExcelTypes(); - details.add(0, requestInner.getHeader()); - ViewDetailField[] detailFields = requestInner.getDetailFields(); - Object[] header = requestInner.getHeader(); - //明细sheet - Sheet detailsSheet = wb.createSheet("数据 " + (i + 1)); - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null); - } - } - } - - private void setFileSize(String filePath, CoreExportTask exportTask) { - File file = new File(filePath); - long length = file.length(); - String unit = "Mb"; - Double size = 0.0; - if ((double) length / 1024 / 1024 > 1) { - if ((double) length / 1024 / 1024 / 1024 > 1) { - unit = "Gb"; - size = Double.valueOf(String.format("%.2f", (double) length / 1024 / 1024 / 1024)); - } else { - size = Double.valueOf(String.format("%.2f", (double) length / 1024 / 1024)); - } - - } else { - unit = "Kb"; - size = Double.valueOf(String.format("%.2f", (double) length / 1024)); - } - exportTask.setFileSize(size); - exportTask.setFileSizeUnit(unit); - } - - - private static final String LOG_RETENTION = "30"; - public void cleanLog() { String key = "basic.exportFileLiveTime"; String val = sysParameterManage.singleVal(key); @@ -845,5 +327,29 @@ public class ExportCenterManage implements BaseExportApi { } } } + + @DeLog(id = "#p0", ot = LogOT.DOWNLOAD, st = LogST.DATA) + public void generateDownloadUri(String id) { + if (!downLoadInfos.stream().anyMatch(downLoadInfo -> downLoadInfo.getId().equals(id))) { + DownLoadInfo downLoadInfo = new DownLoadInfo(); + downLoadInfo.setId(id); + downLoadInfo.setCreateTime(System.currentTimeMillis()); + downLoadInfo.setValidTime(5L); + downLoadInfos.add(downLoadInfo); + } + } + private List downLoadInfos = new ArrayList<>(); + + @Scheduled(fixedRate = 10 * 1000) + public void checkDownLoadInfos() { + downLoadInfos.removeIf(downLoadInfo -> System.currentTimeMillis() - downLoadInfo.createTime > downLoadInfo.validTime * 60 * 1000); + } + + @Data + public class DownLoadInfo { + String id; + Long validTime; // 单位：minutes + Long createTime; + } } diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java b/core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java index 6311d765e5..f191dc8923 100644 --- a/core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/server/ExportCenterServer.java @@ -53,6 +53,12 @@ public class ExportCenterServer implements ExportCenterApi { exportCenterManage.download(id, response); } + @Override + public String generateDownloadUri(String id) throws Exception { + exportCenterManage.generateDownloadUri(id); + return ""; + } + @Override public void retry(String id) { exportCenterManage.retry(id); diff --git a/core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java b/core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java index bdf78e46fd..cfa777ba65 100644 --- a/core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java +++ b/core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java @@ -981,10 +981,10 @@ public class DataVisualizationServer implements DataVisualizationApi { if (viewDTO == null) { return null; } - if (viewDTO.getSceneId() == null) { + if (viewDTO.getPid() == null) { return viewDTO.getTitle(); } - List parents = getParents(viewDTO.getSceneId()); + List parents = getParents(viewDTO.getPid()); StringBuilder stringBuilder = new StringBuilder(); parents.forEach(ele -> { if (ObjectUtils.isNotEmpty(ele)) { diff --git a/core/core-frontend/src/api/dataset.ts b/core/core-frontend/src/api/dataset.ts index 8f9d11fd3b..8f9238421e 100644 --- a/core/core-frontend/src/api/dataset.ts +++ b/core/core-frontend/src/api/dataset.ts @@ -362,6 +362,12 @@ export const exportDelete = async (id): Promise => { }) } +export const generateDownloadUri = async (id): Promise => { + return request.get({ url: '/exportCenter/generateDownloadUri/' + id }).then(res => { + return res?.data + }) +} + export const exportDeleteAll = async (type, data): Promise => { return request.post({ url: '/exportCenter/deleteAll/' + type, data }).then(res => { return res?.data diff --git a/core/core-frontend/src/views/visualized/data/dataset/ExportExcel.vue b/core/core-frontend/src/views/visualized/data/dataset/ExportExcel.vue index e5dc89507d..8604ffbd26 100644 --- a/core/core-frontend/src/views/visualized/data/dataset/ExportExcel.vue +++ b/core/core-frontend/src/views/visualized/data/dataset/ExportExcel.vue @@ -13,7 +13,8 @@ import { exportDelete, exportDeleteAll, exportDeletePost, - exportTasksRecords + exportTasksRecords, + generateDownloadUri } from '@/api/dataset' import { useI18n } from '@/hooks/web/useI18n' import { useEmitt } from '@/hooks/web/useEmitt' @@ -220,12 +221,16 @@ const callbackExportSuc = () => { const downLoadAll = () => { if (multipleSelection.value.length === 0) { tableData.value.forEach(item => { - window.open(PATH_URL + '/exportCenter/download/' + item.id) + generateDownloadUri(item.id).then(res => { + window.open(PATH_URL + '/exportCenter/download/' + item.id) + }) }) return } multipleSelection.value.map(ele => { - window.open(PATH_URL + '/exportCenter/download/' + ele.id) + generateDownloadUri(ele.id).then(res => { + window.open(PATH_URL + '/exportCenter/download/' + ele.id) + }) }) } const showMsg = item => { @@ -242,7 +247,9 @@ const timestampFormatDate = value => { import { PATH_URL } from '@/config/axios/service' import GridTable from '../../../../components/grid-table/src/GridTable.vue' const downloadClick = item => { - window.open(PATH_URL + '/exportCenter/download/' + item.id, openType) + generateDownloadUri(item.id).then(res => { + window.open(PATH_URL + '/exportCenter/download/' + item.id, openType) + }) } const retry = item => { diff --git a/sdk/api/api-base/src/main/java/io/dataease/api/exportCenter/ExportCenterApi.java b/sdk/api/api-base/src/main/java/io/dataease/api/exportCenter/ExportCenterApi.java index e4349c90d4..783ae066bf 100644 --- a/sdk/api/api-base/src/main/java/io/dataease/api/exportCenter/ExportCenterApi.java +++ b/sdk/api/api-base/src/main/java/io/dataease/api/exportCenter/ExportCenterApi.java @@ -47,6 +47,10 @@ public interface ExportCenterApi { @GetMapping("/download/{id}") public void download(@PathVariable String id, HttpServletResponse response) throws Exception; + @Operation(summary = "生成下载Url") + @GetMapping("/generateDownloadUri/{id}") + public String generateDownloadUri(@PathVariable String id) throws Exception; + @Operation(summary = "重试") @PostMapping("/retry/{id}") public void retry(@PathVariable String id); diff --git a/sdk/common/src/main/java/io/dataease/constant/LogOT.java b/sdk/common/src/main/java/io/dataease/constant/LogOT.java index 512da3434c..a9657edc7b 100644 --- a/sdk/common/src/main/java/io/dataease/constant/LogOT.java +++ b/sdk/common/src/main/java/io/dataease/constant/LogOT.java @@ -16,7 +16,8 @@ public enum LogOT { UPLOADFILE(11, "OPERATE_TYPE_UPLOADFILE"), BIND(12, "OPERATE_TYPE_BIND"), UNBIND(13, "OPERATE_TYPE_UNBIND"), - LOGIN(14, "OPERATE_TYPE_LOGIN"); + LOGIN(14, "OPERATE_TYPE_LOGIN"), + DOWNLOAD(15, "OPERATE_TYPE_DOWNLOAD"); private Integer value; private String name; diff --git a/sdk/common/src/main/java/io/dataease/constant/LogST.java b/sdk/common/src/main/java/io/dataease/constant/LogST.java index ba7126f988..f51671a65c 100644 --- a/sdk/common/src/main/java/io/dataease/constant/LogST.java +++ b/sdk/common/src/main/java/io/dataease/constant/LogST.java @@ -16,7 +16,8 @@ public enum LogST { DRIVER_FILE(11, "SOURCE_TYPE_DRIVER_FILE"), MENU(12, "SOURCE_TYPE_MENU"), APIKEY(13, "SOURCE_TYPE_APIKEY"), - DATA_FILLING(14, "SOURCE_TYPE_DATAFILLING"); + DATA_FILLING(14, "SOURCE_TYPE_DATAFILLING"), + DATA(15, "SOURCE_TYPE_DATA"); private Integer value; private String name;