From d38f20fd55d8e280dd43c8b4af524712bb7ca3c6 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Thu, 28 Nov 2024 18:25:34 +0800 Subject: [PATCH] =?UTF-8?q?=20fix:=20=E4=B8=8B=E8=BD=BD=E6=98=8E=E7=BB=86?= =?UTF-8?q?=E8=A1=A8=E5=AE=9E=E7=8E=B0=E5=88=86=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../charts/impl/table/TableInfoHandler.java | 3 +- .../chart/server/ChartDataServer.java | 88 +++++++----- .../manage/ExportCenterManage.java | 134 ++++++++++++------ 3 files changed, 140 insertions(+), 85 deletions(-) diff --git a/core/core-backend/src/main/java/io/dataease/chart/charts/impl/table/TableInfoHandler.java b/core/core-backend/src/main/java/io/dataease/chart/charts/impl/table/TableInfoHandler.java index 0d87d54348..d94b6133ce 100644 --- a/core/core-backend/src/main/java/io/dataease/chart/charts/impl/table/TableInfoHandler.java +++ b/core/core-backend/src/main/java/io/dataease/chart/charts/impl/table/TableInfoHandler.java @@ -44,7 +44,7 @@ public class TableInfoHandler extends DefaultChartHandler { Map mapSize = (Map) mapAttr.get("basicStyle"); var tablePageMode = (String) mapSize.get("tablePageMode"); formatResult.getContext().put("tablePageMode", tablePageMode); - if (StringUtils.equalsIgnoreCase(tablePageMode, "page") && !view.getIsExcelExport()) { + if (StringUtils.equalsIgnoreCase(tablePageMode, "page")) { if (chartExtRequest.getGoPage() == null) { chartExtRequest.setGoPage(1L); } @@ -127,6 +127,7 @@ public class TableInfoHandler extends DefaultChartHandler { querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap); datasourceRequest.setQuery(querySql); logger.debug("calcite chart sql: " + querySql); + System.out.println(querySql); List data = (List) provider.fetchResultField(datasourceRequest).get("data"); //自定义排序 data = ChartDataUtil.resultCustomSort(xAxis, data); diff --git a/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java b/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java index 758cdd18dc..0b69c37147 100644 --- a/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java +++ b/core/core-backend/src/main/java/io/dataease/chart/server/ChartDataServer.java @@ -17,7 +17,6 @@ import io.dataease.exception.DEException; import io.dataease.exportCenter.manage.ExportCenterManage; import io.dataease.extensions.datasource.dto.DatasetTableFieldDTO; import io.dataease.extensions.view.dto.ChartViewDTO; -import io.dataease.extensions.view.dto.ChartViewFieldBaseDTO; import io.dataease.extensions.view.dto.ChartViewFieldDTO; import io.dataease.extensions.view.dto.FormatterCfgDTO; import io.dataease.license.manage.F2CLicLimitedManage; @@ -74,6 +73,9 @@ public class ChartDataServer implements ChartDataApi { @Resource(name = "f2CLicLimitedManage") private F2CLicLimitedManage f2CLicLimitedManage; + @Value("${dataease.export.page.size:50000}") + private Integer extractPageSize; + private final Long sheetLimit = 1000000L; private Integer getExportLimit() { return Math.toIntExact(Math.min(f2CLicLimitedManage.checkDatasetLimit(), limit)); @@ -95,7 +97,8 @@ public class ChartDataServer implements ChartDataApi { return null; } - public void findExcelData(ChartExcelRequest request) { + public ChartViewDTO findExcelData(ChartExcelRequest request) { + ChartViewDTO chartViewInfo = new ChartViewDTO(); try { ChartViewDTO viewDTO = request.getViewInfo(); viewDTO.setIsExcelExport(true); @@ -107,12 +110,8 @@ public class ChartDataServer implements ChartDataApi { List sourceFields = datasetFieldServer.listByDatasetGroup(viewDTO.getTableId()); List fileNames = permissionManage.filterColumnPermissions(sourceFields, new HashMap<>(), viewDTO.getTableId(), null).stream().map(DatasetTableFieldDTO::getDataeaseName).collect(Collectors.toList()); sourceFields = sourceFields.stream().filter(datasetTableFieldDTO -> fileNames.contains(datasetTableFieldDTO.getDataeaseName())).collect(Collectors.toList()); - dsHeader = sourceFields.stream() - .map(DatasetTableFieldDTO::getName) - .toArray(String[]::new); - dsTypes = sourceFields.stream() - .map(DatasetTableFieldDTO::getDeType) - .toArray(Integer[]::new); + dsHeader = sourceFields.stream().map(DatasetTableFieldDTO::getName).toArray(String[]::new); + dsTypes = sourceFields.stream().map(DatasetTableFieldDTO::getDeType).toArray(Integer[]::new); TypeReference> listTypeReference = new TypeReference>() { }; viewDTO.setXAxis(JsonUtil.parseList(JsonUtil.toJSONString(sourceFields).toString(), listTypeReference)); @@ -124,7 +123,7 @@ public class ChartDataServer implements ChartDataApi { } else { viewDTO.setResultCount(curLimit); } - ChartViewDTO chartViewInfo = getData(viewDTO); + chartViewInfo = getData(viewDTO); List tableRow = (List) chartViewInfo.getData().get("sourceData"); if ("dataset".equals(request.getDownloadType())) { request.setHeader(dsHeader); @@ -149,9 +148,10 @@ public class ChartDataServer implements ChartDataApi { } catch (Exception e) { throw new RuntimeException(e); } - + return chartViewInfo; } + public static String valueFormatter(BigDecimal value, FormatterCfgDTO formatter) { if (value == null) { return null; @@ -240,9 +240,7 @@ public class ChartDataServer implements ChartDataApi { if ((StringUtils.isNotEmpty(linkToken) && !request.isDataEaseBi()) || (request.isDataEaseBi() && StringUtils.isEmpty(linkToken))) { OutputStream outputStream = response.getOutputStream(); try { - findExcelData(request); Workbook wb = new SXSSFWorkbook(); - //给单元格设置样式 CellStyle cellStyle = wb.createCellStyle(); Font font = wb.createFont(); @@ -257,35 +255,50 @@ public class ChartDataServer implements ChartDataApi { //设置单元格填充样式(使用纯色背景颜色填充) cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - if (CollectionUtils.isEmpty(request.getMultiInfo())) { - List details = request.getDetails(); - Integer[] excelTypes = request.getExcelTypes(); - details.add(0, request.getHeader()); - ViewDetailField[] detailFields = request.getDetailFields(); - Object[] header = request.getHeader(); - - //明细sheet - Sheet detailsSheet = wb.createSheet("数据"); - - setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); + if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info")) { + List details = new ArrayList<>(); + Sheet detailsSheet; + Integer sheetIndex = 1; + request.getViewInfo().getChartExtRequest().setPageSize(Long.valueOf(extractPageSize)); + ChartViewDTO chartViewDTO = findExcelData(request); + for (long i = 1; i < chartViewDTO.getTotalPage() + 1; i++) { + request.getViewInfo().getChartExtRequest().setGoPage(i); + findExcelData(request); + details.addAll(request.getDetails()); + if ((details.size() + extractPageSize) > sheetLimit) { + detailsSheet = wb.createSheet("数据" + sheetIndex); + Integer[] excelTypes = request.getExcelTypes(); + details.add(0, request.getHeader()); + ViewDetailField[] detailFields = request.getDetailFields(); + Object[] header = request.getHeader(); + ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); + sheetIndex++; + details.clear(); + } + } } else { - //多个sheet - for (int i = 0; i < request.getMultiInfo().size(); i++) { - ChartExcelRequestInner requestInner = request.getMultiInfo().get(i); - - List details = requestInner.getDetails(); - Integer[] excelTypes = requestInner.getExcelTypes(); - details.add(0, requestInner.getHeader()); - ViewDetailField[] detailFields = requestInner.getDetailFields(); - Object[] header = requestInner.getHeader(); - - //明细sheet - Sheet detailsSheet = wb.createSheet("数据 " + (i + 1)); - + findExcelData(request); + if (CollectionUtils.isEmpty(request.getMultiInfo())) { + List details = request.getDetails(); + Integer[] excelTypes = request.getExcelTypes(); + details.add(0, request.getHeader()); + ViewDetailField[] detailFields = request.getDetailFields(); + Object[] header = request.getHeader(); + Sheet detailsSheet = wb.createSheet("数据"); setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); + } else { + for (int i = 0; i < request.getMultiInfo().size(); i++) { + ChartExcelRequestInner requestInner = request.getMultiInfo().get(i); + List details = requestInner.getDetails(); + Integer[] excelTypes = requestInner.getExcelTypes(); + details.add(0, requestInner.getHeader()); + ViewDetailField[] detailFields = requestInner.getDetailFields(); + Object[] header = requestInner.getHeader(); + Sheet detailsSheet = wb.createSheet("数据 " + (i + 1)); + setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); + } } } - response.setContentType("application/vnd.ms-excel"); //文件名称 response.setHeader("Content-disposition", "attachment;filename=" + URLEncoder.encode(request.getViewName(), StandardCharsets.UTF_8) + ".xlsx"); @@ -308,7 +321,6 @@ public class ChartDataServer implements ChartDataApi { } - public static void setExcelData(Sheet detailsSheet, CellStyle cellStyle, Object[] header, List details, ViewDetailField[] detailFields, Integer[] excelTypes) { boolean mergeHead = false; if (ArrayUtils.isNotEmpty(detailFields)) { diff --git a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java index 4770e47bec..ce7e32c1a3 100644 --- a/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java +++ b/core/core-backend/src/main/java/io/dataease/exportCenter/manage/ExportCenterManage.java @@ -38,6 +38,7 @@ import io.dataease.extensions.datasource.dto.DatasourceSchemaDTO; import io.dataease.extensions.datasource.factory.ProviderFactory; import io.dataease.extensions.datasource.model.SQLMeta; import io.dataease.extensions.datasource.provider.Provider; +import io.dataease.extensions.view.dto.ChartViewDTO; import io.dataease.extensions.view.dto.ColumnPermissionItem; import io.dataease.extensions.view.dto.DatasetRowPermissionsTreeObj; import io.dataease.i18n.Translator; @@ -104,6 +105,11 @@ public class ExportCenterManage implements BaseExportApi { private Long limit; private final static String DATA_URL_TITLE = "data:image/jpeg;base64,"; private static final String exportData_path = "/opt/dataease2.0/data/exportData/"; + + public Integer getExtractPageSize() { + return extractPageSize; + } + @Value("${dataease.export.page.size:50000}") private Integer extractPageSize; static private List STATUS = Arrays.asList("SUCCESS", "FAILED", "PENDING", "IN_PROGRESS", "ALL"); @@ -122,7 +128,7 @@ public class ExportCenterManage implements BaseExportApi { private DatasetTableFieldManage datasetTableFieldManage; @Resource private DatasetDataManage datasetDataManage; - + private final Long sheetLimit = 1000000L; @Autowired(required = false) private DataFillingApi dataFillingApi = null; @@ -512,10 +518,17 @@ public class ExportCenterManage implements BaseExportApi { Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null); Long curLimit = getExportLimit(); totalCount = totalCount > curLimit ? curLimit : totalCount; - Long sheetLimit = 1000000L; + Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0); Workbook wb = new SXSSFWorkbook(); - FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx"); + CellStyle cellStyle = wb.createCellStyle(); + Font font = wb.createFont(); + font.setFontHeightInPoints((short) 12); + font.setBold(true); + cellStyle.setFont(font); + cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); + cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + for (Long s = 1L; s < sheetCount + 1; s++) { Long sheetSize; if (s.equals(sheetCount)) { @@ -539,14 +552,7 @@ public class ExportCenterManage implements BaseExportApi { Map previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList); List> data = (List>) previewData.get("data"); if (p.equals(0L)) { - detailsSheet = wb.createSheet("数据-" + s); - CellStyle cellStyle = wb.createCellStyle(); - Font font = wb.createFont(); - font.setFontHeightInPoints((short) 12); - font.setBold(true); - cellStyle.setFont(font); - cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + detailsSheet = wb.createSheet("数据" + s); List header = new ArrayList<>(); for (DatasetTableFieldDTO field : allFields) { header.add(field.getName()); @@ -617,6 +623,7 @@ public class ExportCenterManage implements BaseExportApi { exportTaskMapper.updateById(exportTask); } } + FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx"); wb.write(fileOutputStream); fileOutputStream.flush(); fileOutputStream.close(); @@ -647,53 +654,46 @@ public class ExportCenterManage implements BaseExportApi { try { exportTask.setExportStatus("IN_PROGRESS"); exportTaskMapper.updateById(exportTask); - chartDataServer.findExcelData(request); - Workbook wb = new SXSSFWorkbook(); - - //给单元格设置样式 CellStyle cellStyle = wb.createCellStyle(); Font font = wb.createFont(); - //设置字体大小 font.setFontHeightInPoints((short) 12); - //设置字体加粗 font.setBold(true); - //给字体设置样式 cellStyle.setFont(font); - //设置单元格背景颜色 cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - //设置单元格填充样式(使用纯色背景颜色填充) cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - if (CollectionUtils.isEmpty(request.getMultiInfo())) { - if (request.getViewInfo().getType().equalsIgnoreCase("chart-mix-dual-line")) { - - } else { - List details = request.getDetails(); - Integer[] excelTypes = request.getExcelTypes(); - details.add(0, request.getHeader()); - ViewDetailField[] detailFields = request.getDetailFields(); - Object[] header = request.getHeader(); - Sheet detailsSheet = wb.createSheet("数据"); - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); + List details = new ArrayList<>(); + Sheet detailsSheet; + Integer sheetIndex = 1; + if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info")) { + request.getViewInfo().getChartExtRequest().setPageSize(Long.valueOf(extractPageSize)); + ChartViewDTO chartViewDTO = chartDataServer.findExcelData(request); + for (long i = 1; i < chartViewDTO.getTotalPage() + 1; i++) { + request.getViewInfo().getChartExtRequest().setGoPage(i); + chartDataServer.findExcelData(request); + details.addAll(request.getDetails()); + if ((details.size() + extractPageSize) > sheetLimit) { + detailsSheet = wb.createSheet("数据" + sheetIndex); + Integer[] excelTypes = request.getExcelTypes(); + details.add(0, request.getHeader()); + ViewDetailField[] detailFields = request.getDetailFields(); + Object[] header = request.getHeader(); + ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); + sheetIndex++; + details.clear(); + exportTask.setExportStatus("IN_PROGRESS"); + double exportRogress = (double) (i / chartViewDTO.getTotalPage() + 1); + DecimalFormat df = new DecimalFormat("#.##"); + String formattedResult = df.format((exportRogress) * 100); + exportTask.setExportProgress(formattedResult); + exportTaskMapper.updateById(exportTask); + } } } else { - //多个sheet - for (int i = 0; i < request.getMultiInfo().size(); i++) { - ChartExcelRequestInner requestInner = request.getMultiInfo().get(i); - - List details = requestInner.getDetails(); - Integer[] excelTypes = requestInner.getExcelTypes(); - details.add(0, requestInner.getHeader()); - ViewDetailField[] detailFields = requestInner.getDetailFields(); - Object[] header = requestInner.getHeader(); - - //明细sheet - Sheet detailsSheet = wb.createSheet("数据 " + (i + 1)); - - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); - } + downloadNotTableInfoData(request, wb); } + try (FileOutputStream outputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx")) { wb.write(outputStream); outputStream.flush(); @@ -713,6 +713,48 @@ public class ExportCenterManage implements BaseExportApi { Running_Task.put(exportTask.getId(), future); } + private void downloadNotTableInfoData(ChartExcelRequest request, Workbook wb) { + chartDataServer.findExcelData(request); + //给单元格设置样式 + CellStyle cellStyle = wb.createCellStyle(); + Font font = wb.createFont(); + //设置字体大小 + font.setFontHeightInPoints((short) 12); + //设置字体加粗 + font.setBold(true); + //给字体设置样式 + cellStyle.setFont(font); + //设置单元格背景颜色 + cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); + //设置单元格填充样式(使用纯色背景颜色填充) + cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); + if (CollectionUtils.isEmpty(request.getMultiInfo())) { + if (request.getViewInfo().getType().equalsIgnoreCase("chart-mix-dual-line")) { + } else { + List details = request.getDetails(); + Integer[] excelTypes = request.getExcelTypes(); + details.add(0, request.getHeader()); + ViewDetailField[] detailFields = request.getDetailFields(); + Object[] header = request.getHeader(); + Sheet detailsSheet = wb.createSheet("数据"); + ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); + } + } else { + //多个sheet + for (int i = 0; i < request.getMultiInfo().size(); i++) { + ChartExcelRequestInner requestInner = request.getMultiInfo().get(i); + + List details = requestInner.getDetails(); + Integer[] excelTypes = requestInner.getExcelTypes(); + details.add(0, requestInner.getHeader()); + ViewDetailField[] detailFields = requestInner.getDetailFields(); + Object[] header = requestInner.getHeader(); + //明细sheet + Sheet detailsSheet = wb.createSheet("数据 " + (i + 1)); + ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes); + } + } + } private void setFileSize(String filePath, CoreExportTask exportTask) { File file = new File(filePath);