From 1fa5415fa3dd8d52dd376f14a2d3402e1f54ded9 Mon Sep 17 00:00:00 2001 From: taojinlong Date: Tue, 4 Mar 2025 21:30:55 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E3=80=90=E6=95=B0=E6=8D=AE=E6=BA=90?= =?UTF-8?q?=E3=80=91=E6=95=B0=E6=8D=AE=E6=BA=90=E6=96=B0=E5=A2=9E=E5=AF=B9?= =?UTF-8?q?=E8=BF=9C=E7=A8=8B=20Excel/CSV=20=E7=9A=84=E6=94=AF=E6=8C=81=20?= =?UTF-8?q?#14681?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/core-backend/pom.xml | 5 + .../dataset/manage/DatasetDataManage.java | 4 +- .../dataset/manage/DatasetSQLManage.java | 2 +- .../manage/DatasourceSyncManage.java | 46 +- .../datasource/provider/CalciteProvider.java | 6 +- .../datasource/provider/ExcelUtils.java | 299 +++- .../datasource/server/DatasourceServer.java | 202 ++- .../server/DataVisualizationServer.java | 4 +- core/core-frontend/src/api/datasource.ts | 4 + .../src/assets/svg/Excel-remote-ds.svg | 6 + .../components/icon-group/datasource-list.ts | 2 + core/core-frontend/src/locales/en.ts | 8 +- core/core-frontend/src/locales/tw.ts | 8 +- core/core-frontend/src/locales/zh-CN.ts | 8 +- .../data/datasource/form/CreatDsGroup.vue | 2 +- .../data/datasource/form/ExcelDetail.vue | 5 +- .../datasource/form/ExcelRemoteDetail.vue | 1408 +++++++++++++++++ .../visualized/data/datasource/form/index.vue | 121 +- .../visualized/data/datasource/form/option.ts | 8 +- .../visualized/data/datasource/index.vue | 92 +- pom.xml | 1 + .../io/dataease/api/ds/DatasourceApi.java | 7 +- .../api/ds/vo/ExcelConfiguration.java | 13 + .../api/ds/vo/RemoteExcelRequest.java | 9 + .../io/dataease/constant/DataSourceType.java | 2 +- .../io/dataease/utils/HttpClientUtil.java | 61 +- .../vo/DatasourceConfiguration.java | 1 + 27 files changed, 2194 insertions(+), 140 deletions(-) create mode 100644 core/core-frontend/src/assets/svg/Excel-remote-ds.svg create mode 100644 core/core-frontend/src/views/visualized/data/datasource/form/ExcelRemoteDetail.vue create mode 100644 sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/ExcelConfiguration.java create mode 100644 sdk/api/api-base/src/main/java/io/dataease/api/ds/vo/RemoteExcelRequest.java diff --git a/core/core-backend/pom.xml b/core/core-backend/pom.xml index a1917fcbfd..34caa1e919 100644 --- a/core/core-backend/pom.xml +++ b/core/core-backend/pom.xml @@ -13,6 +13,11 @@ core-backend + + commons-net + commons-net + ${commons-net.version} + com.google.guava guava diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java index 27b34ee420..ed30489df4 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetDataManage.java @@ -104,7 +104,7 @@ public class DatasetDataManage { if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB) || StringUtils.equalsIgnoreCase(type, DatasetTableType.SQL)) { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId()); DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); - if (StringUtils.equalsIgnoreCase("excel", coreDatasource.getType()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { coreDatasource = engineManage.getDeEngine(); } if (StringUtils.isNotEmpty(coreDatasource.getStatus()) && "Error".equalsIgnoreCase(coreDatasource.getStatus())) { @@ -407,7 +407,7 @@ public class DatasetDataManage { public Map previewSql(PreviewSqlDTO dto) throws DEException { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dto.getDatasourceId()); DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); - if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().equalsIgnoreCase("Excel")) { + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name())) { BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine()); } else { BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource); diff --git a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java index 81dcdfa527..93ced0d19e 100644 --- a/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java +++ b/core/core-backend/src/main/java/io/dataease/dataset/manage/DatasetSQLManage.java @@ -510,7 +510,7 @@ public class DatasetSQLManage { if (coreDatasource == null) { DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + ds.getDatasourceId()); } - if (StringUtils.equalsIgnoreCase("excel", coreDatasource.getType()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { coreDatasource = engineManage.getDeEngine(); } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java b/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java index 2ff97d96e0..c1ceb6048e 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java @@ -127,7 +127,11 @@ public class DatasourceSyncManage { record.setQrtzInstance(context.getFireInstanceId()); datasourceMapper.update(record, updateWrapper); } - extractedData(taskId, coreDatasource, updateType, coreDatasourceTask.getSyncRate()); + if (coreDatasource.getType().equalsIgnoreCase("ExcelRemote")) { + extractedExcelData(taskId, coreDatasource, updateType, coreDatasourceTask.getSyncRate()); + } else { + extractedData(taskId, coreDatasource, updateType, coreDatasourceTask.getSyncRate()); + } } catch (Exception e) { LogUtil.error(e); } finally { @@ -173,6 +177,46 @@ public class DatasourceSyncManage { } } + public void extractedExcelData(Long taskId, CoreDatasource coreDatasource, DatasourceServer.UpdateType updateType, String scheduleType) { + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(transDTO(coreDatasource)); + List tables = ExcelUtils.getTables(datasourceRequest); + for (DatasetTableDTO tableDTO : tables) { + CoreDatasourceTaskLog datasetTableTaskLog = datasourceTaskServer.initTaskLog(coreDatasource.getId(), taskId, tableDTO.getTableName(), scheduleType); + datasourceRequest.setTable(tableDTO.getTableName()); + ExcelUtils.getTableFields(datasourceRequest); + List tableFields = ExcelUtils.getTableFields(datasourceRequest); + try { + datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Begin to sync datatable: " + datasourceRequest.getTable()); + createEngineTable(datasourceRequest.getTable(), tableFields); + if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { + createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields); + } + extractExcelData(datasourceRequest, updateType, tableFields); + if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { + replaceTable(datasourceRequest.getTable()); + } + datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n End to sync datatable: " + datasourceRequest.getTable()); + datasetTableTaskLog.setTaskStatus(TaskStatus.Completed.toString()); + datasetTableTaskLog.setEndTime(System.currentTimeMillis()); + } catch (Exception e) { + try { + if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { + dropEngineTable(TableUtils.tmpName(datasourceRequest.getTable())); + } + } catch (Exception ignore) { + } + datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Failed to sync datatable: " + datasourceRequest.getTable() + ", " + e.getMessage()); + datasetTableTaskLog.setTaskStatus(TaskStatus.Error.toString()); + datasetTableTaskLog.setEndTime(System.currentTimeMillis()); + + e.printStackTrace(); + } finally { + datasourceTaskServer.saveLog(datasetTableTaskLog); + } + } + } + private void updateDsTaskStatus(Long datasourceId) { UpdateWrapper updateWrapper = new UpdateWrapper<>(); updateWrapper.eq("id", datasourceId); diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java index b37c9eb7b0..8edc9e96c2 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/CalciteProvider.java @@ -1332,7 +1332,7 @@ public class CalciteProvider extends Provider { public void initConnectionPool() { LogUtil.info("Begin to init datasource pool..."); QueryWrapper datasourceQueryWrapper = new QueryWrapper(); - List coreDatasources = coreDatasourceMapper.selectList(datasourceQueryWrapper).stream().filter(coreDatasource -> !Arrays.asList("folder", "API", "Excel").contains(coreDatasource.getType())).collect(Collectors.toList()); + List coreDatasources = coreDatasourceMapper.selectList(datasourceQueryWrapper).stream().filter(coreDatasource -> !Arrays.asList("folder", "API", "Excel", "ExcelRemote").contains(coreDatasource.getType())).collect(Collectors.toList()); CoreDatasource engine = engineManage.deEngine(); if (engine != null) { coreDatasources.add(engine); @@ -1382,8 +1382,8 @@ public class CalciteProvider extends Provider { buildSchema(datasourceRequest, calciteConnection); } DatasourceConfiguration configuration = JsonUtil.parseObject(datasourceDTO.getConfiguration(), DatasourceConfiguration.class); - if(configuration.isUseSSH()){ - Session session =Provider.getSessions().get(datasourceDTO.getId()); + if (configuration.isUseSSH()) { + Session session = Provider.getSessions().get(datasourceDTO.getId()); session.disconnect(); Provider.getSessions().remove(datasourceDTO.getId()); startSshSession(configuration, null, datasourceDTO.getId()); diff --git a/core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java b/core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java index 61df60cb5d..2eaa30df48 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java @@ -12,6 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.dataease.api.ds.vo.ExcelFileData; import io.dataease.api.ds.vo.ExcelSheetData; +import io.dataease.api.ds.vo.RemoteExcelRequest; import io.dataease.commons.utils.EncryptUtils; import io.dataease.datasource.dao.auto.entity.CoreDatasource; import io.dataease.exception.DEException; @@ -19,9 +20,12 @@ import io.dataease.extensions.datasource.dto.DatasetTableDTO; import io.dataease.extensions.datasource.dto.DatasourceDTO; import io.dataease.extensions.datasource.dto.DatasourceRequest; import io.dataease.extensions.datasource.dto.TableField; +import io.dataease.api.ds.vo.ExcelConfiguration; import io.dataease.utils.*; import lombok.Data; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPClient; import org.springframework.util.CollectionUtils; import org.springframework.web.multipart.MultipartFile; @@ -53,21 +57,42 @@ public class ExcelUtils { }; public static void mergeSheets(CoreDatasource requestDatasource, DatasourceDTO sourceData) { - List newSheets = JsonUtil.parseList(requestDatasource.getConfiguration(), sheets); - List tableNames = newSheets.stream().map(ExcelSheetData::getDeTableName).collect(Collectors.toList()); - List oldSheets = JsonUtil.parseList(sourceData.getConfiguration(), sheets); - for (ExcelSheetData oldSheet : oldSheets) { - if (!tableNames.contains(oldSheet.getDeTableName())) { - newSheets.add(oldSheet); + if (requestDatasource.getType().equalsIgnoreCase("Excel")) { + List newSheets = JsonUtil.parseList(requestDatasource.getConfiguration(), sheets); + List tableNames = newSheets.stream().map(ExcelSheetData::getDeTableName).collect(Collectors.toList()); + List oldSheets = JsonUtil.parseList(sourceData.getConfiguration(), sheets); + for (ExcelSheetData oldSheet : oldSheets) { + if (!tableNames.contains(oldSheet.getDeTableName())) { + newSheets.add(oldSheet); + } } + requestDatasource.setConfiguration(JsonUtil.toJSONString(newSheets).toString()); + } else { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(requestDatasource.getConfiguration(), ExcelConfiguration.class); + List newSheets = excelConfiguration.getSheets(); + List tableNames = newSheets.stream().map(ExcelSheetData::getDeTableName).collect(Collectors.toList()); + List oldSheets = JsonUtil.parseObject(sourceData.getConfiguration(), ExcelConfiguration.class).getSheets(); + for (ExcelSheetData oldSheet : oldSheets) { + if (!tableNames.contains(oldSheet.getDeTableName())) { + newSheets.add(oldSheet); + } + } + excelConfiguration.setSheets(newSheets); + requestDatasource.setConfiguration(JsonUtil.toJSONString(excelConfiguration).toString()); } - requestDatasource.setConfiguration(JsonUtil.toJSONString(newSheets).toString()); + } public static List getTables(DatasourceRequest datasourceRequest) throws DEException { List tableDescs = new ArrayList<>(); try { - JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()); + String sheets = ""; + if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("Excel")) { + sheets = datasourceRequest.getDatasource().getConfiguration(); + } else { + sheets = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()).get("sheets").toString(); + } + JsonNode rootNode = objectMapper.readTree(sheets); for (int i = 0; i < rootNode.size(); i++) { DatasetTableDTO datasetTableDTO = new DatasetTableDTO(); datasetTableDTO.setTableName(rootNode.get(i).get("deTableName").asText()); @@ -83,20 +108,24 @@ public class ExcelUtils { return tableDescs; } - public static Map getTableNamesMap(String configration) throws DEException { + public static Map getTableNamesMap(String type, String configuration) throws DEException { Map result = new HashMap<>(); JsonNode rootNode = null; // 兼容历史未加密信息 + String sheets = configuration; try { - rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(configration)); + if (type.equalsIgnoreCase("ExcelRemote")) { + sheets = objectMapper.readTree(configuration).get("sheets").toString(); + } + rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(sheets)); } catch (Exception e) { try { - rootNode = objectMapper.readTree(configration); + rootNode = objectMapper.readTree(sheets); } catch (Exception ex) { DEException.throwException(ex); } } - if(rootNode != null) { + if (rootNode != null) { for (int i = 0; i < rootNode.size(); i++) { result.put(rootNode.get(i).get("tableName").asText(), rootNode.get(i).get("deTableName").asText()); } @@ -105,6 +134,12 @@ public class ExcelUtils { } public static String getFileName(CoreDatasource datasource) throws DEException { + if (datasource.getType().equalsIgnoreCase("ExcelRemote")) { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasource.getConfiguration(), ExcelConfiguration.class); + for (ExcelSheetData sheet : excelConfiguration.getSheets()) { + return sheet.getFileName(); + } + } JsonNode rootNode = null; try { rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(datasource.getConfiguration())); @@ -120,12 +155,13 @@ public class ExcelUtils { return rootNode.get(i).get("fileName").asText(); } } - - return ""; } public static String getSize(CoreDatasource datasource) throws DEException { + if (datasource.getType().equalsIgnoreCase("ExcelRemote")) { + return "0 B"; + } try { JsonNode rootNode = objectMapper.readTree(datasource.getConfiguration()); for (int i = 0; i < rootNode.size(); i++) { @@ -138,31 +174,50 @@ public class ExcelUtils { return "0 B"; } - public List fetchDataList(DatasourceRequest datasourceRequest) throws DEException { + public List fetchDataList(DatasourceRequest datasourceRequest) throws DEException, IOException { List dataList = new ArrayList<>(); - try { - JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()); - for (int i = 0; i < rootNode.size(); i++) { - if (rootNode.get(i).get("deTableName").asText().equalsIgnoreCase(datasourceRequest.getTable())) { - List tableFields = JsonUtil.parseList(rootNode.get(i).get("fields").toString(), TableFieldListTypeReference); - String suffix = rootNode.get(i).get("path").asText().substring(rootNode.get(i).get("path").asText().lastIndexOf(".") + 1); - InputStream inputStream = new FileInputStream(rootNode.get(i).get("path").asText()); + if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("ExcelRemote")) { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), ExcelConfiguration.class); + Map fileNames = downLoadRemoteExcel(excelConfiguration, datasourceRequest.getDatasource().getCreateBy()); + for (ExcelSheetData sheet : excelConfiguration.getSheets()) { + if (sheet.getDeTableName().equalsIgnoreCase(datasourceRequest.getTable())) { + List tableFields = sheet.getFields(); + String suffix = fileNames.get("fileName").substring(fileNames.get("fileName").lastIndexOf(".") + 1); + InputStream inputStream = new FileInputStream(path + datasourceRequest.getDatasource().getCreateBy() + "/" + fileNames.get("tranName")); if (StringUtils.equalsIgnoreCase(suffix, "csv")) { BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); reader.readLine();//去掉表头 dataList = csvData(reader, false, tableFields.size()); } else { - dataList = fetchExcelDataList(rootNode.get(i).get("tableName").asText(), inputStream); + dataList = fetchExcelDataList(sheet.getTableName(), inputStream); } } } - } catch (Exception e) { - DEException.throwException(e); + } else { + try { + JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()); + for (int i = 0; i < rootNode.size(); i++) { + if (rootNode.get(i).get("deTableName").asText().equalsIgnoreCase(datasourceRequest.getTable())) { + List tableFields = JsonUtil.parseList(rootNode.get(i).get("fields").toString(), TableFieldListTypeReference); + String suffix = rootNode.get(i).get("path").asText().substring(rootNode.get(i).get("path").asText().lastIndexOf(".") + 1); + InputStream inputStream = new FileInputStream(rootNode.get(i).get("path").asText()); + if (StringUtils.equalsIgnoreCase(suffix, "csv")) { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + reader.readLine();//去掉表头 + dataList = csvData(reader, false, tableFields.size()); + } else { + dataList = fetchExcelDataList(rootNode.get(i).get("tableName").asText(), inputStream); + } + } + } + } catch (Exception e) { + DEException.throwException(e); + } } return dataList; } - public List fetchExcelDataList(String sheetName, InputStream inputStream) { + private List fetchExcelDataList(String sheetName, InputStream inputStream) { NoModelDataListener noModelDataListener = new NoModelDataListener(); ExcelReader excelReader = EasyExcel.read(inputStream, noModelDataListener).build(); List sheets = excelReader.excelExecutor().sheetList(); @@ -189,7 +244,13 @@ public class ExcelUtils { TypeReference> listTypeReference = new TypeReference>() { }; try { - JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()); + String sheets = ""; + if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("Excel")) { + sheets = datasourceRequest.getDatasource().getConfiguration(); + } else { + sheets = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()).get("sheets").toString(); + } + JsonNode rootNode = objectMapper.readTree(sheets); for (int i = 0; i < rootNode.size(); i++) { if (rootNode.get(i).get("deTableName").asText().equalsIgnoreCase(datasourceRequest.getTable())) { tableFields = JsonUtil.parseList(rootNode.get(i).get("fields").toString(), listTypeReference); @@ -201,7 +262,7 @@ public class ExcelUtils { return tableFields; } - public ExcelFileData excelSaveAndParse(MultipartFile file) throws DEException { + public ExcelFileData excelSaveAndParse(MultipartFile file, String createBy) throws DEException { String filename = file.getOriginalFilename(); List excelSheetDataList = null; try { @@ -211,12 +272,10 @@ public class ExcelUtils { } List returnSheetDataList = new ArrayList<>(); returnSheetDataList = excelSheetDataList; - returnSheetDataList = returnSheetDataList.stream() - .filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields())) - .collect(Collectors.toList()); + returnSheetDataList = returnSheetDataList.stream().filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields())).collect(Collectors.toList()); // save file String excelId = UUID.randomUUID().toString(); - String filePath = saveFile(file, excelId); + String filePath = saveFile(file, excelId, createBy); for (ExcelSheetData excelSheetData : returnSheetDataList) { excelSheetData.setLastUpdateTime(System.currentTimeMillis()); @@ -272,12 +331,116 @@ public class ExcelUtils { return excelFileData; } - private static String saveFile(MultipartFile file, String fileNameUUID) throws DEException { + public static String checkStatus(DatasourceRequest datasourceRequest) throws FileNotFoundException { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), ExcelConfiguration.class); + Map fileNames = new HashMap<>(); + try { + fileNames = downLoadRemoteExcel(excelConfiguration, datasourceRequest.getDatasource().getCreateBy()); + return "Success"; + } catch (Exception e) { + throw e; + } finally { + String dirPath = path + datasourceRequest.getDatasource().getCreateBy() + "/"; + if (StringUtils.isNotEmpty(fileNames.get("tranName"))) { + FileUtils.deleteFile(dirPath + fileNames.get("tranName")); + } + } + } + + public ExcelFileData parseRemoteExcel(RemoteExcelRequest remoteExcelRequest, String createBy) throws DEException, FileNotFoundException { + String dirPath = path + createBy + "/"; + Map fileNames = downLoadRemoteExcel(remoteExcelRequest, createBy); + FileInputStream fileInputStream = new FileInputStream(dirPath + fileNames.get("tranName")); + List excelSheetDataList = null; + try { + excelSheetDataList = parseExcel(fileNames.get("fileName"), fileInputStream, true); + } catch (Exception e) { + e.printStackTrace(); + DEException.throwException(e); + } + List returnSheetDataList = new ArrayList<>(); + returnSheetDataList = excelSheetDataList; + returnSheetDataList = returnSheetDataList.stream().filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields())).collect(Collectors.toList()); + for (ExcelSheetData excelSheetData : returnSheetDataList) { + excelSheetData.setLastUpdateTime(System.currentTimeMillis()); + excelSheetData.setTableName(excelSheetData.getExcelLabel()); + excelSheetData.setDeTableName("excel_" + excelSheetData.getExcelLabel() + "_" + UUID.randomUUID().toString().replace("-", "").substring(0, 10)); + excelSheetData.setPath(dirPath + fileNames.get("tranName")); + excelSheetData.setSheetId(UUID.randomUUID().toString()); + excelSheetData.setSheetExcelId(fileNames.get("tranName").split("\\.")[0]); + excelSheetData.setFileName(fileNames.get("fileName")); + /** + * dataease字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制 + */ + for (TableField field : excelSheetData.getFields()) { + //TEXT LONG DATETIME DOUBLE + if (field.getFieldType().equalsIgnoreCase("TEXT")) { + field.setDeType(0); + field.setDeExtractType(0); + } + if (field.getFieldType().equalsIgnoreCase("DATETIME")) { + field.setDeType(1); + field.setDeExtractType(1); + } + if (field.getFieldType().equalsIgnoreCase("LONG")) { + field.setDeType(2); + field.setDeExtractType(2); + } + if (field.getFieldType().equalsIgnoreCase("DOUBLE")) { + field.setDeType(3); + field.setDeExtractType(3); + } + } + long size = 0; + File file = new File(dirPath + fileNames.get("tranName")); + String unit = "B"; + if (file.length() / 1024 == 0) { + size = file.length(); + } + if (0 < file.length() / 1024 && file.length() / 1024 < 1024) { + size = file.length() / 1024; + unit = "KB"; + } + if (1024 <= file.length() / 1024) { + size = file.length() / 1024 / 1024; + unit = "MB"; + } + excelSheetData.setSize(size + " " + unit); + } + + ExcelFileData excelFileData = new ExcelFileData(); + excelFileData.setExcelLabel(fileNames.get("fileName").split("\\.")[0]); + excelFileData.setId(fileNames.get("tranName").split("\\.")[0]); + excelFileData.setPath(dirPath + fileNames.get("tranName")); + excelFileData.setSheets(returnSheetDataList); + + return excelFileData; + } + + private static Map downLoadRemoteExcel(ExcelConfiguration remoteExcelRequest, String createBy) throws DEException, FileNotFoundException { + Map fileNames = new HashMap<>(); + if (remoteExcelRequest.getUrl().trim().startsWith("http")) { + HttpClientConfig httpClientConfig = new HttpClientConfig(); + if (StringUtils.isNotEmpty(remoteExcelRequest.getUsername()) && StringUtils.isNotEmpty(remoteExcelRequest.getPassword())) { + String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((remoteExcelRequest.getUsername() + ":" + remoteExcelRequest.getPassword()).getBytes()); + httpClientConfig.addHeader("Authorization", authValue); + } + String dirPath = path + createBy + "/"; + fileNames = HttpClientUtil.downloadFile(remoteExcelRequest.getUrl(), httpClientConfig, dirPath); + }else if (remoteExcelRequest.getUrl().trim().startsWith("ftp")) { + fileNames = downLoadFromFtp(remoteExcelRequest, createBy); + }else { + DEException.throwException("不支持的协议！"); + } + return fileNames; + } + + private static String saveFile(MultipartFile file, String fileNameUUID, String createBy) throws DEException { String filePath = null; try { String filename = file.getOriginalFilename(); String suffix = filename.substring(filename.lastIndexOf(".") + 1); - String dirPath = path + AuthUtils.getUser().getUserId() + "/"; + String dirPath = path + createBy + "/"; File p = new File(dirPath); if (!p.exists()) { p.mkdirs(); @@ -439,7 +602,7 @@ public class ExcelUtils { } - public List parseExcel(String filename, InputStream inputStream, boolean isPreview) throws IOException { + private List parseExcel(String filename, InputStream inputStream, boolean isPreview) throws IOException { List excelSheetDataList = new ArrayList<>(); String suffix = filename.substring(filename.lastIndexOf(".") + 1); if (StringUtils.equalsIgnoreCase(suffix, "xlsx") || StringUtils.equalsIgnoreCase(suffix, "xls")) { @@ -560,5 +723,71 @@ public class ExcelUtils { return excelSheetDataList; } + private static Map downLoadFromFtp(ExcelConfiguration remoteExcelRequest, String createBy) { + Map fileNames = new HashMap<>(); + String username = ""; + String password = ""; + String serverAddress = ""; + String filePath = ""; + if (remoteExcelRequest.getUrl().contains("@")) { + String regex = "ftp://([^:]+):([^@]+)@([^/]+)(.*)"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(remoteExcelRequest.getUrl()); + if (matcher.find()) { + username = matcher.group(1); + password = matcher.group(2); + serverAddress = matcher.group(3); + filePath = matcher.group(4); + } else { + DEException.throwException("无效的地址！"); + } + } else { + String regex = "ftp://([^/]+)(.*)"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(remoteExcelRequest.getUrl()); + if (matcher.find()) { + serverAddress = matcher.group(1); + filePath = matcher.group(2); + } else { + DEException.throwException("无效的地址！"); + } + } + if(StringUtils.isNotEmpty(remoteExcelRequest.getUsername()) && StringUtils.isNotEmpty(remoteExcelRequest.getPassword())){ + username = remoteExcelRequest.getUsername(); + password = remoteExcelRequest.getPassword(); + } + int port = 21; + String suffix = filePath.substring(filePath.lastIndexOf(".") + 1); + String tranName = UUID.randomUUID().toString() + "." + suffix;; + String localFilePath = path + createBy + "/" + tranName; + fileNames.put("fileName", filePath); + fileNames.put("tranName", tranName); + FTPClient ftpClient = new FTPClient(); + try { + ftpClient.connect(serverAddress, port); + ftpClient.login(username, password); + ftpClient.enterLocalPassiveMode(); + ftpClient.setFileType(FTP.BINARY_FILE_TYPE); + File localFile = new File(localFilePath); + try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(localFile))) { + boolean success = ftpClient.retrieveFile(filePath, outputStream); + if (!success) { + DEException.throwException("文件下载失败！"); + } + } + } catch (IOException e) { + DEException.throwException(e); + } finally { + try { + if (ftpClient.isConnected()) { + ftpClient.logout(); + ftpClient.disconnect(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + return fileNames; + } } diff --git a/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java b/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java index 884ad03a4e..3f59d2509c 100644 --- a/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java +++ b/core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java @@ -48,6 +48,8 @@ import io.dataease.system.dao.auto.entity.CoreSysSetting; import io.dataease.system.manage.CoreUserManage; import io.dataease.utils.*; import jakarta.annotation.Resource; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.http.HttpServletResponse; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -55,6 +57,7 @@ import org.quartz.JobDataMap; import org.quartz.JobKey; import org.quartz.TriggerKey; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ResourceLoader; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -62,11 +65,10 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; +import java.io.*; import java.lang.reflect.Method; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; @@ -127,7 +129,7 @@ public class DatasourceServer implements DatasourceApi { } public boolean checkRepeat(@RequestBody BusiDsRequest dataSourceDTO) { - if (Arrays.asList("Excel", "folder", "es").contains(dataSourceDTO.getType()) || dataSourceDTO.getType().contains("API")) { + if (Arrays.asList("folder", "es").contains(dataSourceDTO.getType()) || dataSourceDTO.getType().contains("API") || dataSourceDTO.getType().contains("Excel")) { return false; } BusiNodeRequest request = new BusiNodeRequest(); @@ -282,6 +284,20 @@ public class DatasourceServer implements DatasourceApi { } datasourceSyncManage.extractExcelData(coreDatasource, "all_scope"); } else if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(dataSourceDTO); + List tables = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); + checkName(tables.stream().map(DatasetTableDTO::getName).collect(Collectors.toList())); + for (DatasetTableDTO api : tables) { + datasourceRequest.setTable(api.getTableName()); + List tableFields = (List) invokeMethod(coreDatasource.getType(), "getTableFields", DatasourceRequest.class, datasourceRequest); + try { + datasourceSyncManage.createEngineTable(datasourceRequest.getTable(), tableFields); + } catch (Exception e) { + DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ": " + e.getMessage()); + } + } + CoreDatasourceTask coreDatasourceTask = new CoreDatasourceTask(); BeanUtils.copyBean(coreDatasourceTask, dataSourceDTO.getSyncSetting()); coreDatasourceTask.setName(coreDatasource.getName() + "-task"); @@ -299,19 +315,41 @@ public class DatasourceServer implements DatasourceApi { coreDatasourceTask.setTaskStatus(TaskStatus.WaitingForExecution.name()); datasourceTaskServer.insert(coreDatasourceTask); datasourceSyncManage.addSchedule(coreDatasourceTask); + } else if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dataSourceDTO); - List tables = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); - checkName(tables.stream().map(DatasetTableDTO::getName).collect(Collectors.toList())); - for (DatasetTableDTO api : tables) { - datasourceRequest.setTable(api.getTableName()); - List tableFields = (List) invokeMethod(coreDatasource.getType(), "getTableFields", DatasourceRequest.class, datasourceRequest); + List tables = ExcelUtils.getTables(datasourceRequest); + for (DatasetTableDTO table : tables) { + datasourceRequest.setTable(table.getTableName()); + List tableFields = ExcelUtils.getTableFields(datasourceRequest); try { datasourceSyncManage.createEngineTable(datasourceRequest.getTable(), tableFields); } catch (Exception e) { - DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ": " + e.getMessage()); + e.printStackTrace(); + if (e.getMessage().toLowerCase().contains("Row size too large".toLowerCase())) { + DEException.throwException("文本内容超出最大支持范围： " + datasourceRequest.getTable() + ", " + e.getMessage()); + } else { + DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ", " + e.getMessage()); + } } } + CoreDatasourceTask coreDatasourceTask = new CoreDatasourceTask(); + BeanUtils.copyBean(coreDatasourceTask, dataSourceDTO.getSyncSetting()); + coreDatasourceTask.setName(coreDatasource.getName() + "-task"); + coreDatasourceTask.setDsId(coreDatasource.getId()); + if (coreDatasourceTask.getStartTime() == null) { + coreDatasourceTask.setStartTime(System.currentTimeMillis() - 20 * 1000); + } + if (StringUtils.equalsIgnoreCase(coreDatasourceTask.getSyncRate(), RIGHTNOW.toString())) { + coreDatasourceTask.setCron(null); + } else { + if (coreDatasourceTask.getEndTime() != null && coreDatasourceTask.getEndTime() > 0 && coreDatasourceTask.getStartTime() > coreDatasourceTask.getEndTime()) { + DEException.throwException("结束时间不能小于开始时间！"); + } + } + coreDatasourceTask.setTaskStatus(TaskStatus.WaitingForExecution.name()); + datasourceTaskServer.insert(coreDatasourceTask); + datasourceSyncManage.addSchedule(coreDatasourceTask); } else { checkParams(dataSourceDTO.getConfiguration()); calciteProvider.update(dataSourceDTO); @@ -441,6 +479,51 @@ public class DatasourceServer implements DatasourceApi { dataSourceManage.checkName(dataSourceDTO); dataSourceManage.innerEdit(requestDatasource); } + } else if (dataSourceDTO.getType().equals(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) { + requestDatasource.setEnableDataFill(null); + List sourceTables = ExcelUtils.getTables(sourceTableRequest).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList()); + List tables = ExcelUtils.getTables(datasourceRequest).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList()); + CoreDatasourceTask coreDatasourceTask = new CoreDatasourceTask(); + BeanUtils.copyBean(coreDatasourceTask, dataSourceDTO.getSyncSetting()); + coreDatasourceTask.setName(requestDatasource.getName() + "-task"); + coreDatasourceTask.setDsId(requestDatasource.getId()); + if (StringUtils.equalsIgnoreCase(coreDatasourceTask.getSyncRate(), RIGHTNOW.toString())) { + coreDatasourceTask.setStartTime(System.currentTimeMillis() - 20 * 1000); + coreDatasourceTask.setCron(null); + } else { + if (coreDatasourceTask.getEndTime() != null && coreDatasourceTask.getEndTime() > 0 && coreDatasourceTask.getStartTime() > coreDatasourceTask.getEndTime()) { + DEException.throwException("结束时间不能小于开始时间！"); + } + } + coreDatasourceTask.setTaskStatus(TaskStatus.WaitingForExecution.toString()); + datasourceTaskServer.update(coreDatasourceTask); + datasourceSyncManage.deleteSchedule(datasourceTaskServer.selectByDSId(dataSourceDTO.getId())); + datasourceSyncManage.addSchedule(coreDatasourceTask); + if (Objects.equals(dataSourceDTO.getEditType(), replace)) { + toCreateTables = tables; + toDeleteTables = sourceTables.stream().filter(s -> tables.contains(s)).collect(Collectors.toList()); + for (String deleteTable : toDeleteTables) { + try { + datasourceSyncManage.dropEngineTable(deleteTable); + } catch (Exception ignore) { + } + } + for (String toCreateTable : toCreateTables) { + datasourceRequest.setTable(toCreateTable); + try { + datasourceSyncManage.createEngineTable(toCreateTable, ExcelUtils.getTableFields(datasourceRequest)); + } catch (Exception e) { + DEException.throwException("Failed to create table " + toCreateTable + ", " + e.getMessage()); + } + } + dataSourceManage.checkName(dataSourceDTO); + ExcelUtils.mergeSheets(requestDatasource, sourceData); + dataSourceManage.innerEdit(requestDatasource); + } else { + ExcelUtils.mergeSheets(requestDatasource, sourceData); + dataSourceManage.checkName(dataSourceDTO); + dataSourceManage.innerEdit(requestDatasource); + } } else { if (!LicenseUtil.licenseValid()) { requestDatasource.setEnableDataFill(null); @@ -618,7 +701,7 @@ public class DatasourceServer implements DatasourceApi { } datasourceMapper.deleteById(datasourceId); - if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType())) { + if (!Arrays.asList("API", "Excel", "folder", "APILark", "ExcelRemote").contains(coreDatasource.getType())) { calciteProvider.delete(coreDatasource); } @@ -693,7 +776,7 @@ public class DatasourceServer implements DatasourceApi { }); return datasetTableDTOS; } - if (coreDatasource.getType().equals("Excel")) { + if (coreDatasource.getType().contains("Excel")) { return ExcelUtils.getTables(datasourceRequest); } Provider provider = ProviderFactory.getProvider(datasourceDTO.getType()); @@ -712,7 +795,7 @@ public class DatasourceServer implements DatasourceApi { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(Long.parseLong(datasourceId)); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(transDTO(coreDatasource)); - if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().equals("Excel")) { + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().contains("Excel")) { datasourceRequest.setDatasource(transDTO(engineManage.getDeEngine())); DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine()); @@ -776,13 +859,13 @@ public class DatasourceServer implements DatasourceApi { private static final Integer replace = 0; private static final Integer append = 1; - public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("id") long datasourceId, @RequestParam("editType") Integer editType) throws DEException { + public ExcelFileData uploadFile(@RequestParam("file") MultipartFile file, @RequestParam("id") long datasourceId, @RequestParam("editType") Integer editType) throws DEException { CoreDatasource coreDatasource = null; if (ObjectUtils.isNotEmpty(datasourceId) && 0L != datasourceId) { coreDatasource = dataSourceManage.getCoreDatasource(datasourceId); } ExcelUtils excelUtils = new ExcelUtils(); - ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file); + ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file, String.valueOf(AuthUtils.getUser().getUserId())); if (Objects.equals(editType, append)) { //按照excel sheet 名称匹配，替换：0；追加：1 if (coreDatasource != null) { @@ -833,6 +916,62 @@ public class DatasourceServer implements DatasourceApi { return excelFileData; } + public ExcelFileData loadRemoteFile(RemoteExcelRequest remoteExcelRequest) throws DEException, IOException { + CoreDatasource coreDatasource = null; + if (ObjectUtils.isNotEmpty(remoteExcelRequest.getDatasourceId()) && 0L != remoteExcelRequest.getDatasourceId()) { + coreDatasource = dataSourceManage.getCoreDatasource(remoteExcelRequest.getDatasourceId()); + } + ExcelUtils excelUtils = new ExcelUtils(); + ExcelFileData excelFileData = excelUtils.parseRemoteExcel(remoteExcelRequest, String.valueOf(AuthUtils.getUser().getUserId())); + if (Objects.equals(remoteExcelRequest.getEditType(), append)) { //按照excel sheet 名称匹配，替换：0；追加：1 + if (coreDatasource != null) { + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(transDTO(coreDatasource)); + List datasetTableDTOS = ExcelUtils.getTables(datasourceRequest); + List excelSheetDataList = new ArrayList<>(); + for (ExcelSheetData sheet : excelFileData.getSheets()) { + for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) { + if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName())) { + List newTableFields = sheet.getFields(); + datasourceRequest.setTable(datasetTableDTO.getTableName()); + List oldTableFields = ExcelUtils.getTableFields(datasourceRequest); + if (isEqual(newTableFields, oldTableFields)) { + sheet.setDeTableName(datasetTableDTO.getTableName()); + excelSheetDataList.add(sheet); + } + } + } + } + excelFileData.setSheets(excelSheetDataList); + } + } else { + // 替换 + if (coreDatasource != null) { + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(transDTO(coreDatasource)); + List datasetTableDTOS = ExcelUtils.getTables(datasourceRequest); + for (ExcelSheetData sheet : excelFileData.getSheets()) { + for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) { + if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName())) { + sheet.setDeTableName(datasetTableDTO.getTableName()); + } + } + } + } + } + for (ExcelSheetData sheet : excelFileData.getSheets()) { + for (int i = 0; i < sheet.getFields().size() - 1; i++) { + for (int j = i + 1; j < sheet.getFields().size(); j++) { + if (sheet.getFields().get(i).getName().equalsIgnoreCase(sheet.getFields().get(j).getName())) { + DEException.throwException(sheet.getExcelLabel() + Translator.get("i18n_field_name_repeat") + sheet.getFields().get(i).getName()); + } + } + } + } + return excelFileData; + } + + private boolean isEqual(List newTableFields, List oldTableFields) { if (CollectionUtils.isEmpty(newTableFields) || CollectionUtils.isEmpty(oldTableFields)) { return false; @@ -945,6 +1084,8 @@ public class DatasourceServer implements DatasourceApi { String status = null; if (coreDatasource.getType().startsWith("API")) { status = (String) invokeMethod(coreDatasource.getType(), "checkAPIStatus", DatasourceRequest.class, datasourceRequest); + } else if (coreDatasource.getType().startsWith("Excel")) { + status = ExcelUtils.checkStatus(datasourceRequest); } else { Provider provider = ProviderFactory.getProvider(coreDatasource.getType()); status = provider.checkStatus(datasourceRequest); @@ -1002,7 +1143,12 @@ public class DatasourceServer implements DatasourceApi { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dsId); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(transDTO(coreDatasource)); - List datasetTableDTOS = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); + List datasetTableDTOS = new ArrayList<>(); + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.toString())) { + datasetTableDTOS = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); + } else { + datasetTableDTOS = ExcelUtils.getTables(datasourceRequest); + } for (int i = 0; i < pager.getRecords().size(); i++) { for (int i1 = 0; i1 < datasetTableDTOS.size(); i1++) { if (pager.getRecords().get(i).getTableName().equalsIgnoreCase(datasetTableDTOS.get(i1).getTableName())) { @@ -1208,6 +1354,17 @@ public class DatasourceServer implements DatasourceApi { datasourceDTO.setStatus("Error"); } } + } else { + if (hidePw) { + Provider provider = ProviderFactory.getProvider(datasourceDTO.getType()); + provider.hidePW(datasourceDTO); + } + } + if (datasourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.Excel.toString())) { + datasourceDTO.setFileName(ExcelUtils.getFileName(datasource)); + datasourceDTO.setSize(ExcelUtils.getSize(datasource)); + } + if (datasourceDTO.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.ExcelRemote.name()) || datasourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.toString())) { CoreDatasourceTask coreDatasourceTask = datasourceTaskServer.selectByDSId(datasourceDTO.getId()); TaskDTO taskDTO = new TaskDTO(); BeanUtils.copyBean(taskDTO, coreDatasourceTask); @@ -1216,15 +1373,6 @@ public class DatasourceServer implements DatasourceApi { if (task != null) { datasourceDTO.setLastSyncTime(task.getStartTime()); } - } else { - if (hidePw) { - Provider provider = ProviderFactory.getProvider(datasourceDTO.getType()); - provider.hidePW(datasourceDTO); - } - } - if (datasourceDTO.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.Excel.toString())) { - datasourceDTO.setFileName(ExcelUtils.getFileName(datasource)); - datasourceDTO.setSize(ExcelUtils.getSize(datasource)); } datasourceDTO.setConfiguration(RsaUtils.symmetricEncrypt(datasourceDTO.getConfiguration())); datasourceDTO.setCreator(coreUserManage.getUserName(Long.valueOf(datasourceDTO.getCreateBy()))); @@ -1236,7 +1384,7 @@ public class DatasourceServer implements DatasourceApi { BeanUtils.copyBean(datasourceDTO, coreDatasource); try { checkDatasourceStatus(datasourceDTO); - if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) { calciteProvider.updateDsPoolAfterCheckStatus(datasourceDTO); } } catch (DEException e) { diff --git a/core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java b/core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java index fe1b98677c..9ff6d217b6 100644 --- a/core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java +++ b/core/core-backend/src/main/java/io/dataease/visualization/server/DataVisualizationServer.java @@ -233,7 +233,7 @@ public class DataVisualizationServer implements DataVisualizationApi { // Excel 数据表明映射 if (StringUtils.isNotEmpty(datasourceOld.getConfiguration())) { if (datasourceOld.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) { - dsTableNamesMap.put(datasourceOld.getId(), ExcelUtils.getTableNamesMap(datasourceOld.getConfiguration())); + dsTableNamesMap.put(datasourceOld.getId(), ExcelUtils.getTableNamesMap(datasourceOld.getType(), datasourceOld.getConfiguration())); } else if (datasourceOld.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { dsTableNamesMap.put(datasourceOld.getId(), (Map) datasourceServer.invokeMethod(datasourceOld.getType(), "getTableNamesMap", String.class, datasourceOld.getConfiguration())); } @@ -245,7 +245,7 @@ public class DataVisualizationServer implements DataVisualizationApi { // Excel 数据表明映射 if (StringUtils.isNotEmpty(datasourceNew.getConfiguration())) { if (datasourceNew.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) { - dsTableNamesMap.put(datasourceNew.getId(), ExcelUtils.getTableNamesMap(datasourceNew.getConfiguration())); + dsTableNamesMap.put(datasourceNew.getId(), ExcelUtils.getTableNamesMap(datasourceNew.getType(), datasourceNew.getConfiguration())); } else if (datasourceNew.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { dsTableNamesMap.put(datasourceNew.getId(), (Map) datasourceServer.invokeMethod(datasourceNew.getType(), "getTableNamesMap", String.class, datasourceNew.getConfiguration())); } diff --git a/core/core-frontend/src/api/datasource.ts b/core/core-frontend/src/api/datasource.ts index 4a6530d499..a99c219a46 100644 --- a/core/core-frontend/src/api/datasource.ts +++ b/core/core-frontend/src/api/datasource.ts @@ -175,6 +175,10 @@ export const uploadFile = async (data): Promise => { }) } +export const loadRemoteFile = async (data = {}) => { + return request.post({ url: '/datasource/loadRemoteFile', data }) +} + export const listSyncRecord = (page: number, limit: number, dsId: number | string) => request.post({ url: '/datasource/listSyncRecord/' + dsId + '/' + page + '/' + limit }) diff --git a/core/core-frontend/src/assets/svg/Excel-remote-ds.svg b/core/core-frontend/src/assets/svg/Excel-remote-ds.svg new file mode 100644 index 0000000000..1c9b0b30d6 --- /dev/null +++ b/core/core-frontend/src/assets/svg/Excel-remote-ds.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/core/core-frontend/src/components/icon-group/datasource-list.ts b/core/core-frontend/src/components/icon-group/datasource-list.ts index 2c86934665..fe516e549f 100644 --- a/core/core-frontend/src/components/icon-group/datasource-list.ts +++ b/core/core-frontend/src/components/icon-group/datasource-list.ts @@ -12,6 +12,7 @@ import db2Ds from '@/assets/svg/db2-ds.svg' import redshiftDs from '@/assets/svg/redshift-ds.svg' import APIDs from '@/assets/svg/API-ds.svg' import ExcelDs from '@/assets/svg/Excel-ds.svg' +import ExcelRemoteDs from '@/assets/svg/Excel-remote-ds.svg' import dorisDs from '@/assets/svg/doris-ds.svg' import esDs from '@/assets/svg/es-ds.svg' const iconDatasourceMap = { @@ -29,6 +30,7 @@ const iconDatasourceMap = { redshift: redshiftDs, API: APIDs, Excel: ExcelDs, + ExcelRemote: ExcelRemoteDs, doris: dorisDs, es: esDs } diff --git a/core/core-frontend/src/locales/en.ts b/core/core-frontend/src/locales/en.ts index 0ea86fc194..5b1df71fcb 100644 --- a/core/core-frontend/src/locales/en.ts +++ b/core/core-frontend/src/locales/en.ts @@ -1077,6 +1077,7 @@ export default { has_repeat_field_name: 'Duplicate field name, please modify before selecting', primary_key_change: 'Primary key cannot be changed:', api_field_not_empty: 'Field cannot be empty', + file_not_empty: 'File cannot be empty', success_copy: 'Copy successfully', valid: 'Valid', invalid: 'Invalid', @@ -1138,7 +1139,12 @@ export default { table_id: 'table_id', input_table_id: 'Please select a data table', view_id: 'view_id', - input_view_id: 'Please select a view' + input_view_id: 'Please select a view', + remote_excel_url: 'Remote Excel/CSV Address', + remote_excel_url_placeholder: + 'Please enter the remote Excel/CSV address, for example, ftp://192.168.1.101/files/data.xlsx', + remote_excel_url_empty: 'The remote address cannot be empty', + load_data: 'Load Data' }, chart: { align: 'Alignment', diff --git a/core/core-frontend/src/locales/tw.ts b/core/core-frontend/src/locales/tw.ts index f9968f2a6d..1aaa6fa945 100644 --- a/core/core-frontend/src/locales/tw.ts +++ b/core/core-frontend/src/locales/tw.ts @@ -1041,6 +1041,7 @@ export default { has_repeat_field_name: '欄位名稱重複，請修改後再選擇', primary_key_change: '主鍵不能改變:', api_field_not_empty: '欄位不能為空', + file_not_empty: '文件不能為空', success_copy: '複製成功', primary_key_length: '主鍵必須設置長度: ', valid: '有效', @@ -1104,7 +1105,12 @@ export default { table_id: 'table_id', input_table_id: '請選擇數據表', view_id: 'view_id', - input_view_id: '請選擇視圖' + input_view_id: '請選擇視圖', + remote_excel_url: '遠端 Excel/CSV 地址', + remote_excel_url_placeholder: + '請輸入遠端 Excel/CSV 地址，例如 ftp://192.168.1.101/files/data.xlsx', + remote_excel_url_empty: '遠端地址不能為空', + load_data: '載入資料' }, chart: { align: '對齊方式', diff --git a/core/core-frontend/src/locales/zh-CN.ts b/core/core-frontend/src/locales/zh-CN.ts index 44b6d655ce..326cb3cf09 100644 --- a/core/core-frontend/src/locales/zh-CN.ts +++ b/core/core-frontend/src/locales/zh-CN.ts @@ -1047,6 +1047,7 @@ export default { primary_key_change: '主键不能改变:', primary_key_length: '主键必须设置长度: ', api_field_not_empty: '字段不能为空', + file_not_empty: '文件不能为空', success_copy: '复制成功', valid: '有效', invalid: '无效', @@ -1109,7 +1110,12 @@ export default { table_id: 'table_id', input_table_id: '请选择数据表', view_id: 'view_id', - input_view_id: '请选择视图' + input_view_id: '请选择视图', + remote_excel_url: '远程 Excel/CSV 地址', + remote_excel_url_placeholder: + '请输入远程 Excel/CSV 地址，例如 ftp://192.168.1.101/files/data.xlsx', + remote_excel_url_empty: '远程地址不能为空', + load_data: '加载数据' }, chart: { align: '对齐方式', diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/CreatDsGroup.vue b/core/core-frontend/src/views/visualized/data/datasource/form/CreatDsGroup.vue index 01b11484f2..4ae729fa91 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/CreatDsGroup.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/CreatDsGroup.vue @@ -291,7 +291,7 @@ const saveDataset = () => { request.apiConfiguration = '' checkRepeat(request).then(res => { let method = request.id === '' ? save : update - if (!request.type.startsWith('API')) { + if (!request.type.startsWith('API') || request.type !== 'ExcelRemote') { request.syncSetting = null } if (res) { diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue b/core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue index 33dd6f4c11..e0b55e0eed 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/ExcelDetail.vue @@ -1,6 +1,5 @@ + + + + + + diff --git a/core/core-frontend/src/views/visualized/data/datasource/form/index.vue b/core/core-frontend/src/views/visualized/data/datasource/form/index.vue index 0e5249ebc7..1eede51d2e 100644 --- a/core/core-frontend/src/views/visualized/data/datasource/form/index.vue +++ b/core/core-frontend/src/views/visualized/data/datasource/form/index.vue @@ -9,7 +9,15 @@ import DsTypeList from './DsTypeList.vue' import { useI18n } from '@/hooks/web/useI18n' import EditorDetail from './EditorDetail.vue' import ExcelDetail from './ExcelDetail.vue' -import { save, update, validate, latestUse, isShowFinishPage, checkRepeat } from '@/api/datasource' +import { + save, + update, + validate, + latestUse, + isShowFinishPage, + checkRepeat, + loadRemoteFile +} from '@/api/datasource' import { Base64 } from 'js-base64' import type { Param } from './ExcelDetail.vue' import type { Configuration, ApiConfiguration, SyncSetting } from './option' @@ -23,6 +31,7 @@ import { useCache } from '@/hooks/web/useCache' import Icon from '@/components/icon-custom/src/Icon.vue' import { XpackComponent, PluginComponent } from '@/components/plugin' import { iconDatasourceMap } from '@/components/icon-group/datasource-list' +import ExcelRemoteDetail from '@/views/visualized/data/datasource/form/ExcelRemoteDetail.vue' interface Node { name: string @@ -73,6 +82,7 @@ const activeStep = ref(0) const detail = ref() const xpack = ref() const excel = ref() +const excelRemote = ref() const latestUseTypes = ref([]) const currentType = ref('OLTP') const filterText = ref('') @@ -99,6 +109,7 @@ const selectDsType = (type: string) => { methodName: 'initForm', args: type }) + excelRemote?.value?.initForm(type) if (!dsTree.value) return currentTypeList.value .map(ele => ele.dbList) @@ -204,7 +215,11 @@ const activeApiStep = ref(0) const setNextStep = () => { activeApiStep.value = activeStep.value + 1 - if (currentDsType.value.includes('API') && activeStep.value === 1) return + if ( + (currentDsType.value.includes('API') || currentDsType.value === 'ExcelRemote') && + activeStep.value === 1 + ) + return activeStep.value = activeStep.value + 1 } @@ -223,6 +238,14 @@ const next = () => { return } + if (currentDsType.value.includes('ExcelRemote') && activeStep.value !== 2) { + const validate = excelRemote.value.validateExcel() + if (validate) { + setNextStep() + } + return + } + if (currentDsType.value.includes('API') && activeStep.value !== 2) { const validateFrom = detail.value.submitForm() validateFrom(val => { @@ -237,7 +260,15 @@ const next = () => { } const complete = (params, successCb, finallyCb) => { - excel.value.saveExcelDs( + excel?.value?.saveExcelDs( + params, + () => { + pid.value = params.pid + successCb() + }, + finallyCb + ) + excelRemote?.value?.saveExcelDs( params, () => { pid.value = params.pid @@ -293,7 +324,8 @@ emitter.on('showFinishPage', handleShowFinishPage) const prev = () => { if ( - (currentDsType.value.includes('API') && activeApiStep.value === 1) || + ((currentDsType.value.includes('API') || currentDsType.value === 'ExcelRemote') && + activeApiStep.value === 1) || activeStep.value === 1 ) { ElMessageBox.confirm(t('data_source.the_previous_step'), { @@ -310,7 +342,10 @@ const prev = () => { } const prevConfirm = () => { - if (currentDsType.value.includes('API') && activeApiStep.value === 2) { + if ( + (currentDsType.value.includes('API') || currentDsType.value === 'ExcelRemote') && + activeApiStep.value === 2 + ) { activeApiStep.value = 1 activeStep.value = 1 return @@ -369,7 +404,10 @@ const validateDS = () => { args: [{ eventName: 'validateDs', args: request }] }) } else { - const validateFrom = detail?.value?.submitForm() + let validateFrom = detail?.value?.submitForm() + if (excelRemote?.value?.submitForm()) { + validateFrom = excelRemote?.value?.submitForm() + } validateFrom(val => { if (val) { doValidateDs(request) @@ -380,9 +418,23 @@ const validateDS = () => { const doValidateDs = request => { dsLoading.value = true + if (currentDsType.value === 'ExcelRemote') { + let excelRequest = JSON.parse(JSON.stringify(form2.configuration)) + excelRequest.datasourceId = form2.id || 0 + excelRequest.editType = form2.editType + return loadRemoteFile(excelRequest) + .then(() => { + ElMessage.success(t('datasource.validate_success')) + dsLoading.value = false + }) + .catch(() => { + ElMessage.error(t('data_source.verification_failed')) + dsLoading.value = false + }) + } validate(request) .then(res => { - if (res.data.type === 'API') { + if (res.data.type.includes('API')) { let error = 0 const status = JSON.parse(res.data.status) as Array<{ status: string; name: string }> for (let i = 0; i < status.length; i++) { @@ -435,13 +487,13 @@ const saveDS = () => { configuration: string apiConfiguration: string } + if (currentDsType.value === 'Excel') { excel.value.uploadStatus(false) if (!excel.value.sheetFile?.name) { excel.value.uploadStatus(true) return } - const validate = excel.value.submitForm() validate(val => { if (val) { @@ -452,7 +504,25 @@ const saveDS = () => { } } }) + return + } + if (currentDsType.value === 'ExcelRemote') { + const validate = excelRemote.value.submitForm() + validate(val => { + if (val) { + const validateApi = excelRemote?.value?.submitSyncSettingForm() + validateApi(v => { + if (v) { + if (editDs.value) { + complete(null, null, null) + } else { + creatDsFolder.value.createInit('datasource', { id: pid.value }, '', form2.name) + } + } + }) + } + }) return } else if (currentDsType.value.includes('API')) { for (let i = 0; i < request.apiConfiguration.length; i++) { @@ -569,16 +639,18 @@ const defaultForm = { paramsConfiguration: [], enableDataFill: false } -const form = reactive
(cloneDeep(defaultForm)) -const origin = reactive(cloneDeep(defaultForm)) const defaultForm2 = { type: '', id: '0', editType: 0, name: '', - creator: '' + creator: '', + configuration: {} } +const origin = reactive(cloneDeep(defaultForm)) +const form = reactive(cloneDeep(defaultForm)) const form2 = reactive(cloneDeep(defaultForm2)) + const visible = ref(false) const editDs = ref(false) const pid = ref('0') @@ -611,7 +683,7 @@ const init = (nodeInfo: Form | Param, id?: string, res?: object, supportSetKey: showFinishPage.value = false if (!!nodeInfo) { - if (nodeInfo.type == 'Excel') { + if (nodeInfo.type.startsWith('Excel')) { Object.assign(form2, cloneDeep(nodeInfo)) } else { Object.assign(form, cloneDeep(nodeInfo)) @@ -833,7 +905,7 @@ defineExpose({ v-if=" activeStep !== 0 && currentDsType && - currentDsType !== 'Excel' && + !currentDsType.startsWith('Excel') && visible && (!isPlugin || currentDsType.startsWith('API')) " @@ -863,6 +935,15 @@ defineExpose({ :param="form2" > +
@@ -883,8 +964,11 @@ defineExpose({ > { } const editDatasource = (editType?: number) => { - if (nodeInfo.type === 'Excel') { + if (nodeInfo.type.startsWith('Excel')) { nodeInfo.editType = editType } return getById(nodeInfo.id).then(res => { @@ -940,7 +940,7 @@ const handleClick = (tabName: TabPaneName) => { switch (tabName) { case 'config': tableData.value = [] - if (nodeInfo.type === 'Excel') { + if (nodeInfo.type.startsWith('Excel')) { listDatasourceTables({ datasourceId: nodeInfo.id }).then(res => { tabList.value = res.data.map(ele => { const { name, tableName } = ele @@ -1485,7 +1485,17 @@ const getMenuList = (val: boolean) => { - + + + {{ nodeInfo.configuration.url }} + + + + + + + + {{ nodeInfo.description }} @@ -1495,6 +1505,7 @@ const getMenuList = (val: boolean) => { v-if=" !['Excel', 'es'].includes(nodeInfo.type) && !nodeInfo.type.startsWith('API') && + !nodeInfo.type.startsWith('Excel') && nodeInfo.weight >= 7 " > @@ -1698,42 +1709,7 @@ const getMenuList = (val: boolean) => { - - - - {{ t('dataset.update_records') }} - - - @@ -1767,6 +1743,44 @@ const getMenuList = (val: boolean) => {
+ + + + + {{ t('dataset.update_records') }} + +