public/dataease
1
0
Fork 0
mirror of https://github.com/dataease/dataease.git synced 2026-05-24 06:18:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13574 from dataease/pr@dev-v2@fixds

Browse Source 
Pr@dev v2@fixds
This commit is contained in:
xuwei-fit2cloud
2024-11-26 17:51:05 +08:00
committed by GitHub
parent 00c5557645 52a52dd581
commit 44951bcfaf
8 changed files with 238 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
core/core-backend/src/main/java/io/dataease/datasource/manage/DatasourceSyncManage.java
View File

@@ -22,6 +22,7 @@ import io.dataease.extensions.datasource.provider.Provider;
import io.dataease.job.schedule.ExtractDataJob;
import io.dataease.job.schedule.ScheduleManager;
import io.dataease.utils.BeanUtils;
import io.dataease.utils.JsonUtil;
import io.dataease.utils.LogUtil;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.StringUtils;
@@ -30,6 +31,7 @@ import org.quartz.JobKey;
import org.quartz.TriggerKey;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
@@ -245,17 +247,7 @@ public class DatasourceSyncManage {
private void extractExcelData(DatasourceRequest datasourceRequest, DatasourceServer.UpdateType extractType, List<TableField> tableFields) throws Exception {
ExcelUtils excelUtils = new ExcelUtils();
List<String[]> dataList = excelUtils.fetchDataList(datasourceRequest);
String engineTableName;
switch (extractType) {
case all_scope:
engineTableName = TableUtils.tmpName(TableUtils.tableName(datasourceRequest.getTable()));
break;
default:
engineTableName = TableUtils.tableName(datasourceRequest.getTable());
break;
}
CoreDeEngine engine = engineManage.info();
EngineRequest engineRequest = new EngineRequest();
engineRequest.setEngine(engine);
EngineProvider engineProvider = ProviderUtil.getEngineProvider(engine.getType());
@@ -267,7 +259,7 @@ public class DatasourceSyncManage {
totalPage = dataList.size() / pageNumber;
}
for (int page = 1; page <= totalPage; page++) {
engineRequest.setQuery(engineProvider.insertSql(engineTableName, extractType, dataList, page, pageNumber, tableFields));
engineRequest.setQuery(engineProvider.insertSql(datasourceRequest.getTable(), extractType, dataList, page, pageNumber, tableFields));
calciteProvider.exec(engineRequest);
}
}

4
core/core-backend/src/main/java/io/dataease/datasource/provider/ExcelUtils.java
View File

@@ -335,9 +335,6 @@ public class ExcelUtils {
Double d = Double.valueOf(value);
double eps = 1e-10;
if (d - Math.floor(d) < eps) {
if (value.length() > 1 && value.startsWith("0")) {
return "TEXT";
}
return "LONG";
} else {
return "DOUBLE";
@@ -440,6 +437,7 @@ public class ExcelUtils {
tableFiled.setFieldType(null);
tableFiled.setName(s);
tableFiled.setOriginName(s);
tableFiled.setChecked(true);
fields.add(tableFiled);
}
List<String[]> data = new ArrayList<>(noModelDataListener.getData());

17
core/core-backend/src/main/java/io/dataease/datasource/provider/H2EngineProvider.java
View File

@@ -40,12 +40,16 @@ public class H2EngineProvider extends EngineProvider {
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) {
String[] strings1 = new String[strings.length];
int length = 0;
String[] strings1 = new String[tableFields.stream().filter(TableField::isChecked).toList().size()];
for (int i = 0; i < strings.length; i++) {
if (StringUtils.isEmpty(strings[i])) {
strings1[i] = null;
} else {
strings1[i] = strings[i].replace("'", "\\'");
if (tableFields.get(i).isChecked()) {
if (StringUtils.isEmpty(strings[i])) {
strings1[length] = null;
} else {
strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
}
length++;
}
}
values.append("('").append(String.join("','", Arrays.asList(strings1)))
@@ -81,6 +85,9 @@ public class H2EngineProvider extends EngineProvider {
StringBuilder columnFields = new StringBuilder("`");
StringBuilder key = new StringBuilder();
for (TableField tableField : tableFields) {
if (!tableField.isChecked()) {
continue;
}
if (tableField.isPrimaryKey()) {
key.append("`").append(tableField.getName()).append("`, ");
}

21
core/core-backend/src/main/java/io/dataease/datasource/provider/MysqlEngineProvider.java
View File

@@ -47,19 +47,23 @@ public class MysqlEngineProvider extends EngineProvider {
Integer realSize = page * pageNumber < dataList.size() ? page * pageNumber : dataList.size();
for (String[] strings : dataList.subList((page - 1) * pageNumber, realSize)) {
String[] strings1 = new String[strings.length];
int length = 0;
String[] strings1 = new String[tableFields.stream().filter(TableField::isChecked).toList().size()];
for (int i = 0; i < strings.length; i++) {
if (StringUtils.isEmpty(strings[i])) {
strings1[i] = null;
} else {
strings1[i] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
if (tableFields.get(i).isChecked()) {
if (StringUtils.isEmpty(strings[i])) {
strings1[length] = null;
} else {
strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
}
length++;
}
}
values.append("('").append(String.join("','", Arrays.asList(strings1)))
.append("'),");
}
List<TableField> keys = tableFields.stream().filter(TableField::isPrimaryKey).toList();
List<TableField> notKeys = tableFields.stream().filter(tableField -> !tableField.isPrimaryKey()).toList();
List<TableField> keys = tableFields.stream().filter(tableField -> tableField.isPrimaryKey() && tableField.isChecked()).toList();
List<TableField> notKeys = tableFields.stream().filter(tableField -> tableField.isChecked() && !tableField.isPrimaryKey()).toList();
String insetSql = (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null");
if (CollectionUtils.isNotEmpty(keys) && extractType.equals(DatasourceServer.UpdateType.add_scope)) {
insetSql = insetSql + " ON DUPLICATE KEY UPDATE ";
@@ -101,6 +105,9 @@ public class MysqlEngineProvider extends EngineProvider {
StringBuilder columnFields = new StringBuilder("`");
StringBuilder key = new StringBuilder();
for (TableField tableField : tableFields) {
if (!tableField.isChecked()) {
continue;
}
if (tableField.isPrimaryKey()) {
key.append("`").append(tableField.getName()).append("`, ");
}

80
core/core-backend/src/main/java/io/dataease/datasource/server/DatasourceServer.java
View File

@@ -408,14 +408,13 @@ public class DatasourceServer implements DatasourceApi {
requestDatasource.setEnableDataFill(null);
List<String> sourceTables = ExcelUtils.getTables(sourceTableRequest).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList());
List<String> tables = ExcelUtils.getTables(datasourceRequest).stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList());
if (dataSourceDTO.getEditType() == 0) {
if (Objects.equals(dataSourceDTO.getEditType(), replace)) {
toCreateTables = tables;
toDeleteTables = sourceTables.stream().filter(s -> tables.contains(s)).collect(Collectors.toList());
for (String deleteTable : toDeleteTables) {
try {
datasourceSyncManage.dropEngineTable(deleteTable);
} catch (Exception e) {
DEException.throwException("Failed to drop table " + deleteTable + ", " + e.getMessage());
} catch (Exception ignore) {
}
}
for (String toCreateTable : toCreateTables) {
@@ -426,12 +425,16 @@ public class DatasourceServer implements DatasourceApi {
DEException.throwException("Failed to create table " + toCreateTable + ", " + e.getMessage());
}
}
datasourceSyncManage.extractExcelData(requestDatasource, "all_scope");
commonThreadPool.addTask(() -> {
datasourceSyncManage.extractExcelData(requestDatasource, "all_scope");
});
dataSourceManage.checkName(dataSourceDTO);
ExcelUtils.mergeSheets(requestDatasource, sourceData);
dataSourceManage.innerEdit(requestDatasource);
} else {
datasourceSyncManage.extractExcelData(requestDatasource, "add_scope");
commonThreadPool.addTask(() -> {
datasourceSyncManage.extractExcelData(requestDatasource, "add_scope");
});
dataSourceManage.checkName(dataSourceDTO);
dataSourceManage.innerEdit(requestDatasource);
}
@@ -767,6 +770,7 @@ public class DatasourceServer implements DatasourceApi {
ExcelUtils excelUtils = new ExcelUtils();
ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file);
if (Objects.equals(editType, append)) { //按照excel sheet 名称匹配，替换：0；追加：1
if (coreDatasource != null) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
@@ -776,15 +780,9 @@ public class DatasourceServer implements DatasourceApi {
for (ExcelSheetData sheet : excelFileData.getSheets()) {
for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) {
if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName()) || isCsv(file.getOriginalFilename())) {
List<TableField> newTableFields = deepCopy(sheet.getFields());
newTableFields.sort((o1, o2) -> {
return o1.getName().compareTo(o2.getName());
});
List<TableField> newTableFields = sheet.getFields();
datasourceRequest.setTable(datasetTableDTO.getTableName());
List<TableField> oldTableFields = ExcelUtils.getTableFields(datasourceRequest);
oldTableFields.sort((o1, o2) -> {
return o1.getName().compareTo(o2.getName());
});
if (isEqual(newTableFields, oldTableFields)) {
sheet.setDeTableName(datasetTableDTO.getTableName());
excelSheetDataList.add(sheet);
@@ -798,20 +796,29 @@ public class DatasourceServer implements DatasourceApi {
excelFileData.setSheets(excelSheetDataList);
}
} else {
// 替换
if (coreDatasource != null) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> datasetTableDTOS = ExcelUtils.getTables(datasourceRequest);
for (ExcelSheetData sheet : excelFileData.getSheets()) {
boolean find = false;
for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) {
if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName()) || isCsv(file.getOriginalFilename())) {
find = true;
sheet.setDeTableName(datasetTableDTO.getTableName());
datasourceRequest.setTable(datasetTableDTO.getTableName());
List<TableField> oldTableFields = ExcelUtils.getTableFields(datasourceRequest);
mergeFields(sheet.getFields(), oldTableFields);
}
}
if (!find) {
sheet.setNewSheet(true);
}
}
}
}
for (ExcelSheetData sheet : excelFileData.getSheets()) {
for (int i = 0; i < sheet.getFields().size() - 1; i++) {
for (int j = i + 1; j < sheet.getFields().size(); j++) {
@@ -825,31 +832,42 @@ public class DatasourceServer implements DatasourceApi {
}
private boolean isEqual(List<TableField> newTableFields, List<TableField> oldTableFields) {
boolean isEqual = true;
if (CollectionUtils.isEmpty(newTableFields) || CollectionUtils.isEmpty(oldTableFields)) {
isEqual = false;
return false;
}
for (int i = 0; i < newTableFields.size(); i++) {
if (!newTableFields.get(i).getName().equals(oldTableFields.get(i).getName())) {
isEqual = false;
break;
newTableFields.forEach(tableField -> tableField.setChecked(false));
for (TableField oldField : oldTableFields) {
if (!oldField.isChecked()) {
continue;
}
if (!newTableFields.get(i).getFieldType().equals(oldTableFields.get(i).getFieldType())) {
if (oldTableFields.get(i).getFieldType().equals("TEXT")) {
continue;
boolean find = false;
for (TableField newField : newTableFields) {
if (oldField.getName().equals(newField.getName())) {
find = true;
newField.setChecked(oldField.isChecked());
newField.setPrimaryKey(oldField.isPrimaryKey());
newField.setLength(oldField.getLength());
break;
}
if (oldTableFields.get(i).getFieldType().equals("DOUBLE")) {
if (newTableFields.get(i).getFieldType().equals("LONG")) {
continue;
}
}
isEqual = false;
break;
}
if (!find) {
return find;
}
}
return true;
}
return isEqual;
private void mergeFields(List<TableField> oldFields, List<TableField> newFields) {
oldFields.forEach(tableField -> tableField.setChecked(false));
for (TableField newField : newFields) {
for (TableField oldField : oldFields) {
if (oldField.getName().equals(newField.getName())) {
newField.setChecked(oldField.isChecked());
newField.setPrimaryKey(oldField.isPrimaryKey());
newField.setLength(oldField.getLength());
}
}
}
}
private boolean isCsv(String fileName) {