feat(系统管理-同步管理):

1.添加目标数据源同步到数据准备
2.UI调整

sdk/api/api-sync/pom.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>api</artifactId>
+        <groupId>io.dataease</groupId>
+        <version>2.1.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>api-sync</artifactId>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+</project>

sdk/api/api-sync/src/main/java/io/dataease/api/sync/datasource/api/SyncDatasourceApi.java

@@ -0,0 +1,80 @@
+package io.dataease.api.sync.datasource.api;
+
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import io.dataease.api.sync.datasource.dto.DBTableDTO;
+import io.dataease.api.sync.datasource.dto.GetDatasourceRequest;
+import io.dataease.api.sync.datasource.dto.SyncDatasourceDTO;
+import io.dataease.api.sync.datasource.vo.SyncDatasourceVO;
+import io.dataease.auth.DeApiPath;
+import io.dataease.auth.DePermit;
+import io.dataease.exception.DEException;
+import io.dataease.request.BaseGridRequest;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import java.util.List;
+import java.util.Map;
+
+import static io.dataease.constant.AuthResourceEnum.SYNC_DATASOURCE;
+
+/**
+ * @author fit2cloud
+ * @date 2023/11/20 10:14
+ **/
+@DeApiPath(value = "/sync/datasource", rt = SYNC_DATASOURCE)
+public interface SyncDatasourceApi {
+
+    @DePermit("m:read")
+    @PostMapping("/source/pager/{goPage}/{pageSize}")
+    IPage<SyncDatasourceVO> sourcePager(@PathVariable("goPage") int goPage, @PathVariable("pageSize") int pageSize, @RequestBody BaseGridRequest request);
+
+    @DePermit("m:read")
+    @PostMapping("/target/pager/{goPage}/{pageSize}")
+    IPage<SyncDatasourceVO> targetPager(@PathVariable("goPage") int goPage, @PathVariable("pageSize") int pageSize, @RequestBody BaseGridRequest request);
+
+    @PostMapping("/save")
+    void save(@RequestBody SyncDatasourceDTO dataSourceDTO) throws DEException;
+
+    @PostMapping("/update")
+    void update(@RequestBody SyncDatasourceDTO dataSourceDTO) throws DEException;
+
+    @PostMapping("/delete/{datasourceId}")
+    void delete(@PathVariable("datasourceId") String datasourceId) throws DEException;
+
+    @GetMapping("/types")
+    Object datasourceTypes() throws DEException;
+
+    @PostMapping("/validate")
+    String validate(@RequestBody SyncDatasourceDTO dataSourceDTO) throws DEException;
+
+    @PostMapping("/getSchema")
+    List<String> getSchema(@RequestBody SyncDatasourceDTO dataSourceDTO) throws DEException;
+
+    @DePermit({"#p0+':manage'"})
+    @GetMapping("/validate/{datasourceId}")
+    SyncDatasourceDTO validate(@PathVariable("datasourceId") String datasourceId) throws DEException;
+
+    @PostMapping("/latestUse")
+    List<String> latestUse();
+
+    @GetMapping("/get/{datasourceId}")
+    SyncDatasourceDTO get(@PathVariable("datasourceId") String datasourceId) throws DEException;
+
+    @PostMapping("/batchDel")
+    void batchDel(@RequestBody List<String> ids) throws DEException;
+
+    @PostMapping("/fields")
+    Map<String, Object> getFields(@RequestBody GetDatasourceRequest getDsRequest) throws DEException ;
+
+    @GetMapping("/list/{type}")
+    List<SyncDatasourceDTO> listByType(@PathVariable("type") String type) throws DEException;
+
+    @GetMapping("/table/list/{dsId}")
+    List<DBTableDTO> getTableList(@PathVariable("dsId") String dsId) throws DEException;
+
+
+
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/datasource/dto/DBTableDTO.java

@@ -0,0 +1,18 @@
+package io.dataease.api.sync.datasource.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * @Author gin
+ * @Date 2021/4/30 10:57 上午
+ */
+@Getter
+@Setter
+public class DBTableDTO {
+    private String datasourceId;
+    private String name;
+    private String remark;
+    private boolean enableCheck;
+    private String datasetPath;
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/datasource/dto/GetDatasourceRequest.java

@@ -0,0 +1,25 @@
+package io.dataease.api.sync.datasource.dto;
+
+import lombok.Data;
+
+@Data
+public class GetDatasourceRequest extends SyncDatasourceDTO {
+
+    /**
+     * 查询sql
+     */
+    private String query;
+    /**
+     * 表名
+     */
+    private String table;
+    /**
+     * 表格的抽取数据方式
+     */
+    private boolean tableExtract;
+    /**
+     * 不为空时，获取源数据库表字段，将转换为doris的数据类型
+     */
+    private String targetDbType;
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/datasource/dto/SyncDatasourceDTO.java

@@ -0,0 +1,61 @@
+package io.dataease.api.sync.datasource.dto;
+
+import lombok.Data;
+
+/**
+ * @author fit2cloud
+ * @date 2023/11/20 11:14
+ **/
+@Data
+public class SyncDatasourceDTO {
+
+    /**
+     * ID
+     */
+    private String id;
+
+    /**
+     * 名称
+     */
+    private String name;
+
+    /**
+     * 描述
+     */
+    private String desc;
+
+    /**
+     * 类型
+     */
+    private String type;
+
+    /**
+     * 详细信息
+     */
+    private String configuration;
+
+    /**
+     * Create timestamp
+     */
+    private Long createTime;
+
+    /**
+     * Update timestamp
+     */
+    private Long updateTime;
+
+    /**
+     * 创建人ID
+     */
+    private Long createBy;
+    private String createByUserName;
+
+    /**
+     * 状态
+     */
+    private String status;
+
+    private String statusRemark;
+
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/datasource/vo/SyncDatasourceVO.java

@@ -0,0 +1,57 @@
+package io.dataease.api.sync.datasource.vo;
+
+import lombok.Data;
+
+/**
+ * @author fit2cloud
+ * @date 2023/11/20 11:18
+ **/
+@Data
+public class SyncDatasourceVO {
+    /**
+     * ID
+     */
+    private String id;
+
+    /**
+     * 名称
+     */
+    private String name;
+
+    /**
+     * 描述
+     */
+    private String desc;
+
+    /**
+     * 类型
+     */
+    private String type;
+
+    /**
+     * 详细信息
+     */
+    private String configuration;
+
+    /**
+     * Create timestamp
+     */
+    private Long createTime;
+
+    /**
+     * Update timestamp
+     */
+    private Long updateTime;
+
+    /**
+     * 创建人
+     */
+    private Long createBy;
+    private String createByName;
+
+    /**
+     * 状态
+     */
+    private String status;
+    private String statusRemark;
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/summary/api/SummaryApi.java

@@ -0,0 +1,26 @@
+package io.dataease.api.sync.summary.api;
+
+import io.dataease.auth.DeApiPath;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+
+import java.util.Map;
+
+import static io.dataease.constant.AuthResourceEnum.SUMMARY;
+
+/**
+ * @author fit2cloud
+ * @date 2023/12/4 12:43
+ **/
+@DeApiPath(value = "/sync/summary", rt = SUMMARY)
+public interface SummaryApi {
+
+    @GetMapping("/resourceCount")
+    Map<String, Long> resourceCount();
+
+    @PostMapping("/logChartData")
+    Map<String, Object> logChartData(@RequestBody String executeTaskLogDate);
+
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/api/TaskApi.java

@@ -0,0 +1,52 @@
+package io.dataease.api.sync.task.api;
+
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import io.dataease.api.sync.task.dto.TaskInfoDTO;
+import io.dataease.api.sync.task.vo.TaskInfoVO;
+import io.dataease.auth.DeApiPath;
+import io.dataease.exception.DEException;
+import io.dataease.request.BaseGridRequest;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.List;
+
+import static io.dataease.constant.AuthResourceEnum.TASK;
+
+/**
+ * @author fit2cloud
+ * @date 2023/11/20 10:14
+ **/
+@DeApiPath(value = "/sync/task", rt = TASK)
+public interface TaskApi {
+
+    @PostMapping("/pager/{goPage}/{pageSize}")
+    IPage<TaskInfoVO> pager(@PathVariable("goPage") int goPage, @PathVariable("pageSize") int pageSize, @RequestBody BaseGridRequest request);
+
+    @PostMapping("/add")
+    void add(@RequestBody TaskInfoDTO jobInfo) throws DEException;
+
+    @PostMapping("/update")
+    void update(@RequestBody TaskInfoDTO jobInfo) throws DEException;
+
+    @DeleteMapping("/remove/{id}")
+    void remove(@PathVariable(value = "id") String id) throws DEException;
+
+    @GetMapping("start/{id}")
+    void startJob(@PathVariable(value = "id") String id) throws DEException;
+
+    @GetMapping("stop/{id}")
+    void stopJob(@PathVariable(value = "id") String id) throws DEException;
+
+    @GetMapping("/get/{id}")
+    TaskInfoVO getOneById(@PathVariable(value = "id") String id) throws DEException;
+
+    @GetMapping("/execute/{id}")
+    void execute(@PathVariable(value = "id") String id) throws DEException;
+
+    @PostMapping("/batch/del")
+    void batchDelete(@RequestBody List<String> ids) throws DEException;
+
+    @GetMapping("/count")
+    Long count() throws DEException;
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/api/TaskLogApi.java

@@ -0,0 +1,39 @@
+package io.dataease.api.sync.task.api;
+
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import io.dataease.api.sync.task.vo.LogResultVO;
+import io.dataease.api.sync.task.vo.TaskLogVO;
+import io.dataease.auth.DeApiPath;
+import io.dataease.request.BaseGridRequest;
+import org.springframework.web.bind.annotation.*;
+
+import static io.dataease.constant.AuthResourceEnum.TASK_LOG;
+
+/**
+ * @author fit2cloud
+ * @date 2023/12/4 12:43
+ **/
+@DeApiPath(value = "/sync/task/log", rt = TASK_LOG)
+public interface TaskLogApi {
+    @PostMapping("/pager/{goPage}/{pageSize}")
+    IPage<TaskLogVO> pager(@PathVariable("goPage") int goPage, @PathVariable("pageSize") int pageSize, @RequestBody BaseGridRequest request);
+
+    @GetMapping("/detail/{logId}/{fromLineNum}")
+    LogResultVO logDetail(@PathVariable("logId") String logId, @PathVariable("fromLineNum") int fromLineNum);
+
+    @PostMapping("/save")
+    void saveLog(@RequestBody TaskLogVO logDetail);
+
+    @PostMapping("/update")
+    void updateLog(@RequestBody TaskLogVO logDetail);
+
+    @DeleteMapping("/deleteByJobId/{jobId}")
+    void deleteByJobId(@PathVariable("jobId") String jobId);
+
+    @DeleteMapping("/delete/{logId}")
+    void deleteById(@PathVariable("logId") String logId);
+
+    @PostMapping("/clear")
+    void clearJobLog(@RequestBody TaskLogVO taskLogVO);
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/dto/Source.java

@@ -0,0 +1,23 @@
+package io.dataease.api.sync.task.dto;
+
+import io.dataease.api.sync.datasource.dto.SyncDatasourceDTO;
+import lombok.Data;
+
+import java.util.List;
+
+
+/**
+ * @author fit2cloud
+ * @date 2023/8/10 16:38
+ **/
+@Data
+public class Source {
+    private String type;
+    private String query;
+    private String tables;
+    private SyncDatasourceDTO datasource;
+    private String datasourceId;
+    private String tableExtract;
+    private List<TableField> fieldList;
+    private String incrementField;
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/dto/TableField.java

@@ -0,0 +1,22 @@
+package io.dataease.api.sync.task.dto;
+
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * @author fit2cloud
+ */
+@Setter
+@Getter
+public class TableField {
+    private String fieldSource;
+    private String fieldName;
+    private String remarks;
+    private String fieldType;
+    private int fieldSize;
+    private boolean fieldPk;
+    private boolean fieldIndex;
+    private int accuracy;
+    private Object defaultValue;
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/dto/Target.java

@@ -0,0 +1,21 @@
+package io.dataease.api.sync.task.dto;
+
+import io.dataease.api.sync.datasource.dto.SyncDatasourceDTO;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @author fit2cloud
+ * @date 2023/8/10 16:39
+ **/
+@Data
+public class Target {
+    private String type;
+    private String createTable;
+    private List<TableField> fieldList;
+    private String tableName;
+    private SyncDatasourceDTO datasource;
+    private String datasourceId;
+    private String targetProperty;
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/dto/TaskInfoDTO.java

@@ -0,0 +1,112 @@
+package io.dataease.api.sync.task.dto;
+
+import lombok.Data;
+
+import java.time.LocalDateTime;
+
+/**
+ * @author fit2cloud
+ * @date 2023/11/28 17:17
+ **/
+@Data
+public class TaskInfoDTO {
+    private String id;
+
+    private String name;
+
+    /**
+     * 任务类型KEY
+     */
+    private String jobKey;
+
+    private String desc;
+
+    private LocalDateTime createTime;
+
+    private LocalDateTime modifyTime;
+
+    /**
+     * 创建人
+     */
+    private Long createBy;
+
+    /**
+     * 修改人
+     */
+    private Long modifyBy;
+
+    /**
+     * 任务参数
+     */
+    private String parameter;
+
+    /**
+     * 扩展参数
+     */
+    private String extParameter;
+
+    /**
+     * 当前任务状态
+     * unexecuted未执行 currentTime<startTime
+     * waiting等待执行 stopTime>=currentTime>=startTime,status=1
+     * suspend暂停 stopTime>=currentTime>=startTime,status=0
+     * done执行结束 currentTime>stopTime
+     * running执行中,通过当前任务的日志状态判断，如果有日志在执行中
+     */
+    private String status;
+
+    /**
+     * 删除标识
+     */
+    private Boolean deleted;
+
+    /**
+     * 任务执行超时时间
+     */
+    private Long executorTimeout;
+
+    /**
+     * 任务执行失败重试次数
+     */
+    private Long executorFailRetryCount;
+
+    /**
+     * 上次调度时间
+     */
+    private Long triggerLastTime;
+
+    /**
+     * 下次次调度时间
+     */
+    private Long triggerNextTime;
+
+    /**
+     * 调度类型,NONE,CRON,FIX_RATE,FIX_DELAY
+     */
+    private String schedulerType;
+
+    /**
+     * 调度配置，取决于调度类型
+     */
+    private String schedulerConf;
+
+    /**
+     * 开始时间
+     */
+    private String startTime;
+
+    /**
+     * 结束时间
+     */
+    private String stopTime;
+
+
+    /**
+     * 源数据源信息
+     */
+    private Source source;
+    /**
+     * 目标数据源信息
+     */
+    private Target target;
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/vo/LogResultVO.java

@@ -0,0 +1,46 @@
+package io.dataease.api.sync.task.vo;
+
+import lombok.Data;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.io.Serial;
+import java.io.Serializable;
+
+/**
+ * 日志返回结果
+ *
+ * @author fit2cloud
+ */
+@Getter
+@Setter
+@NoArgsConstructor
+public class LogResultVO implements Serializable{
+
+    /**
+     * 日志开始行号
+     */
+    private int fromLineNum;
+    /**
+     * 日志结束行号
+     */
+    private int toLineNum;
+    /**
+     * 日志内容
+     */
+    private String logContent;
+    /**
+     * 是否是最后一行
+     */
+    private boolean isEnd;
+
+    public LogResultVO(int fromLineNum, int toLineNum, String logContent, boolean isEnd) {
+        this.fromLineNum = fromLineNum;
+        this.toLineNum = toLineNum;
+        this.logContent = logContent;
+        this.isEnd = isEnd;
+    }
+
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/vo/TaskInfoVO.java

@@ -0,0 +1,132 @@
+package io.dataease.api.sync.task.vo;
+
+import io.dataease.api.sync.task.dto.Source;
+import io.dataease.api.sync.task.dto.Target;
+import lombok.Data;
+
+import java.time.LocalDateTime;
+
+/**
+ * @author fit2cloud
+ * @date 2023/11/28 17:15
+ **/
+@Data
+public class TaskInfoVO {
+
+    private String id;
+
+    private String name;
+
+    /**
+     * 任务类型KEY
+     */
+    private String jobKey;
+
+    private String desc;
+
+    private LocalDateTime createTime;
+
+    private LocalDateTime modifyTime;
+
+    /**
+     * 创建人
+     */
+    private Long createBy;;
+    /**
+     * 创建人
+     */
+    private String userName;
+
+    /**
+     * 任务参数
+     */
+    private String parameter;
+
+    /**
+     * 扩展参数
+     */
+    private String extParameter;
+
+    /**
+     * 任务状态
+     * unexecuted未执行 currentTime<startTime
+     * waiting等待执行 stopTime>=currentTime>=startTime,status=1
+     * suspend暂停 stopTime>=currentTime>=startTime,status=0
+     * done执行结束 currentTime>stopTime
+     * running执行中,通过当前任务的日志状态判断，如果有日志在执行中
+     */
+    private String status;
+
+    /**
+     * 删除标识
+     */
+    private Boolean deleted;
+
+    /**
+     * 任务执行超时时间
+     */
+    private Long executorTimeout;
+
+    /**
+     * 任务执行失败重试次数
+     */
+    private Long executorFailRetryCount;
+
+    /**
+     * 上次调度时间
+     */
+    private Long triggerLastTime;
+
+    /**
+     * 下次次调度时间
+     */
+    private Long triggerNextTime;
+
+    /**
+     * 调度类型,NONE,CRON,FIX_RATE,FIX_DELAY
+     */
+    private String schedulerType;
+
+    /**
+     * 调度配置，取决于调度类型
+     */
+    private String schedulerConf;
+
+    /**
+     * 开始时间
+     */
+    private Long startTime;
+
+    /**
+     * 结束时间
+     */
+    private Long stopTime;
+
+    private Source source;
+    private Target target;
+
+    /**
+     * 上次执行结果，获取任务最新的日志状态
+     * running执行中
+     * success
+     * fail失败
+     */
+    private String lastExecuteStatus;
+    private Long incrementValue;
+
+    // 以下为日志信息
+    private String logId;
+    private Long executorStartTime;
+    private Long executorEndTime;
+    private String executorMsg;
+    /**
+     * 成功SUCCESS,失败FAIL,执行中RUNNING
+     */
+    private String logStatus;
+
+    /**
+     * 在执行周期内
+     */
+    private boolean withinCycle;
+
+}

sdk/api/api-sync/src/main/java/io/dataease/api/sync/task/vo/TaskLogVO.java

@@ -0,0 +1,25 @@
+package io.dataease.api.sync.task.vo;
+
+import lombok.Data;
+
+/**
+ * 任务日志
+ * @author fit2cloud
+ * @date 2023/9/19 17:44
+ **/
+@Data
+public class TaskLogVO {
+
+    private String id;
+    private String jobId;
+    private String jobName;
+    private String jobDesc;
+    private Long executorStartTime;
+    private Long executorEndTime;
+    private String status;
+    private String executorMsg;
+    private String executorAddress;
+
+    private String clearType;
+
+}