数据源

apps/visual/api/data_source.go

@@ -6,8 +6,11 @@ package api
 // 生成人：panda
 // ==========================================================================
 import (
+	"github.com/XM-GO/PandaKit/biz"
 	"github.com/XM-GO/PandaKit/model"
 	"github.com/XM-GO/PandaKit/restfulx"
+	"github.com/kakuilan/kgo"
+	"pandax/apps/visual/driver"
 	"strings"
 
 	"pandax/apps/visual/entity"
@@ -24,6 +27,7 @@ func (p *VisualDataSourceApi) GetVisualDataSourceList(rc *restfulx.ReqCtx) {
 	pageNum := restfulx.QueryInt(rc, "pageNum", 1)
 	pageSize := restfulx.QueryInt(rc, "pageSize", 10)
 	data.SourceName = restfulx.QueryParam(rc, "sourceName")
+	data.SourceType = restfulx.QueryParam(rc, "sourceType")
 	data.Status = restfulx.QueryParam(rc, "status")
 
 	list, total := p.VisualDataSourceApp.FindListPage(pageNum, pageSize, data)
@@ -46,7 +50,13 @@ func (p *VisualDataSourceApi) GetVisualDataSource(rc *restfulx.ReqCtx) {
 func (p *VisualDataSourceApi) InsertVisualDataSource(rc *restfulx.ReqCtx) {
 	var data entity.VisualDataSource
 	restfulx.BindQuery(rc, &data)
-
+	data.SourceId = kgo.KStr.Uniqid("px")
+	err := driver.TestConnection(&data)
+	if err != nil {
+		data.Status = "0"
+	} else {
+		data.Status = "1"
+	}
 	p.VisualDataSourceApp.Insert(data)
 }
 
@@ -64,3 +74,30 @@ func (p *VisualDataSourceApi) DeleteVisualDataSource(rc *restfulx.ReqCtx) {
 	sourceIds := strings.Split(sourceId, ",")
 	p.VisualDataSourceApp.Delete(sourceIds)
 }
+
+// GetDataSourceTest 校验数据源连接性
+func (p *VisualDataSourceApi) GetDataSourceTest(rc *restfulx.ReqCtx) {
+	var data entity.VisualDataSource
+	restfulx.BindQuery(rc, &data)
+	err := driver.TestConnection(&data)
+	biz.ErrIsNilAppendErr(err, "数据库连接失败: %s")
+}
+
+// GetDataSourceTables 获取数据源下所有表
+func (p *VisualDataSourceApi) GetDataSourceTables(rc *restfulx.ReqCtx) {
+	sourceId := restfulx.PathParam(rc, "sourceId")
+	one := p.VisualDataSourceApp.FindOne(sourceId)
+	instance := driver.NewDbInstance(one)
+	biz.IsTrue(instance != nil, "获取数据源下所有表失败")
+	rc.ResData = instance.GetMeta().GetTableInfos()
+}
+
+// GetDataSourceTableDetails 获取表下面的所有字段
+func (p *VisualDataSourceApi) GetDataSourceTableDetails(rc *restfulx.ReqCtx) {
+	sourceId := restfulx.PathParam(rc, "sourceId")
+	tableName := restfulx.QueryParam(rc, "tableName")
+	one := p.VisualDataSourceApp.FindOne(sourceId)
+	instance := driver.NewDbInstance(one)
+	biz.IsTrue(instance != nil, "获取表下所有字段失败")
+	rc.ResData = instance.GetMeta().GetColumns(tableName)
+}

apps/visual/driver/DbInstance.go

@@ -0,0 +1,270 @@
+package driver
+
+import (
+	"database/sql"
+	"fmt"
+	"github.com/XM-GO/PandaKit/cache"
+	"pandax/apps/visual/entity"
+	"pandax/pkg/global"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// db实例
+type DbInstance struct {
+	Id   string
+	Info *entity.VisualDataSource
+	db   *sql.DB
+}
+
+func NewDbInstance(source *entity.VisualDataSource) *DbInstance {
+	byCache := GetDbInstanceByCache(source.SourceId)
+	if byCache != nil {
+		return byCache
+	}
+	conn, err := GetDbConn(source)
+	if err != nil {
+		return nil
+	}
+	di := &DbInstance{
+		Id:   source.SourceId,
+		Info: source,
+		db:   conn,
+	}
+	AddDbInstanceToCache(source.SourceId, di)
+	return di
+}
+
+// 执行查询语句
+// 依次返回 列名数组，结果map，错误
+func (d *DbInstance) SelectData(execSql string) ([]string, []map[string]interface{}, error) {
+	return SelectDataByDb(d.db, execSql)
+}
+
+// 将查询结果映射至struct，可具体参考sqlx库
+func (d *DbInstance) SelectData2Struct(execSql string, dest interface{}) error {
+	return Select2StructByDb(d.db, execSql, dest)
+}
+
+// 执行内部查询语句，不返回列名以及不限制行数
+// 依次返回 结果map，错误
+func (d *DbInstance) innerSelect(execSql string) ([]map[string]interface{}, error) {
+	_, res, err := SelectDataByDb(d.db, execSql)
+	return res, err
+}
+
+// 执行 update, insert, delete，建表等sql
+// 返回影响条数和错误
+func (d *DbInstance) Exec(sql string) (int64, error) {
+	res, err := d.db.Exec(sql)
+	if err != nil {
+		return 0, err
+	}
+	return res.RowsAffected()
+}
+
+// 获取数据库元信息实现接口
+func (di *DbInstance) GetMeta() DbMetadata {
+	dbType := di.Info.SourceType
+	if dbType == entity.DbTypeMysql {
+		return &MysqlMetadata{di: di}
+	}
+	if dbType == entity.DbTypePostgres {
+		return &PgsqlMetadata{di: di}
+	}
+	return nil
+}
+
+// 关闭连接
+func (d *DbInstance) Close() {
+	if d.db != nil {
+		if err := d.db.Close(); err != nil {
+			global.Log.Errorf("关闭数据库实例[%s]连接失败: %s", d.Id, err.Error())
+		}
+		d.db = nil
+	}
+}
+
+const DbConnExpireTime = 45 * time.Minute
+
+//------------------------------------------------------------------------------
+
+// 客户端连接缓存，指定时间内没有访问则会被关闭, key为数据库实例id:数据库
+var dbCache = cache.NewTimedCache(DbConnExpireTime, 5*time.Second).
+	WithUpdateAccessTime(true).
+	OnEvicted(func(key interface{}, value interface{}) {
+		global.Log.Info(fmt.Sprintf("删除db连接缓存 id = %s", key))
+		value.(*DbInstance).Close()
+	})
+
+func GetDbCacheKey(dbId uint64, db string) string {
+	return fmt.Sprintf("%d:%s", dbId, db)
+}
+
+// 删除db缓存并关闭该数据库所有连接
+func CloseDb(id string) {
+	dbCache.Delete(id)
+}
+func GetDbInstanceByCache(id string) *DbInstance {
+	if load, ok := dbCache.Get(id); ok {
+		return load.(*DbInstance)
+	}
+	return nil
+}
+
+// 将实体添加到缓存中
+func AddDbInstanceToCache(id string, di *DbInstance) {
+	dbCache.AddIfAbsent(id, di)
+}
+
+func TestConnection(d *entity.VisualDataSource) error {
+	// 验证第一个库是否可以连接即可
+	DB, err := GetDbConn(d)
+	if err != nil {
+		return err
+	} else {
+		DB.Close()
+		return nil
+	}
+}
+
+// 获取数据库连接
+func GetDbConn(d *entity.VisualDataSource) (*sql.DB, error) {
+	var DB *sql.DB
+	var err error
+	if d.SourceType == entity.DbTypeMysql {
+		DB, err = getMysqlDB(d)
+	} else if d.SourceType == entity.DbTypePostgres {
+		DB, err = getPgsqlDB(d)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+	err = DB.Ping()
+	if err != nil {
+		DB.Close()
+		return nil, err
+	}
+
+	return DB, nil
+}
+
+func SelectDataByDb(db *sql.DB, selectSql string) ([]string, []map[string]interface{}, error) {
+	rows, err := db.Query(selectSql)
+	if err != nil {
+		return nil, nil, err
+	}
+	// rows对象一定要close掉，如果出错，不关掉则会很迅速的达到设置最大连接数，
+	// 后面的链接过来直接报错或拒绝，实际上也没有起效果
+	defer func() {
+		if rows != nil {
+			rows.Close()
+		}
+	}()
+	colTypes, _ := rows.ColumnTypes()
+	// 这里表示一行填充数据
+	scans := make([]interface{}, len(colTypes))
+	// 这里表示一行所有列的值，用[]byte表示
+	vals := make([][]byte, len(colTypes))
+	// 这里scans引用vals，把数据填充到[]byte里
+	for k := range vals {
+		scans[k] = &vals[k]
+	}
+
+	result := make([]map[string]interface{}, 0)
+	// 列名用于前端表头名称按照数据库与查询字段顺序显示
+	colNames := make([]string, 0)
+	// 是否第一次遍历，列名数组只需第一次遍历时加入
+	isFirst := true
+	for rows.Next() {
+		// 不Scan也会导致等待，该链接实际处于未工作的状态，然后也会导致连接数迅速达到最大
+		err := rows.Scan(scans...)
+		if err != nil {
+			return nil, nil, err
+		}
+		// 每行数据
+		rowData := make(map[string]interface{})
+		// 把vals中的数据复制到row中
+		for i, v := range vals {
+			colType := colTypes[i]
+			colName := colType.Name()
+			// 如果是第一行，则将列名加入到列信息中，由于map是无序的，所有需要返回列名的有序数组
+			if isFirst {
+				colNames = append(colNames, colName)
+			}
+			rowData[colName] = valueConvert(v, colType)
+		}
+		// 放入结果集
+		result = append(result, rowData)
+		isFirst = false
+	}
+	return colNames, result, nil
+}
+
+// 将查询的值转为对应列类型的实际值，不全部转为字符串
+func valueConvert(data []byte, colType *sql.ColumnType) interface{} {
+	if data == nil {
+		return nil
+	}
+	// 列的数据库类型名
+	colDatabaseTypeName := strings.ToLower(colType.DatabaseTypeName())
+
+	// 如果类型是bit，则直接返回第一个字节即可
+	if strings.Contains(colDatabaseTypeName, "bit") {
+		return data[0]
+	}
+
+	// 这里把[]byte数据转成string
+	stringV := string(data)
+	if stringV == "" {
+		return ""
+	}
+	colScanType := strings.ToLower(colType.ScanType().Name())
+
+	if strings.Contains(colScanType, "int") {
+		// 如果长度超过16位，则返回字符串，因为前端js长度大于16会丢失精度
+		if len(stringV) > 16 {
+			return stringV
+		}
+		intV, _ := strconv.Atoi(stringV)
+		switch colType.ScanType().Kind() {
+		case reflect.Int8:
+			return int8(intV)
+		case reflect.Uint8:
+			return uint8(intV)
+		case reflect.Int64:
+			return int64(intV)
+		case reflect.Uint64:
+			return uint64(intV)
+		case reflect.Uint:
+			return uint(intV)
+		default:
+			return intV
+		}
+	}
+	if strings.Contains(colScanType, "float") || strings.Contains(colDatabaseTypeName, "decimal") {
+		floatV, _ := strconv.ParseFloat(stringV, 64)
+		return floatV
+	}
+
+	return stringV
+}
+
+// 查询数据结果映射至struct。可参考sqlx库
+func Select2StructByDb(db *sql.DB, selectSql string, dest interface{}) error {
+	rows, err := db.Query(selectSql)
+	if err != nil {
+		return err
+	}
+	// rows对象一定要close掉，如果出错，不关掉则会很迅速的达到设置最大连接数，
+	// 后面的链接过来直接报错或拒绝，实际上也没有起效果
+	defer func() {
+		if rows != nil {
+			rows.Close()
+		}
+	}()
+	return scanAll(rows, dest, false)
+}

apps/visual/driver/meta.go

@@ -0,0 +1,9 @@
+package driver
+
+type DbMetadata interface {
+	GetTables() []map[string]interface{}
+	GetColumns(tableNames ...string) []map[string]interface{}
+	GetPrimaryKey(tableName string) string
+	GetTableInfos() []map[string]interface{}
+	GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error)
+}

apps/visual/driver/mysql_meta.go

@@ -0,0 +1,118 @@
+package driver
+
+import (
+	"database/sql"
+	"fmt"
+	"github.com/XM-GO/PandaKit/biz"
+	"log"
+	"pandax/apps/visual/entity"
+
+	_ "github.com/go-sql-driver/mysql"
+)
+
+func getMysqlDB(d *entity.VisualDataSource) (*sql.DB, error) {
+	// SSH Conect
+	// 设置dataSourceName  -> 更多参数参考：https://github.com/go-sql-driver/mysql#dsn-data-source-name
+	dsn := fmt.Sprintf("%s:%s@%s(%s:%d)/%s?timeout=8s",
+		d.Db.Username, d.Db.Password, "tcp", d.Db.Host, d.Db.Port, d.Db.Dbname)
+	if d.Db.Config != "" {
+		dsn = fmt.Sprintf("%s&%s", dsn, d.Db.Config)
+	}
+	log.Println(dsn)
+	return sql.Open("mysql", dsn)
+}
+
+// ---------------------------------- mysql元数据 -----------------------------------
+const (
+	// mysql 表信息元数据
+	MYSQL_TABLE_MA = `SELECT table_name tableName, table_comment tableComment from information_schema.tables WHERE table_schema = (SELECT database())`
+
+	// mysql 表信息
+	MYSQL_TABLE_INFO = `SELECT table_name tableName, table_comment tableComment, table_rows tableRows,
+	data_length dataLength, index_length indexLength, create_time createTime 
+	FROM information_schema.tables 
+    WHERE table_schema = (SELECT database())`
+
+	// mysql 索引信息
+	MYSQL_INDEX_INFO = `SELECT index_name indexName, column_name columnName, index_type indexType, non_unique nonUnique, 
+	SEQ_IN_INDEX seqInIndex, INDEX_COMMENT indexComment
+	FROM information_schema.STATISTICS 
+	WHERE table_schema = (SELECT database()) AND table_name = '%s' ORDER BY index_name asc , SEQ_IN_INDEX asc`
+
+	// mysql 列信息元数据
+	MYSQL_COLUMN_MA = `SELECT table_name tableName, column_name columnName, column_type columnType, column_default columnDefault,
+	column_comment columnComment, column_key columnKey, extra, is_nullable nullable from information_schema.columns
+	WHERE table_schema = (SELECT database()) AND table_name in (%s) ORDER BY tableName, ordinal_position`
+)
+
+type MysqlMetadata struct {
+	di *DbInstance
+}
+
+// 获取表基础元信息, 如表名等
+func (mm *MysqlMetadata) GetTables() []map[string]interface{} {
+	res, err := mm.di.innerSelect(MYSQL_TABLE_MA)
+	biz.ErrIsNilAppendErr(err, "获取表基本信息失败: %s")
+	return res
+}
+
+// 获取列元信息, 如列名等
+func (mm *MysqlMetadata) GetColumns(tableNames ...string) []map[string]interface{} {
+	tableName := ""
+	for i := 0; i < len(tableNames); i++ {
+		if i != 0 {
+			tableName = tableName + ", "
+		}
+		tableName = tableName + "'" + tableNames[i] + "'"
+	}
+	result, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_COLUMN_MA, tableName))
+	biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s")
+	return result
+}
+
+// 获取表主键字段名，不存在主键标识则默认第一个字段
+func (mm *MysqlMetadata) GetPrimaryKey(tablename string) string {
+	columns := mm.GetColumns(tablename)
+	biz.IsTrue(len(columns) > 0, "[%s] 表不存在", tablename)
+	for _, v := range columns {
+		if v["columnKey"].(string) == "PRI" {
+			return v["columnName"].(string)
+		}
+	}
+
+	return columns[0]["columnName"].(string)
+}
+
+// 获取表信息，比GetTableMetedatas获取更详细的表信息
+func (mm *MysqlMetadata) GetTableInfos() []map[string]interface{} {
+	res, err := mm.di.innerSelect(MYSQL_TABLE_INFO)
+	biz.ErrIsNilAppendErr(err, "获取表信息失败: %s")
+	return res
+}
+
+// 获取表索引信息
+func (mm *MysqlMetadata) GetTableIndex(tableName string) []map[string]interface{} {
+	res, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_INDEX_INFO, tableName))
+	biz.ErrIsNilAppendErr(err, "获取表索引信息失败: %s")
+	// 把查询结果以索引名分组，索引字段以逗号连接
+	result := make([]map[string]interface{}, 0)
+	key := ""
+	for _, v := range res {
+		// 当前的索引名
+		in := v["indexName"].(string)
+		if key == in {
+			// 索引字段已根据名称和顺序排序，故取最后一个即可
+			i := len(result) - 1
+			// 同索引字段以逗号连接
+			result[i]["columnName"] = result[i]["columnName"].(string) + "," + v["columnName"].(string)
+		} else {
+			key = in
+			result = append(result, v)
+		}
+	}
+	return result
+}
+
+func (mm *MysqlMetadata) GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error) {
+	return mm.di.SelectData(fmt.Sprintf("SELECT * FROM %s LIMIT %d, %d", tableName, (pageNum-1)*pageSize, pageSize))
+}

apps/visual/driver/pgsql_meta.go

@@ -0,0 +1,111 @@
+package driver
+
+import (
+	"database/sql"
+	"fmt"
+	"github.com/XM-GO/PandaKit/biz"
+	"pandax/apps/visual/entity"
+	"strings"
+
+	_ "github.com/lib/pq"
+)
+
+func getPgsqlDB(d *entity.VisualDataSource) (*sql.DB, error) {
+	dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
+		d.Db.Host, d.Db.Port, d.Db.Username, d.Db.Password, d.Db.Dbname)
+	if d.Db.Config != "" {
+		dsn = fmt.Sprintf("%s %s", dsn, strings.Join(strings.Split(d.Db.Config, "&"), " "))
+	}
+	return sql.Open("postgres", dsn)
+}
+
+// ---------------------------------- pgsql元数据 -----------------------------------
+const (
+	// postgres 表信息元数据
+	PGSQL_TABLE_MA = `SELECT obj_description(c.oid) AS "tableComment", c.relname AS "tableName" FROM pg_class c 
+	JOIN pg_namespace n ON c.relnamespace = n.oid WHERE n.nspname = (select current_schema()) AND c.reltype > 0`
+
+	PGSQL_TABLE_INFO = `SELECT obj_description(c.oid) AS "tableComment", c.relname AS "tableName" FROM pg_class c 
+	JOIN pg_namespace n ON c.relnamespace = n.oid WHERE n.nspname = (select current_schema()) AND c.reltype > 0`
+
+	PGSQL_INDEX_INFO = `SELECT indexname AS "indexName", indexdef AS "indexComment"
+	FROM pg_indexes WHERE schemaname =  (select current_schema()) AND tablename = '%s'`
+
+	PGSQL_COLUMN_MA = `SELECT
+		C.relname AS "tableName",
+		A.attname AS "columnName",
+		tc.is_nullable AS "nullable",
+		concat_ws ( '', t.typname, SUBSTRING ( format_type ( a.atttypid, a.atttypmod ) FROM '\(.*\)' ) ) AS "columnType",
+		(CASE WHEN ( SELECT COUNT(*) FROM pg_constraint WHERE conrelid = a.attrelid AND conkey[1]= attnum AND contype = 'p' ) > 0 THEN 'PRI' ELSE '' END ) AS "columnKey",
+		d.description AS "columnComment" 
+	FROM
+		pg_attribute a LEFT JOIN pg_description d ON d.objoid = a.attrelid 
+		AND d.objsubid = A.attnum
+		LEFT JOIN pg_class c ON A.attrelid = c.oid
+		LEFT JOIN pg_namespace pn ON c.relnamespace = pn.oid
+		LEFT JOIN pg_type t ON a.atttypid = t.oid 
+		JOIN information_schema.columns tc ON tc.column_name = a.attname AND tc.table_name = C.relname AND tc.table_schema = pn.nspname
+	WHERE
+		A.attnum >= 0 
+		AND pn.nspname = (select current_schema())
+		AND C.relname in (%s)
+	ORDER BY
+		C.relname DESC,
+		A.attnum ASC	
+	`
+)
+
+type PgsqlMetadata struct {
+	di *DbInstance
+}
+
+// 获取表基础元信息, 如表名等
+func (pm *PgsqlMetadata) GetTables() []map[string]interface{} {
+	res, err := pm.di.innerSelect(PGSQL_TABLE_MA)
+	biz.ErrIsNilAppendErr(err, "获取表基本信息失败: %s")
+	return res
+}
+
+// 获取列元信息, 如列名等
+func (pm *PgsqlMetadata) GetColumns(tableNames ...string) []map[string]interface{} {
+	tableName := ""
+	for i := 0; i < len(tableNames); i++ {
+		if i != 0 {
+			tableName = tableName + ", "
+		}
+		tableName = tableName + "'" + tableNames[i] + "'"
+	}
+	result, err := pm.di.innerSelect(fmt.Sprintf(PGSQL_COLUMN_MA, tableName))
+	biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s")
+	return result
+}
+
+func (pm *PgsqlMetadata) GetPrimaryKey(tablename string) string {
+	columns := pm.GetColumns(tablename)
+	biz.IsTrue(len(columns) > 0, "[%s] 表不存在", tablename)
+	for _, v := range columns {
+		if v["columnKey"].(string) == "PRI" {
+			return v["columnName"].(string)
+		}
+	}
+
+	return columns[0]["columnName"].(string)
+}
+
+// 获取表信息，比GetTables获取更详细的表信息
+func (pm *PgsqlMetadata) GetTableInfos() []map[string]interface{} {
+	res, err := pm.di.innerSelect(PGSQL_TABLE_INFO)
+	biz.ErrIsNilAppendErr(err, "获取表信息失败: %s")
+	return res
+}
+
+// 获取表索引信息
+func (pm *PgsqlMetadata) GetTableIndex(tableName string) []map[string]interface{} {
+	res, err := pm.di.innerSelect(fmt.Sprintf(PGSQL_INDEX_INFO, tableName))
+	biz.ErrIsNilAppendErr(err, "获取表索引信息失败: %s")
+	return res
+}
+
+func (pm *PgsqlMetadata) GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error) {
+	return pm.di.SelectData(fmt.Sprintf("SELECT * FROM %s OFFSET %d LIMIT %d", tableName, (pageNum-1)*pageSize, pageSize))
+}

apps/visual/driver/sqlx.go

@@ -0,0 +1,630 @@
+package driver
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+)
+
+// 将结果scan至结构体，copy至 sqlx库: https://github.com/jmoiron/sqlx
+func scanAll(rows *sql.Rows, dest interface{}, structOnly bool) error {
+	var v, vp reflect.Value
+
+	value := reflect.ValueOf(dest)
+
+	// json.Unmarshal returns errors for these
+	if value.Kind() != reflect.Ptr {
+		return errors.New("must pass a pointer, not a value, to StructScan destination")
+	}
+	if value.IsNil() {
+		return errors.New("nil pointer passed to StructScan destination")
+	}
+	direct := reflect.Indirect(value)
+
+	slice, err := baseType(value.Type(), reflect.Slice)
+	if err != nil {
+		return err
+	}
+	direct.SetLen(0)
+
+	isPtr := slice.Elem().Kind() == reflect.Ptr
+	base := Deref(slice.Elem())
+	scannable := isScannable(base)
+
+	if structOnly && scannable {
+		return structOnlyError(base)
+	}
+
+	columns, err := rows.Columns()
+	if err != nil {
+		return err
+	}
+
+	// if it's a base type make sure it only has 1 column;  if not return an error
+	if scannable && len(columns) > 1 {
+		return fmt.Errorf("non-struct dest type %s with >1 columns (%d)", base.Kind(), len(columns))
+	}
+
+	if !scannable {
+		var values []interface{}
+		var m *Mapper = mapper()
+
+		fields := m.TraversalsByName(base, columns)
+		// if we are not unsafe and are missing fields, return an error
+		if f, err := missingFields(fields); err != nil {
+			return fmt.Errorf("missing destination name %s in %T", columns[f], dest)
+		}
+		values = make([]interface{}, len(columns))
+
+		for rows.Next() {
+			// create a new struct type (which returns PtrTo) and indirect it
+			vp = reflect.New(base)
+			v = reflect.Indirect(vp)
+
+			err = fieldsByTraversal(v, fields, values, true)
+			if err != nil {
+				return err
+			}
+
+			// scan into the struct field pointers and append to our results
+			err = rows.Scan(values...)
+			if err != nil {
+				return err
+			}
+
+			if isPtr {
+				direct.Set(reflect.Append(direct, vp))
+			} else {
+				direct.Set(reflect.Append(direct, v))
+			}
+		}
+	} else {
+		for rows.Next() {
+			vp = reflect.New(base)
+			err = rows.Scan(vp.Interface())
+			if err != nil {
+				return err
+			}
+			// append
+			if isPtr {
+				direct.Set(reflect.Append(direct, vp))
+			} else {
+				direct.Set(reflect.Append(direct, reflect.Indirect(vp)))
+			}
+		}
+	}
+
+	return rows.Err()
+}
+
+func baseType(t reflect.Type, expected reflect.Kind) (reflect.Type, error) {
+	t = Deref(t)
+	if t.Kind() != expected {
+		return nil, fmt.Errorf("expected %s but got %s", expected, t.Kind())
+	}
+	return t, nil
+}
+
+// structOnlyError returns an error appropriate for type when a non-scannable
+// struct is expected but something else is given
+func structOnlyError(t reflect.Type) error {
+	isStruct := t.Kind() == reflect.Struct
+	isScanner := reflect.PtrTo(t).Implements(_scannerInterface)
+	if !isStruct {
+		return fmt.Errorf("expected %s but got %s", reflect.Struct, t.Kind())
+	}
+	if isScanner {
+		return fmt.Errorf("structscan expects a struct dest but the provided struct type %s implements scanner", t.Name())
+	}
+	return fmt.Errorf("expected a struct, but struct %s has no exported fields", t.Name())
+}
+
+var _scannerInterface = reflect.TypeOf((*sql.Scanner)(nil)).Elem()
+
+func isScannable(t reflect.Type) bool {
+	if reflect.PtrTo(t).Implements(_scannerInterface) {
+		return true
+	}
+	if t.Kind() != reflect.Struct {
+		return true
+	}
+
+	// it's not important that we use the right mapper for this particular object,
+	// we're only concerned on how many exported fields this struct has
+	return len(mapper().TypeMap(t).Index) == 0
+}
+
+var NameMapper = strings.ToLower
+var origMapper = reflect.ValueOf(NameMapper)
+
+// Rather than creating on init, this is created when necessary so that
+// importers have time to customize the NameMapper.
+var mpr *Mapper
+
+// mprMu protects mpr.
+var mprMu sync.Mutex
+
+// mapper returns a valid mapper using the configured NameMapper func.
+func mapper() *Mapper {
+	mprMu.Lock()
+	defer mprMu.Unlock()
+
+	if mpr == nil {
+		mpr = NewMapperFunc("db", NameMapper)
+	} else if origMapper != reflect.ValueOf(NameMapper) {
+		// if NameMapper has changed, create a new mapper
+		mpr = NewMapperFunc("db", NameMapper)
+		origMapper = reflect.ValueOf(NameMapper)
+	}
+	return mpr
+}
+
+func missingFields(transversals [][]int) (field int, err error) {
+	for i, t := range transversals {
+		if len(t) == 0 {
+			return i, errors.New("missing field")
+		}
+	}
+	return 0, nil
+}
+
+// fieldsByName fills a values interface with fields from the passed value based
+// on the traversals in int.  If ptrs is true, return addresses instead of values.
+// We write this instead of using FieldsByName to save allocations and map lookups
+// when iterating over many rows.  Empty traversals will get an interface pointer.
+// Because of the necessity of requesting ptrs or values, it's considered a bit too
+// specialized for inclusion in reflectx itself.
+func fieldsByTraversal(v reflect.Value, traversals [][]int, values []interface{}, ptrs bool) error {
+	v = reflect.Indirect(v)
+	if v.Kind() != reflect.Struct {
+		return errors.New("argument not a struct")
+	}
+
+	for i, traversal := range traversals {
+		if len(traversal) == 0 {
+			values[i] = new(interface{})
+			continue
+		}
+		f := FieldByIndexes(v, traversal)
+		if ptrs {
+			values[i] = f.Addr().Interface()
+		} else {
+			values[i] = f.Interface()
+		}
+	}
+	return nil
+}
+
+// A FieldInfo is metadata for a struct field.
+type FieldInfo struct {
+	Index    []int
+	Path     string
+	Field    reflect.StructField
+	Zero     reflect.Value
+	Name     string
+	Options  map[string]string
+	Embedded bool
+	Children []*FieldInfo
+	Parent   *FieldInfo
+}
+
+// A StructMap is an index of field metadata for a struct.
+type StructMap struct {
+	Tree  *FieldInfo
+	Index []*FieldInfo
+	Paths map[string]*FieldInfo
+	Names map[string]*FieldInfo
+}
+
+// GetByPath returns a *FieldInfo for a given string path.
+func (f StructMap) GetByPath(path string) *FieldInfo {
+	return f.Paths[path]
+}
+
+// GetByTraversal returns a *FieldInfo for a given integer path.  It is
+// analogous to reflect.FieldByIndex, but using the cached traversal
+// rather than re-executing the reflect machinery each time.
+func (f StructMap) GetByTraversal(index []int) *FieldInfo {
+	if len(index) == 0 {
+		return nil
+	}
+
+	tree := f.Tree
+	for _, i := range index {
+		if i >= len(tree.Children) || tree.Children[i] == nil {
+			return nil
+		}
+		tree = tree.Children[i]
+	}
+	return tree
+}
+
+// Mapper is a general purpose mapper of names to struct fields.  A Mapper
+// behaves like most marshallers in the standard library, obeying a field tag
+// for name mapping but also providing a basic transform function.
+type Mapper struct {
+	cache      map[reflect.Type]*StructMap
+	tagName    string
+	tagMapFunc func(string) string
+	mapFunc    func(string) string
+	mutex      sync.Mutex
+}
+
+// NewMapper returns a new mapper using the tagName as its struct field tag.
+// If tagName is the empty string, it is ignored.
+func NewMapper(tagName string) *Mapper {
+	return &Mapper{
+		cache:   make(map[reflect.Type]*StructMap),
+		tagName: tagName,
+	}
+}
+
+// NewMapperTagFunc returns a new mapper which contains a mapper for field names
+// AND a mapper for tag values.  This is useful for tags like json which can
+// have values like "name,omitempty".
+func NewMapperTagFunc(tagName string, mapFunc, tagMapFunc func(string) string) *Mapper {
+	return &Mapper{
+		cache:      make(map[reflect.Type]*StructMap),
+		tagName:    tagName,
+		mapFunc:    mapFunc,
+		tagMapFunc: tagMapFunc,
+	}
+}
+
+// NewMapperFunc returns a new mapper which optionally obeys a field tag and
+// a struct field name mapper func given by f.  Tags will take precedence, but
+// for any other field, the mapped name will be f(field.Name)
+func NewMapperFunc(tagName string, f func(string) string) *Mapper {
+	return &Mapper{
+		cache:   make(map[reflect.Type]*StructMap),
+		tagName: tagName,
+		mapFunc: f,
+	}
+}
+
+// TypeMap returns a mapping of field strings to int slices representing
+// the traversal down the struct to reach the field.
+func (m *Mapper) TypeMap(t reflect.Type) *StructMap {
+	m.mutex.Lock()
+	mapping, ok := m.cache[t]
+	if !ok {
+		mapping = getMapping(t, m.tagName, m.mapFunc, m.tagMapFunc)
+		m.cache[t] = mapping
+	}
+	m.mutex.Unlock()
+	return mapping
+}
+
+// FieldMap returns the mapper's mapping of field names to reflect values.  Panics
+// if v's Kind is not Struct, or v is not Indirectable to a struct kind.
+func (m *Mapper) FieldMap(v reflect.Value) map[string]reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	r := map[string]reflect.Value{}
+	tm := m.TypeMap(v.Type())
+	for tagName, fi := range tm.Names {
+		r[tagName] = FieldByIndexes(v, fi.Index)
+	}
+	return r
+}
+
+// FieldByName returns a field by its mapped name as a reflect.Value.
+// Panics if v's Kind is not Struct or v is not Indirectable to a struct Kind.
+// Returns zero Value if the name is not found.
+func (m *Mapper) FieldByName(v reflect.Value, name string) reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	tm := m.TypeMap(v.Type())
+	fi, ok := tm.Names[name]
+	if !ok {
+		return v
+	}
+	return FieldByIndexes(v, fi.Index)
+}
+
+// FieldsByName returns a slice of values corresponding to the slice of names
+// for the value.  Panics if v's Kind is not Struct or v is not Indirectable
+// to a struct Kind.  Returns zero Value for each name not found.
+func (m *Mapper) FieldsByName(v reflect.Value, names []string) []reflect.Value {
+	v = reflect.Indirect(v)
+	mustBe(v, reflect.Struct)
+
+	tm := m.TypeMap(v.Type())
+	vals := make([]reflect.Value, 0, len(names))
+	for _, name := range names {
+		fi, ok := tm.Names[name]
+		if !ok {
+			vals = append(vals, *new(reflect.Value))
+		} else {
+			vals = append(vals, FieldByIndexes(v, fi.Index))
+		}
+	}
+	return vals
+}
+
+// TraversalsByName returns a slice of int slices which represent the struct
+// traversals for each mapped name.  Panics if t is not a struct or Indirectable
+// to a struct.  Returns empty int slice for each name not found.
+func (m *Mapper) TraversalsByName(t reflect.Type, names []string) [][]int {
+	r := make([][]int, 0, len(names))
+	m.TraversalsByNameFunc(t, names, func(_ int, i []int) error {
+		if i == nil {
+			r = append(r, []int{})
+		} else {
+			r = append(r, i)
+		}
+
+		return nil
+	})
+	return r
+}
+
+// TraversalsByNameFunc traverses the mapped names and calls fn with the index of
+// each name and the struct traversal represented by that name. Panics if t is not
+// a struct or Indirectable to a struct. Returns the first error returned by fn or nil.
+func (m *Mapper) TraversalsByNameFunc(t reflect.Type, names []string, fn func(int, []int) error) error {
+	t = Deref(t)
+	mustBe(t, reflect.Struct)
+	tm := m.TypeMap(t)
+	for i, name := range names {
+		fi, ok := tm.Names[name]
+		if !ok {
+			if err := fn(i, nil); err != nil {
+				return err
+			}
+		} else {
+			if err := fn(i, fi.Index); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// FieldByIndexes returns a value for the field given by the struct traversal
+// for the given value.
+func FieldByIndexes(v reflect.Value, indexes []int) reflect.Value {
+	for _, i := range indexes {
+		v = reflect.Indirect(v).Field(i)
+		// if this is a pointer and it's nil, allocate a new value and set it
+		if v.Kind() == reflect.Ptr && v.IsNil() {
+			alloc := reflect.New(Deref(v.Type()))
+			v.Set(alloc)
+		}
+		if v.Kind() == reflect.Map && v.IsNil() {
+			v.Set(reflect.MakeMap(v.Type()))
+		}
+	}
+	return v
+}
+
+// FieldByIndexesReadOnly returns a value for a particular struct traversal,
+// but is not concerned with allocating nil pointers because the value is
+// going to be used for reading and not setting.
+func FieldByIndexesReadOnly(v reflect.Value, indexes []int) reflect.Value {
+	for _, i := range indexes {
+		v = reflect.Indirect(v).Field(i)
+	}
+	return v
+}
+
+// Deref is Indirect for reflect.Types
+func Deref(t reflect.Type) reflect.Type {
+	if t.Kind() == reflect.Ptr {
+		t = t.Elem()
+	}
+	return t
+}
+
+// -- helpers & utilities --
+
+type kinder interface {
+	Kind() reflect.Kind
+}
+
+// mustBe checks a value against a kind, panicing with a reflect.ValueError
+// if the kind isn't that which is required.
+func mustBe(v kinder, expected reflect.Kind) {
+	if k := v.Kind(); k != expected {
+		panic(&reflect.ValueError{Method: methodName(), Kind: k})
+	}
+}
+
+// methodName returns the caller of the function calling methodName
+func methodName() string {
+	pc, _, _, _ := runtime.Caller(2)
+	f := runtime.FuncForPC(pc)
+	if f == nil {
+		return "unknown method"
+	}
+	return f.Name()
+}
+
+type typeQueue struct {
+	t  reflect.Type
+	fi *FieldInfo
+	pp string // Parent path
+}
+
+// A copying append that creates a new slice each time.
+func apnd(is []int, i int) []int {
+	x := make([]int, len(is)+1)
+	copy(x, is)
+	x[len(x)-1] = i
+	return x
+}
+
+type mapf func(string) string
+
+// parseName parses the tag and the target name for the given field using
+// the tagName (eg 'json' for `json:"foo"` tags), mapFunc for mapping the
+// field's name to a target name, and tagMapFunc for mapping the tag to
+// a target name.
+func parseName(field reflect.StructField, tagName string, mapFunc, tagMapFunc mapf) (tag, fieldName string) {
+	// first, set the fieldName to the field's name
+	fieldName = field.Name
+	// if a mapFunc is set, use that to override the fieldName
+	if mapFunc != nil {
+		fieldName = mapFunc(fieldName)
+	}
+
+	// if there's no tag to look for, return the field name
+	if tagName == "" {
+		return "", fieldName
+	}
+
+	// if this tag is not set using the normal convention in the tag,
+	// then return the fieldname..  this check is done because according
+	// to the reflect documentation:
+	//    If the tag does not have the conventional format,
+	//    the value returned by Get is unspecified.
+	// which doesn't sound great.
+	if !strings.Contains(string(field.Tag), tagName+":") {
+		return "", fieldName
+	}
+
+	// at this point we're fairly sure that we have a tag, so lets pull it out
+	tag = field.Tag.Get(tagName)
+
+	// if we have a mapper function, call it on the whole tag
+	// XXX: this is a change from the old version, which pulled out the name
+	// before the tagMapFunc could be run, but I think this is the right way
+	if tagMapFunc != nil {
+		tag = tagMapFunc(tag)
+	}
+
+	// finally, split the options from the name
+	parts := strings.Split(tag, ",")
+	fieldName = parts[0]
+
+	return tag, fieldName
+}
+
+// parseOptions parses options out of a tag string, skipping the name
+func parseOptions(tag string) map[string]string {
+	parts := strings.Split(tag, ",")
+	options := make(map[string]string, len(parts))
+	if len(parts) > 1 {
+		for _, opt := range parts[1:] {
+			// short circuit potentially expensive split op
+			if strings.Contains(opt, "=") {
+				kv := strings.Split(opt, "=")
+				options[kv[0]] = kv[1]
+				continue
+			}
+			options[opt] = ""
+		}
+	}
+	return options
+}
+
+// getMapping returns a mapping for the t type, using the tagName, mapFunc and
+// tagMapFunc to determine the canonical names of fields.
+func getMapping(t reflect.Type, tagName string, mapFunc, tagMapFunc mapf) *StructMap {
+	m := []*FieldInfo{}
+
+	root := &FieldInfo{}
+	queue := []typeQueue{}
+	queue = append(queue, typeQueue{Deref(t), root, ""})
+
+QueueLoop:
+	for len(queue) != 0 {
+		// pop the first item off of the queue
+		tq := queue[0]
+		queue = queue[1:]
+
+		// ignore recursive field
+		for p := tq.fi.Parent; p != nil; p = p.Parent {
+			if tq.fi.Field.Type == p.Field.Type {
+				continue QueueLoop
+			}
+		}
+
+		nChildren := 0
+		if tq.t.Kind() == reflect.Struct {
+			nChildren = tq.t.NumField()
+		}
+		tq.fi.Children = make([]*FieldInfo, nChildren)
+
+		// iterate through all of its fields
+		for fieldPos := 0; fieldPos < nChildren; fieldPos++ {
+
+			f := tq.t.Field(fieldPos)
+
+			// parse the tag and the target name using the mapping options for this field
+			tag, name := parseName(f, tagName, mapFunc, tagMapFunc)
+
+			// if the name is "-", disabled via a tag, skip it
+			if name == "-" {
+				continue
+			}
+
+			fi := FieldInfo{
+				Field:   f,
+				Name:    name,
+				Zero:    reflect.New(f.Type).Elem(),
+				Options: parseOptions(tag),
+			}
+
+			// if the path is empty this path is just the name
+			if tq.pp == "" {
+				fi.Path = fi.Name
+			} else {
+				fi.Path = tq.pp + "." + fi.Name
+			}
+
+			// skip unexported fields
+			if len(f.PkgPath) != 0 && !f.Anonymous {
+				continue
+			}
+
+			// bfs search of anonymous embedded structs
+			if f.Anonymous {
+				pp := tq.pp
+				if tag != "" {
+					pp = fi.Path
+				}
+
+				fi.Embedded = true
+				fi.Index = apnd(tq.fi.Index, fieldPos)
+				nChildren := 0
+				ft := Deref(f.Type)
+				if ft.Kind() == reflect.Struct {
+					nChildren = ft.NumField()
+				}
+				fi.Children = make([]*FieldInfo, nChildren)
+				queue = append(queue, typeQueue{Deref(f.Type), &fi, pp})
+			} else if fi.Zero.Kind() == reflect.Struct || (fi.Zero.Kind() == reflect.Ptr && fi.Zero.Type().Elem().Kind() == reflect.Struct) {
+				fi.Index = apnd(tq.fi.Index, fieldPos)
+				fi.Children = make([]*FieldInfo, Deref(f.Type).NumField())
+				queue = append(queue, typeQueue{Deref(f.Type), &fi, fi.Path})
+			}
+
+			fi.Index = apnd(tq.fi.Index, fieldPos)
+			fi.Parent = tq.fi
+			tq.fi.Children[fieldPos] = &fi
+			m = append(m, &fi)
+		}
+	}
+
+	flds := &StructMap{Index: m, Tree: root, Paths: map[string]*FieldInfo{}, Names: map[string]*FieldInfo{}}
+	for _, fi := range flds.Index {
+		// check if nothing has already been pushed with the same path
+		// sometimes you can choose to override a type using embedded struct
+		fld, ok := flds.Paths[fi.Path]
+		if !ok || fld.Embedded {
+			flds.Paths[fi.Path] = fi
+			if fi.Name != "" && !fi.Embedded {
+				flds.Names[fi.Path] = fi
+			}
+		}
+	}
+
+	return flds
+}

apps/visual/entity/data_set.go

@@ -18,7 +18,7 @@ func (VisualDataSetGroup) TableName() string {
 type VisualDataSetTable struct {
 	model.BaseModelD
 	TableId      string               `gorm:"primary_key;tableId;comment:表id" json:"tableId"`
-	DataSourceId string               `gorm:"dataSourceId;type:varchar(64);comment:数据圆ID" json:"dataSourceId"`
+	DataSourceId string               `gorm:"dataSourceId;type:varchar(64);comment:数据源ID" json:"dataSourceId"`
 	TableType    string               `gorm:"tableType;type:varchar(64);comment:db,sql,excel,custom,api" json:"tableType"` //'db,sql,excel,custom',
 	Mode         string               `gorm:"mode;type:varchar(1);comment:原始表信息" json:"mode"`                              //'连接模式：0-直连，1-定时同步',
 	Info         string               `gorm:"info;type:TEXT;comment:原始表信息" json:"info"`

apps/visual/entity/data_source.go

@@ -1,35 +1,49 @@
 package entity
 
-import "github.com/XM-GO/PandaKit/model"
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"errors"
+	"github.com/XM-GO/PandaKit/model"
+)
+
+const (
+	DbTypeMysql    = "MySQL"
+	DbTypePostgres = "PostgreSQL"
+)
 
 type VisualDataSource struct {
 	model.BaseModel
-	SourceId      string `gorm:"primary_key;source_id;comment:数据源Id"   json:"sourceId"`                           // 数据源Id
-	SourceType    string `gorm:"source_type;type:varchar(50);comment:数据源类型"         json:"sourceType"`            // 数据源类型
-	SourceName    string `gorm:"source_name;type:varchar(50);comment:数据源名称"      json:"sourceName"`               // 原名称
-	SourceComment string `gorm:"source_comment;type:varchar(50);comment:数据源描述"             json:"source_comment"` // 描述
-	Status        string `gorm:"status;type:varchar(1);comment:数据源状态" json:"status"`
-	Configuration string `gorm:"configuration;type:text;comment:详细信息" json:"configuration"`
-	CreateBy      int64  `gorm:"api" json:"createBy"` //创建人ID
+	SourceId      string   `gorm:"primary_key;source_id;comment:数据源Id"   json:"sourceId"`                          // 数据源Id
+	SourceType    string   `gorm:"source_type;type:varchar(50);comment:数据源类型"         json:"sourceType"`           // 数据源类型
+	SourceName    string   `gorm:"source_name;type:varchar(50);comment:数据源名称"      json:"sourceName"`              // 原名称
+	SourceComment string   `gorm:"source_comment;type:varchar(50);comment:数据源描述"             json:"sourceComment"` // 描述
+	Status        string   `gorm:"status;type:varchar(1);comment:数据源状态" json:"status"`
+	Db            VisualDb `gorm:"db;type:text;comment:详细信息" json:"db"`
+	CreateBy      int64    `gorm:"api" json:"createBy"` //创建人ID
 }
 
 type VisualDb struct {
-	DbIp         string `gorm:"db_ip" json:"dbIp"`
-	DbPort       string `gorm:"db_port" json:"dbPort"`
-	DbName       string `gorm:"db_name" json:"dbName"`
-	DbUsername   string `gorm:"db_username" json:"dbUsername"`
-	DbPassword   string `gorm:"db_password" json:"dbPassword"`
-	DbJointParam string `gorm:"db_joint_param" json:"dbJointParam"` //额外的链接参数
-}
-
-type VisualApi struct {
-	Method      string                 `gorm:"method" json:"method"`
-	Url         string                 `gorm:"url" json:"url"`
-	Headers     map[string]interface{} `gorm:"headers" json:"headers"`
-	RequestBody string                 `gorm:"db_username" json:"dbUsername"`
-	Auth        string                 `gorm:"db_password" json:"dbPassword"`
+	Host     string `gorm:"host" json:"host"`
+	Port     int64  `gorm:"port" json:"port"`
+	Dbname   string `gorm:"dbname" json:"dbname"`
+	Username string `gorm:"username" json:"username"`
+	Password string `gorm:"password" json:"password"`
+	Config   string `gorm:"config" json:"config"` //额外的链接参数
+	Schema   string `gorm:"schema" json:"schema"`
 }
 
 func (VisualDataSource) TableName() string {
 	return "visual_data_source"
 }
+
+func (a VisualDb) Value() (driver.Value, error) {
+	return json.Marshal(a)
+}
+func (a *VisualDb) Scan(value interface{}) error {
+	b, ok := value.([]byte)
+	if !ok {
+		return errors.New("type assertion to []byte failed")
+	}
+	return json.Unmarshal(b, &a)
+}

apps/visual/router/data_source.go

@@ -26,7 +26,7 @@ func InitVisualDataSourceRouter(container *restful.Container) {
 	tags := []string{"datasource"}
 
 	ws.Route(ws.GET("/list").To(func(request *restful.Request, response *restful.Response) {
-		restfulx.NewReqCtx(request, response).WithLog("获取DataSource分页列表").Handle(s.GetVisualDataSourceList)
+		restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("获取DataSource分页列表").Handle(s.GetVisualDataSourceList)
 	}).
 		Doc("获取DataSource分页列表").
 		Param(ws.QueryParameter("pageNum", "页数").Required(true).DataType("int")).
@@ -46,25 +46,46 @@ func InitVisualDataSourceRouter(container *restful.Container) {
 		Returns(404, "Not Found", nil))
 
 	ws.Route(ws.POST("").To(func(request *restful.Request, response *restful.Response) {
-		restfulx.NewReqCtx(request, response).WithLog("添加DataSource信息").Handle(s.InsertVisualDataSource)
+		restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("添加DataSource信息").Handle(s.InsertVisualDataSource)
 	}).
 		Doc("添加DataSource信息").
 		Metadata(restfulspec.KeyOpenAPITags, tags).
 		Reads(entity.VisualDataSource{}))
 
 	ws.Route(ws.PUT("").To(func(request *restful.Request, response *restful.Response) {
-		restfulx.NewReqCtx(request, response).WithLog("修改DataSource信息").Handle(s.UpdateVisualDataSource)
+		restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("修改DataSource信息").Handle(s.UpdateVisualDataSource)
 	}).
 		Doc("修改DataSource信息").
 		Metadata(restfulspec.KeyOpenAPITags, tags).
 		Reads(entity.VisualDataSource{}))
 
 	ws.Route(ws.DELETE("/{sourceId}").To(func(request *restful.Request, response *restful.Response) {
-		restfulx.NewReqCtx(request, response).WithLog("删除DataSource信息").Handle(s.DeleteVisualDataSource)
+		restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("删除DataSource信息").Handle(s.DeleteVisualDataSource)
 	}).
 		Doc("删除DataSource信息").
 		Metadata(restfulspec.KeyOpenAPITags, tags).
 		Param(ws.PathParameter("sourceId", "多id 1,2,3").DataType("string")))
 
+	ws.Route(ws.POST("/test").To(func(request *restful.Request, response *restful.Response) {
+		restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("验证数据源").Handle(s.GetDataSourceTest)
+	}).
+		Doc("验证数据源").
+		Metadata(restfulspec.KeyOpenAPITags, tags))
+
+	ws.Route(ws.GET("/tables/{sourceId}").To(func(request *restful.Request, response *restful.Response) {
+		restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("验证数据源下所有表").Handle(s.GetDataSourceTables)
+	}).
+		Doc("验证数据源下所有表").
+		Param(ws.PathParameter("sourceId", "sourceId").DataType("string")).
+		Metadata(restfulspec.KeyOpenAPITags, tags))
+
+	ws.Route(ws.GET("/tablesDetails/{sourceId}").To(func(request *restful.Request, response *restful.Response) {
+		restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("验证数据源下所有表").Handle(s.GetDataSourceTableDetails)
+	}).
+		Doc("验证数据源下所有表").
+		Param(ws.PathParameter("sourceId", "sourceId").DataType("string")).
+		Param(ws.QueryParameter("tableName", "表明").Required(true).DataType("string")).
+		Metadata(restfulspec.KeyOpenAPITags, tags))
+
 	container.Add(ws)
 }

apps/visual/services/data_source.go

@@ -55,9 +55,6 @@ func (m *datasourceModelImpl) FindListPage(page, pageSize int, data entity.Visua
 		db = db.Where("source_id = ?", data.SourceId)
 	}
 	db.Where("delete_time IS NULL")
-	if data.SourceComment != "" {
-		db = db.Where("source_comment = ?", data.SourceComment)
-	}
 	if data.SourceType != "" {
 		db = db.Where("source_type = ?", data.SourceType)
 	}
@@ -67,9 +64,6 @@ func (m *datasourceModelImpl) FindListPage(page, pageSize int, data entity.Visua
 	if data.Status != "" {
 		db = db.Where("status = ?", data.Status)
 	}
-	if data.Configuration != "" {
-		db = db.Where("configuration = ?", data.Configuration)
-	}
 	err := db.Count(&total).Error
 	err = db.Order("create_time").Limit(pageSize).Offset(offset).Find(&list).Error
 	biz.ErrIsNil(err, "查询数据源分页列表失败")
@@ -84,9 +78,6 @@ func (m *datasourceModelImpl) FindList(data entity.VisualDataSource) *[]entity.V
 		db = db.Where("source_id = ?", data.SourceId)
 	}
 	db.Where("delete_time IS NULL")
-	if data.SourceComment != "" {
-		db = db.Where("source_comment = ?", data.SourceComment)
-	}
 	if data.SourceType != "" {
 		db = db.Where("source_type = ?", data.SourceType)
 	}
@@ -96,9 +87,6 @@ func (m *datasourceModelImpl) FindList(data entity.VisualDataSource) *[]entity.V
 	if data.Status != "" {
 		db = db.Where("status = ?", data.Status)
 	}
-	if data.Configuration != "" {
-		db = db.Where("configuration = ?", data.Configuration)
-	}
 	biz.ErrIsNil(db.Order("create_time").Find(&list).Error, "查询数据源列表失败")
 	return &list
 }

go.mod

@@ -46,7 +46,7 @@ require (
 	github.com/go-openapi/swag v0.19.15 // indirect
 	github.com/go-redis/redis v6.15.9+incompatible // indirect
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
-	github.com/go-sql-driver/mysql v1.6.0 // indirect
+	github.com/go-sql-driver/mysql v1.7.0 // indirect
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect

go.sum

@@ -113,6 +113,8 @@ github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5Nq
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
+github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=

pkg/initialize/router.go

@@ -57,6 +57,9 @@ func InitRouter() *transport.HttpServer {
 		visualRouter.InitRuleChainRouter(container)
 		visualRouter.InitVisualScreenGroupRouter(container)
 		visualRouter.InitVisualScreenRouter(container)
+		visualRouter.InitVisualDataSourceRouter(container)
+		visualRouter.InitVisualDataSetTableRouter(container)
+		visualRouter.InitVisualDataSetFieldRouter(container)
 	}
 	// 任务
 	{

pkg/rule_engine/tool/email.go

@@ -1 +0,0 @@
-package tool

pkg/rule_engine/tool/kafka.go

@@ -1 +0,0 @@
-package tool

pkg/rule_engine/tool/mq.go

@@ -1 +0,0 @@
-package tool

pkg/rule_engine/tool/mqtt.go

@@ -1 +0,0 @@
-package tool

pkg/rule_engine/tool/rest_api.go

@@ -1 +0,0 @@
-package tool

pkg/rule_engine/tool/sms.go

@@ -1 +0,0 @@
-package tool