diff --git a/apps/visual/api/data_source.go b/apps/visual/api/data_source.go index c45dcae..f6734f3 100644 --- a/apps/visual/api/data_source.go +++ b/apps/visual/api/data_source.go @@ -6,8 +6,11 @@ package api // 生成人:panda // ========================================================================== import ( + "github.com/XM-GO/PandaKit/biz" "github.com/XM-GO/PandaKit/model" "github.com/XM-GO/PandaKit/restfulx" + "github.com/kakuilan/kgo" + "pandax/apps/visual/driver" "strings" "pandax/apps/visual/entity" @@ -24,6 +27,7 @@ func (p *VisualDataSourceApi) GetVisualDataSourceList(rc *restfulx.ReqCtx) { pageNum := restfulx.QueryInt(rc, "pageNum", 1) pageSize := restfulx.QueryInt(rc, "pageSize", 10) data.SourceName = restfulx.QueryParam(rc, "sourceName") + data.SourceType = restfulx.QueryParam(rc, "sourceType") data.Status = restfulx.QueryParam(rc, "status") list, total := p.VisualDataSourceApp.FindListPage(pageNum, pageSize, data) @@ -46,7 +50,13 @@ func (p *VisualDataSourceApi) GetVisualDataSource(rc *restfulx.ReqCtx) { func (p *VisualDataSourceApi) InsertVisualDataSource(rc *restfulx.ReqCtx) { var data entity.VisualDataSource restfulx.BindQuery(rc, &data) - + data.SourceId = kgo.KStr.Uniqid("px") + err := driver.TestConnection(&data) + if err != nil { + data.Status = "0" + } else { + data.Status = "1" + } p.VisualDataSourceApp.Insert(data) } @@ -64,3 +74,30 @@ func (p *VisualDataSourceApi) DeleteVisualDataSource(rc *restfulx.ReqCtx) { sourceIds := strings.Split(sourceId, ",") p.VisualDataSourceApp.Delete(sourceIds) } + +// GetDataSourceTest 校验数据源连接性 +func (p *VisualDataSourceApi) GetDataSourceTest(rc *restfulx.ReqCtx) { + var data entity.VisualDataSource + restfulx.BindQuery(rc, &data) + err := driver.TestConnection(&data) + biz.ErrIsNilAppendErr(err, "数据库连接失败: %s") +} + +// GetDataSourceTables 获取数据源下所有表 +func (p *VisualDataSourceApi) GetDataSourceTables(rc *restfulx.ReqCtx) { + sourceId := restfulx.PathParam(rc, "sourceId") + one := p.VisualDataSourceApp.FindOne(sourceId) + instance := driver.NewDbInstance(one) + biz.IsTrue(instance != nil, "获取数据源下所有表失败") + rc.ResData = instance.GetMeta().GetTableInfos() +} + +// GetDataSourceTableDetails 获取表下面的所有字段 +func (p *VisualDataSourceApi) GetDataSourceTableDetails(rc *restfulx.ReqCtx) { + sourceId := restfulx.PathParam(rc, "sourceId") + tableName := restfulx.QueryParam(rc, "tableName") + one := p.VisualDataSourceApp.FindOne(sourceId) + instance := driver.NewDbInstance(one) + biz.IsTrue(instance != nil, "获取表下所有字段失败") + rc.ResData = instance.GetMeta().GetColumns(tableName) +} diff --git a/apps/visual/driver/DbInstance.go b/apps/visual/driver/DbInstance.go new file mode 100644 index 0000000..9cf7277 --- /dev/null +++ b/apps/visual/driver/DbInstance.go @@ -0,0 +1,270 @@ +package driver + +import ( + "database/sql" + "fmt" + "github.com/XM-GO/PandaKit/cache" + "pandax/apps/visual/entity" + "pandax/pkg/global" + "reflect" + "strconv" + "strings" + "time" +) + +// db实例 +type DbInstance struct { + Id string + Info *entity.VisualDataSource + db *sql.DB +} + +func NewDbInstance(source *entity.VisualDataSource) *DbInstance { + byCache := GetDbInstanceByCache(source.SourceId) + if byCache != nil { + return byCache + } + conn, err := GetDbConn(source) + if err != nil { + return nil + } + di := &DbInstance{ + Id: source.SourceId, + Info: source, + db: conn, + } + AddDbInstanceToCache(source.SourceId, di) + return di +} + +// 执行查询语句 +// 依次返回 列名数组,结果map,错误 +func (d *DbInstance) SelectData(execSql string) ([]string, []map[string]interface{}, error) { + return SelectDataByDb(d.db, execSql) +} + +// 将查询结果映射至struct,可具体参考sqlx库 +func (d *DbInstance) SelectData2Struct(execSql string, dest interface{}) error { + return Select2StructByDb(d.db, execSql, dest) +} + +// 执行内部查询语句,不返回列名以及不限制行数 +// 依次返回 结果map,错误 +func (d *DbInstance) innerSelect(execSql string) ([]map[string]interface{}, error) { + _, res, err := SelectDataByDb(d.db, execSql) + return res, err +} + +// 执行 update, insert, delete,建表等sql +// 返回影响条数和错误 +func (d *DbInstance) Exec(sql string) (int64, error) { + res, err := d.db.Exec(sql) + if err != nil { + return 0, err + } + return res.RowsAffected() +} + +// 获取数据库元信息实现接口 +func (di *DbInstance) GetMeta() DbMetadata { + dbType := di.Info.SourceType + if dbType == entity.DbTypeMysql { + return &MysqlMetadata{di: di} + } + if dbType == entity.DbTypePostgres { + return &PgsqlMetadata{di: di} + } + return nil +} + +// 关闭连接 +func (d *DbInstance) Close() { + if d.db != nil { + if err := d.db.Close(); err != nil { + global.Log.Errorf("关闭数据库实例[%s]连接失败: %s", d.Id, err.Error()) + } + d.db = nil + } +} + +const DbConnExpireTime = 45 * time.Minute + +//------------------------------------------------------------------------------ + +// 客户端连接缓存,指定时间内没有访问则会被关闭, key为数据库实例id:数据库 +var dbCache = cache.NewTimedCache(DbConnExpireTime, 5*time.Second). + WithUpdateAccessTime(true). + OnEvicted(func(key interface{}, value interface{}) { + global.Log.Info(fmt.Sprintf("删除db连接缓存 id = %s", key)) + value.(*DbInstance).Close() + }) + +func GetDbCacheKey(dbId uint64, db string) string { + return fmt.Sprintf("%d:%s", dbId, db) +} + +// 删除db缓存并关闭该数据库所有连接 +func CloseDb(id string) { + dbCache.Delete(id) +} +func GetDbInstanceByCache(id string) *DbInstance { + if load, ok := dbCache.Get(id); ok { + return load.(*DbInstance) + } + return nil +} + +// 将实体添加到缓存中 +func AddDbInstanceToCache(id string, di *DbInstance) { + dbCache.AddIfAbsent(id, di) +} + +func TestConnection(d *entity.VisualDataSource) error { + // 验证第一个库是否可以连接即可 + DB, err := GetDbConn(d) + if err != nil { + return err + } else { + DB.Close() + return nil + } +} + +// 获取数据库连接 +func GetDbConn(d *entity.VisualDataSource) (*sql.DB, error) { + var DB *sql.DB + var err error + if d.SourceType == entity.DbTypeMysql { + DB, err = getMysqlDB(d) + } else if d.SourceType == entity.DbTypePostgres { + DB, err = getPgsqlDB(d) + } + + if err != nil { + return nil, err + } + err = DB.Ping() + if err != nil { + DB.Close() + return nil, err + } + + return DB, nil +} + +func SelectDataByDb(db *sql.DB, selectSql string) ([]string, []map[string]interface{}, error) { + rows, err := db.Query(selectSql) + if err != nil { + return nil, nil, err + } + // rows对象一定要close掉,如果出错,不关掉则会很迅速的达到设置最大连接数, + // 后面的链接过来直接报错或拒绝,实际上也没有起效果 + defer func() { + if rows != nil { + rows.Close() + } + }() + colTypes, _ := rows.ColumnTypes() + // 这里表示一行填充数据 + scans := make([]interface{}, len(colTypes)) + // 这里表示一行所有列的值,用[]byte表示 + vals := make([][]byte, len(colTypes)) + // 这里scans引用vals,把数据填充到[]byte里 + for k := range vals { + scans[k] = &vals[k] + } + + result := make([]map[string]interface{}, 0) + // 列名用于前端表头名称按照数据库与查询字段顺序显示 + colNames := make([]string, 0) + // 是否第一次遍历,列名数组只需第一次遍历时加入 + isFirst := true + for rows.Next() { + // 不Scan也会导致等待,该链接实际处于未工作的状态,然后也会导致连接数迅速达到最大 + err := rows.Scan(scans...) + if err != nil { + return nil, nil, err + } + // 每行数据 + rowData := make(map[string]interface{}) + // 把vals中的数据复制到row中 + for i, v := range vals { + colType := colTypes[i] + colName := colType.Name() + // 如果是第一行,则将列名加入到列信息中,由于map是无序的,所有需要返回列名的有序数组 + if isFirst { + colNames = append(colNames, colName) + } + rowData[colName] = valueConvert(v, colType) + } + // 放入结果集 + result = append(result, rowData) + isFirst = false + } + return colNames, result, nil +} + +// 将查询的值转为对应列类型的实际值,不全部转为字符串 +func valueConvert(data []byte, colType *sql.ColumnType) interface{} { + if data == nil { + return nil + } + // 列的数据库类型名 + colDatabaseTypeName := strings.ToLower(colType.DatabaseTypeName()) + + // 如果类型是bit,则直接返回第一个字节即可 + if strings.Contains(colDatabaseTypeName, "bit") { + return data[0] + } + + // 这里把[]byte数据转成string + stringV := string(data) + if stringV == "" { + return "" + } + colScanType := strings.ToLower(colType.ScanType().Name()) + + if strings.Contains(colScanType, "int") { + // 如果长度超过16位,则返回字符串,因为前端js长度大于16会丢失精度 + if len(stringV) > 16 { + return stringV + } + intV, _ := strconv.Atoi(stringV) + switch colType.ScanType().Kind() { + case reflect.Int8: + return int8(intV) + case reflect.Uint8: + return uint8(intV) + case reflect.Int64: + return int64(intV) + case reflect.Uint64: + return uint64(intV) + case reflect.Uint: + return uint(intV) + default: + return intV + } + } + if strings.Contains(colScanType, "float") || strings.Contains(colDatabaseTypeName, "decimal") { + floatV, _ := strconv.ParseFloat(stringV, 64) + return floatV + } + + return stringV +} + +// 查询数据结果映射至struct。可参考sqlx库 +func Select2StructByDb(db *sql.DB, selectSql string, dest interface{}) error { + rows, err := db.Query(selectSql) + if err != nil { + return err + } + // rows对象一定要close掉,如果出错,不关掉则会很迅速的达到设置最大连接数, + // 后面的链接过来直接报错或拒绝,实际上也没有起效果 + defer func() { + if rows != nil { + rows.Close() + } + }() + return scanAll(rows, dest, false) +} diff --git a/apps/visual/driver/meta.go b/apps/visual/driver/meta.go new file mode 100644 index 0000000..096b108 --- /dev/null +++ b/apps/visual/driver/meta.go @@ -0,0 +1,9 @@ +package driver + +type DbMetadata interface { + GetTables() []map[string]interface{} + GetColumns(tableNames ...string) []map[string]interface{} + GetPrimaryKey(tableName string) string + GetTableInfos() []map[string]interface{} + GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error) +} diff --git a/apps/visual/driver/mysql_meta.go b/apps/visual/driver/mysql_meta.go new file mode 100644 index 0000000..52e5922 --- /dev/null +++ b/apps/visual/driver/mysql_meta.go @@ -0,0 +1,118 @@ +package driver + +import ( + "database/sql" + "fmt" + "github.com/XM-GO/PandaKit/biz" + "log" + "pandax/apps/visual/entity" + + _ "github.com/go-sql-driver/mysql" +) + +func getMysqlDB(d *entity.VisualDataSource) (*sql.DB, error) { + // SSH Conect + // 设置dataSourceName -> 更多参数参考:https://github.com/go-sql-driver/mysql#dsn-data-source-name + dsn := fmt.Sprintf("%s:%s@%s(%s:%d)/%s?timeout=8s", + d.Db.Username, d.Db.Password, "tcp", d.Db.Host, d.Db.Port, d.Db.Dbname) + if d.Db.Config != "" { + dsn = fmt.Sprintf("%s&%s", dsn, d.Db.Config) + } + log.Println(dsn) + return sql.Open("mysql", dsn) +} + +// ---------------------------------- mysql元数据 ----------------------------------- +const ( + // mysql 表信息元数据 + MYSQL_TABLE_MA = `SELECT table_name tableName, table_comment tableComment from information_schema.tables WHERE table_schema = (SELECT database())` + + // mysql 表信息 + MYSQL_TABLE_INFO = `SELECT table_name tableName, table_comment tableComment, table_rows tableRows, + data_length dataLength, index_length indexLength, create_time createTime + FROM information_schema.tables + WHERE table_schema = (SELECT database())` + + // mysql 索引信息 + MYSQL_INDEX_INFO = `SELECT index_name indexName, column_name columnName, index_type indexType, non_unique nonUnique, + SEQ_IN_INDEX seqInIndex, INDEX_COMMENT indexComment + FROM information_schema.STATISTICS + WHERE table_schema = (SELECT database()) AND table_name = '%s' ORDER BY index_name asc , SEQ_IN_INDEX asc` + + // mysql 列信息元数据 + MYSQL_COLUMN_MA = `SELECT table_name tableName, column_name columnName, column_type columnType, column_default columnDefault, + column_comment columnComment, column_key columnKey, extra, is_nullable nullable from information_schema.columns + WHERE table_schema = (SELECT database()) AND table_name in (%s) ORDER BY tableName, ordinal_position` +) + +type MysqlMetadata struct { + di *DbInstance +} + +// 获取表基础元信息, 如表名等 +func (mm *MysqlMetadata) GetTables() []map[string]interface{} { + res, err := mm.di.innerSelect(MYSQL_TABLE_MA) + biz.ErrIsNilAppendErr(err, "获取表基本信息失败: %s") + return res +} + +// 获取列元信息, 如列名等 +func (mm *MysqlMetadata) GetColumns(tableNames ...string) []map[string]interface{} { + tableName := "" + for i := 0; i < len(tableNames); i++ { + if i != 0 { + tableName = tableName + ", " + } + tableName = tableName + "'" + tableNames[i] + "'" + } + result, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_COLUMN_MA, tableName)) + biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s") + return result +} + +// 获取表主键字段名,不存在主键标识则默认第一个字段 +func (mm *MysqlMetadata) GetPrimaryKey(tablename string) string { + columns := mm.GetColumns(tablename) + biz.IsTrue(len(columns) > 0, "[%s] 表不存在", tablename) + for _, v := range columns { + if v["columnKey"].(string) == "PRI" { + return v["columnName"].(string) + } + } + + return columns[0]["columnName"].(string) +} + +// 获取表信息,比GetTableMetedatas获取更详细的表信息 +func (mm *MysqlMetadata) GetTableInfos() []map[string]interface{} { + res, err := mm.di.innerSelect(MYSQL_TABLE_INFO) + biz.ErrIsNilAppendErr(err, "获取表信息失败: %s") + return res +} + +// 获取表索引信息 +func (mm *MysqlMetadata) GetTableIndex(tableName string) []map[string]interface{} { + res, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_INDEX_INFO, tableName)) + biz.ErrIsNilAppendErr(err, "获取表索引信息失败: %s") + // 把查询结果以索引名分组,索引字段以逗号连接 + result := make([]map[string]interface{}, 0) + key := "" + for _, v := range res { + // 当前的索引名 + in := v["indexName"].(string) + if key == in { + // 索引字段已根据名称和顺序排序,故取最后一个即可 + i := len(result) - 1 + // 同索引字段以逗号连接 + result[i]["columnName"] = result[i]["columnName"].(string) + "," + v["columnName"].(string) + } else { + key = in + result = append(result, v) + } + } + return result +} + +func (mm *MysqlMetadata) GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error) { + return mm.di.SelectData(fmt.Sprintf("SELECT * FROM %s LIMIT %d, %d", tableName, (pageNum-1)*pageSize, pageSize)) +} diff --git a/apps/visual/driver/pgsql_meta.go b/apps/visual/driver/pgsql_meta.go new file mode 100644 index 0000000..e34933e --- /dev/null +++ b/apps/visual/driver/pgsql_meta.go @@ -0,0 +1,111 @@ +package driver + +import ( + "database/sql" + "fmt" + "github.com/XM-GO/PandaKit/biz" + "pandax/apps/visual/entity" + "strings" + + _ "github.com/lib/pq" +) + +func getPgsqlDB(d *entity.VisualDataSource) (*sql.DB, error) { + dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", + d.Db.Host, d.Db.Port, d.Db.Username, d.Db.Password, d.Db.Dbname) + if d.Db.Config != "" { + dsn = fmt.Sprintf("%s %s", dsn, strings.Join(strings.Split(d.Db.Config, "&"), " ")) + } + return sql.Open("postgres", dsn) +} + +// ---------------------------------- pgsql元数据 ----------------------------------- +const ( + // postgres 表信息元数据 + PGSQL_TABLE_MA = `SELECT obj_description(c.oid) AS "tableComment", c.relname AS "tableName" FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid WHERE n.nspname = (select current_schema()) AND c.reltype > 0` + + PGSQL_TABLE_INFO = `SELECT obj_description(c.oid) AS "tableComment", c.relname AS "tableName" FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid WHERE n.nspname = (select current_schema()) AND c.reltype > 0` + + PGSQL_INDEX_INFO = `SELECT indexname AS "indexName", indexdef AS "indexComment" + FROM pg_indexes WHERE schemaname = (select current_schema()) AND tablename = '%s'` + + PGSQL_COLUMN_MA = `SELECT + C.relname AS "tableName", + A.attname AS "columnName", + tc.is_nullable AS "nullable", + concat_ws ( '', t.typname, SUBSTRING ( format_type ( a.atttypid, a.atttypmod ) FROM '\(.*\)' ) ) AS "columnType", + (CASE WHEN ( SELECT COUNT(*) FROM pg_constraint WHERE conrelid = a.attrelid AND conkey[1]= attnum AND contype = 'p' ) > 0 THEN 'PRI' ELSE '' END ) AS "columnKey", + d.description AS "columnComment" + FROM + pg_attribute a LEFT JOIN pg_description d ON d.objoid = a.attrelid + AND d.objsubid = A.attnum + LEFT JOIN pg_class c ON A.attrelid = c.oid + LEFT JOIN pg_namespace pn ON c.relnamespace = pn.oid + LEFT JOIN pg_type t ON a.atttypid = t.oid + JOIN information_schema.columns tc ON tc.column_name = a.attname AND tc.table_name = C.relname AND tc.table_schema = pn.nspname + WHERE + A.attnum >= 0 + AND pn.nspname = (select current_schema()) + AND C.relname in (%s) + ORDER BY + C.relname DESC, + A.attnum ASC + ` +) + +type PgsqlMetadata struct { + di *DbInstance +} + +// 获取表基础元信息, 如表名等 +func (pm *PgsqlMetadata) GetTables() []map[string]interface{} { + res, err := pm.di.innerSelect(PGSQL_TABLE_MA) + biz.ErrIsNilAppendErr(err, "获取表基本信息失败: %s") + return res +} + +// 获取列元信息, 如列名等 +func (pm *PgsqlMetadata) GetColumns(tableNames ...string) []map[string]interface{} { + tableName := "" + for i := 0; i < len(tableNames); i++ { + if i != 0 { + tableName = tableName + ", " + } + tableName = tableName + "'" + tableNames[i] + "'" + } + result, err := pm.di.innerSelect(fmt.Sprintf(PGSQL_COLUMN_MA, tableName)) + biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s") + return result +} + +func (pm *PgsqlMetadata) GetPrimaryKey(tablename string) string { + columns := pm.GetColumns(tablename) + biz.IsTrue(len(columns) > 0, "[%s] 表不存在", tablename) + for _, v := range columns { + if v["columnKey"].(string) == "PRI" { + return v["columnName"].(string) + } + } + + return columns[0]["columnName"].(string) +} + +// 获取表信息,比GetTables获取更详细的表信息 +func (pm *PgsqlMetadata) GetTableInfos() []map[string]interface{} { + res, err := pm.di.innerSelect(PGSQL_TABLE_INFO) + biz.ErrIsNilAppendErr(err, "获取表信息失败: %s") + return res +} + +// 获取表索引信息 +func (pm *PgsqlMetadata) GetTableIndex(tableName string) []map[string]interface{} { + res, err := pm.di.innerSelect(fmt.Sprintf(PGSQL_INDEX_INFO, tableName)) + biz.ErrIsNilAppendErr(err, "获取表索引信息失败: %s") + return res +} + +func (pm *PgsqlMetadata) GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error) { + return pm.di.SelectData(fmt.Sprintf("SELECT * FROM %s OFFSET %d LIMIT %d", tableName, (pageNum-1)*pageSize, pageSize)) +} diff --git a/apps/visual/driver/sqlx.go b/apps/visual/driver/sqlx.go new file mode 100644 index 0000000..ddd4bd5 --- /dev/null +++ b/apps/visual/driver/sqlx.go @@ -0,0 +1,630 @@ +package driver + +import ( + "database/sql" + "errors" + "fmt" + "reflect" + "runtime" + "strings" + "sync" +) + +// 将结果scan至结构体,copy至 sqlx库: https://github.com/jmoiron/sqlx +func scanAll(rows *sql.Rows, dest interface{}, structOnly bool) error { + var v, vp reflect.Value + + value := reflect.ValueOf(dest) + + // json.Unmarshal returns errors for these + if value.Kind() != reflect.Ptr { + return errors.New("must pass a pointer, not a value, to StructScan destination") + } + if value.IsNil() { + return errors.New("nil pointer passed to StructScan destination") + } + direct := reflect.Indirect(value) + + slice, err := baseType(value.Type(), reflect.Slice) + if err != nil { + return err + } + direct.SetLen(0) + + isPtr := slice.Elem().Kind() == reflect.Ptr + base := Deref(slice.Elem()) + scannable := isScannable(base) + + if structOnly && scannable { + return structOnlyError(base) + } + + columns, err := rows.Columns() + if err != nil { + return err + } + + // if it's a base type make sure it only has 1 column; if not return an error + if scannable && len(columns) > 1 { + return fmt.Errorf("non-struct dest type %s with >1 columns (%d)", base.Kind(), len(columns)) + } + + if !scannable { + var values []interface{} + var m *Mapper = mapper() + + fields := m.TraversalsByName(base, columns) + // if we are not unsafe and are missing fields, return an error + if f, err := missingFields(fields); err != nil { + return fmt.Errorf("missing destination name %s in %T", columns[f], dest) + } + values = make([]interface{}, len(columns)) + + for rows.Next() { + // create a new struct type (which returns PtrTo) and indirect it + vp = reflect.New(base) + v = reflect.Indirect(vp) + + err = fieldsByTraversal(v, fields, values, true) + if err != nil { + return err + } + + // scan into the struct field pointers and append to our results + err = rows.Scan(values...) + if err != nil { + return err + } + + if isPtr { + direct.Set(reflect.Append(direct, vp)) + } else { + direct.Set(reflect.Append(direct, v)) + } + } + } else { + for rows.Next() { + vp = reflect.New(base) + err = rows.Scan(vp.Interface()) + if err != nil { + return err + } + // append + if isPtr { + direct.Set(reflect.Append(direct, vp)) + } else { + direct.Set(reflect.Append(direct, reflect.Indirect(vp))) + } + } + } + + return rows.Err() +} + +func baseType(t reflect.Type, expected reflect.Kind) (reflect.Type, error) { + t = Deref(t) + if t.Kind() != expected { + return nil, fmt.Errorf("expected %s but got %s", expected, t.Kind()) + } + return t, nil +} + +// structOnlyError returns an error appropriate for type when a non-scannable +// struct is expected but something else is given +func structOnlyError(t reflect.Type) error { + isStruct := t.Kind() == reflect.Struct + isScanner := reflect.PtrTo(t).Implements(_scannerInterface) + if !isStruct { + return fmt.Errorf("expected %s but got %s", reflect.Struct, t.Kind()) + } + if isScanner { + return fmt.Errorf("structscan expects a struct dest but the provided struct type %s implements scanner", t.Name()) + } + return fmt.Errorf("expected a struct, but struct %s has no exported fields", t.Name()) +} + +var _scannerInterface = reflect.TypeOf((*sql.Scanner)(nil)).Elem() + +func isScannable(t reflect.Type) bool { + if reflect.PtrTo(t).Implements(_scannerInterface) { + return true + } + if t.Kind() != reflect.Struct { + return true + } + + // it's not important that we use the right mapper for this particular object, + // we're only concerned on how many exported fields this struct has + return len(mapper().TypeMap(t).Index) == 0 +} + +var NameMapper = strings.ToLower +var origMapper = reflect.ValueOf(NameMapper) + +// Rather than creating on init, this is created when necessary so that +// importers have time to customize the NameMapper. +var mpr *Mapper + +// mprMu protects mpr. +var mprMu sync.Mutex + +// mapper returns a valid mapper using the configured NameMapper func. +func mapper() *Mapper { + mprMu.Lock() + defer mprMu.Unlock() + + if mpr == nil { + mpr = NewMapperFunc("db", NameMapper) + } else if origMapper != reflect.ValueOf(NameMapper) { + // if NameMapper has changed, create a new mapper + mpr = NewMapperFunc("db", NameMapper) + origMapper = reflect.ValueOf(NameMapper) + } + return mpr +} + +func missingFields(transversals [][]int) (field int, err error) { + for i, t := range transversals { + if len(t) == 0 { + return i, errors.New("missing field") + } + } + return 0, nil +} + +// fieldsByName fills a values interface with fields from the passed value based +// on the traversals in int. If ptrs is true, return addresses instead of values. +// We write this instead of using FieldsByName to save allocations and map lookups +// when iterating over many rows. Empty traversals will get an interface pointer. +// Because of the necessity of requesting ptrs or values, it's considered a bit too +// specialized for inclusion in reflectx itself. +func fieldsByTraversal(v reflect.Value, traversals [][]int, values []interface{}, ptrs bool) error { + v = reflect.Indirect(v) + if v.Kind() != reflect.Struct { + return errors.New("argument not a struct") + } + + for i, traversal := range traversals { + if len(traversal) == 0 { + values[i] = new(interface{}) + continue + } + f := FieldByIndexes(v, traversal) + if ptrs { + values[i] = f.Addr().Interface() + } else { + values[i] = f.Interface() + } + } + return nil +} + +// A FieldInfo is metadata for a struct field. +type FieldInfo struct { + Index []int + Path string + Field reflect.StructField + Zero reflect.Value + Name string + Options map[string]string + Embedded bool + Children []*FieldInfo + Parent *FieldInfo +} + +// A StructMap is an index of field metadata for a struct. +type StructMap struct { + Tree *FieldInfo + Index []*FieldInfo + Paths map[string]*FieldInfo + Names map[string]*FieldInfo +} + +// GetByPath returns a *FieldInfo for a given string path. +func (f StructMap) GetByPath(path string) *FieldInfo { + return f.Paths[path] +} + +// GetByTraversal returns a *FieldInfo for a given integer path. It is +// analogous to reflect.FieldByIndex, but using the cached traversal +// rather than re-executing the reflect machinery each time. +func (f StructMap) GetByTraversal(index []int) *FieldInfo { + if len(index) == 0 { + return nil + } + + tree := f.Tree + for _, i := range index { + if i >= len(tree.Children) || tree.Children[i] == nil { + return nil + } + tree = tree.Children[i] + } + return tree +} + +// Mapper is a general purpose mapper of names to struct fields. A Mapper +// behaves like most marshallers in the standard library, obeying a field tag +// for name mapping but also providing a basic transform function. +type Mapper struct { + cache map[reflect.Type]*StructMap + tagName string + tagMapFunc func(string) string + mapFunc func(string) string + mutex sync.Mutex +} + +// NewMapper returns a new mapper using the tagName as its struct field tag. +// If tagName is the empty string, it is ignored. +func NewMapper(tagName string) *Mapper { + return &Mapper{ + cache: make(map[reflect.Type]*StructMap), + tagName: tagName, + } +} + +// NewMapperTagFunc returns a new mapper which contains a mapper for field names +// AND a mapper for tag values. This is useful for tags like json which can +// have values like "name,omitempty". +func NewMapperTagFunc(tagName string, mapFunc, tagMapFunc func(string) string) *Mapper { + return &Mapper{ + cache: make(map[reflect.Type]*StructMap), + tagName: tagName, + mapFunc: mapFunc, + tagMapFunc: tagMapFunc, + } +} + +// NewMapperFunc returns a new mapper which optionally obeys a field tag and +// a struct field name mapper func given by f. Tags will take precedence, but +// for any other field, the mapped name will be f(field.Name) +func NewMapperFunc(tagName string, f func(string) string) *Mapper { + return &Mapper{ + cache: make(map[reflect.Type]*StructMap), + tagName: tagName, + mapFunc: f, + } +} + +// TypeMap returns a mapping of field strings to int slices representing +// the traversal down the struct to reach the field. +func (m *Mapper) TypeMap(t reflect.Type) *StructMap { + m.mutex.Lock() + mapping, ok := m.cache[t] + if !ok { + mapping = getMapping(t, m.tagName, m.mapFunc, m.tagMapFunc) + m.cache[t] = mapping + } + m.mutex.Unlock() + return mapping +} + +// FieldMap returns the mapper's mapping of field names to reflect values. Panics +// if v's Kind is not Struct, or v is not Indirectable to a struct kind. +func (m *Mapper) FieldMap(v reflect.Value) map[string]reflect.Value { + v = reflect.Indirect(v) + mustBe(v, reflect.Struct) + + r := map[string]reflect.Value{} + tm := m.TypeMap(v.Type()) + for tagName, fi := range tm.Names { + r[tagName] = FieldByIndexes(v, fi.Index) + } + return r +} + +// FieldByName returns a field by its mapped name as a reflect.Value. +// Panics if v's Kind is not Struct or v is not Indirectable to a struct Kind. +// Returns zero Value if the name is not found. +func (m *Mapper) FieldByName(v reflect.Value, name string) reflect.Value { + v = reflect.Indirect(v) + mustBe(v, reflect.Struct) + + tm := m.TypeMap(v.Type()) + fi, ok := tm.Names[name] + if !ok { + return v + } + return FieldByIndexes(v, fi.Index) +} + +// FieldsByName returns a slice of values corresponding to the slice of names +// for the value. Panics if v's Kind is not Struct or v is not Indirectable +// to a struct Kind. Returns zero Value for each name not found. +func (m *Mapper) FieldsByName(v reflect.Value, names []string) []reflect.Value { + v = reflect.Indirect(v) + mustBe(v, reflect.Struct) + + tm := m.TypeMap(v.Type()) + vals := make([]reflect.Value, 0, len(names)) + for _, name := range names { + fi, ok := tm.Names[name] + if !ok { + vals = append(vals, *new(reflect.Value)) + } else { + vals = append(vals, FieldByIndexes(v, fi.Index)) + } + } + return vals +} + +// TraversalsByName returns a slice of int slices which represent the struct +// traversals for each mapped name. Panics if t is not a struct or Indirectable +// to a struct. Returns empty int slice for each name not found. +func (m *Mapper) TraversalsByName(t reflect.Type, names []string) [][]int { + r := make([][]int, 0, len(names)) + m.TraversalsByNameFunc(t, names, func(_ int, i []int) error { + if i == nil { + r = append(r, []int{}) + } else { + r = append(r, i) + } + + return nil + }) + return r +} + +// TraversalsByNameFunc traverses the mapped names and calls fn with the index of +// each name and the struct traversal represented by that name. Panics if t is not +// a struct or Indirectable to a struct. Returns the first error returned by fn or nil. +func (m *Mapper) TraversalsByNameFunc(t reflect.Type, names []string, fn func(int, []int) error) error { + t = Deref(t) + mustBe(t, reflect.Struct) + tm := m.TypeMap(t) + for i, name := range names { + fi, ok := tm.Names[name] + if !ok { + if err := fn(i, nil); err != nil { + return err + } + } else { + if err := fn(i, fi.Index); err != nil { + return err + } + } + } + return nil +} + +// FieldByIndexes returns a value for the field given by the struct traversal +// for the given value. +func FieldByIndexes(v reflect.Value, indexes []int) reflect.Value { + for _, i := range indexes { + v = reflect.Indirect(v).Field(i) + // if this is a pointer and it's nil, allocate a new value and set it + if v.Kind() == reflect.Ptr && v.IsNil() { + alloc := reflect.New(Deref(v.Type())) + v.Set(alloc) + } + if v.Kind() == reflect.Map && v.IsNil() { + v.Set(reflect.MakeMap(v.Type())) + } + } + return v +} + +// FieldByIndexesReadOnly returns a value for a particular struct traversal, +// but is not concerned with allocating nil pointers because the value is +// going to be used for reading and not setting. +func FieldByIndexesReadOnly(v reflect.Value, indexes []int) reflect.Value { + for _, i := range indexes { + v = reflect.Indirect(v).Field(i) + } + return v +} + +// Deref is Indirect for reflect.Types +func Deref(t reflect.Type) reflect.Type { + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + return t +} + +// -- helpers & utilities -- + +type kinder interface { + Kind() reflect.Kind +} + +// mustBe checks a value against a kind, panicing with a reflect.ValueError +// if the kind isn't that which is required. +func mustBe(v kinder, expected reflect.Kind) { + if k := v.Kind(); k != expected { + panic(&reflect.ValueError{Method: methodName(), Kind: k}) + } +} + +// methodName returns the caller of the function calling methodName +func methodName() string { + pc, _, _, _ := runtime.Caller(2) + f := runtime.FuncForPC(pc) + if f == nil { + return "unknown method" + } + return f.Name() +} + +type typeQueue struct { + t reflect.Type + fi *FieldInfo + pp string // Parent path +} + +// A copying append that creates a new slice each time. +func apnd(is []int, i int) []int { + x := make([]int, len(is)+1) + copy(x, is) + x[len(x)-1] = i + return x +} + +type mapf func(string) string + +// parseName parses the tag and the target name for the given field using +// the tagName (eg 'json' for `json:"foo"` tags), mapFunc for mapping the +// field's name to a target name, and tagMapFunc for mapping the tag to +// a target name. +func parseName(field reflect.StructField, tagName string, mapFunc, tagMapFunc mapf) (tag, fieldName string) { + // first, set the fieldName to the field's name + fieldName = field.Name + // if a mapFunc is set, use that to override the fieldName + if mapFunc != nil { + fieldName = mapFunc(fieldName) + } + + // if there's no tag to look for, return the field name + if tagName == "" { + return "", fieldName + } + + // if this tag is not set using the normal convention in the tag, + // then return the fieldname.. this check is done because according + // to the reflect documentation: + // If the tag does not have the conventional format, + // the value returned by Get is unspecified. + // which doesn't sound great. + if !strings.Contains(string(field.Tag), tagName+":") { + return "", fieldName + } + + // at this point we're fairly sure that we have a tag, so lets pull it out + tag = field.Tag.Get(tagName) + + // if we have a mapper function, call it on the whole tag + // XXX: this is a change from the old version, which pulled out the name + // before the tagMapFunc could be run, but I think this is the right way + if tagMapFunc != nil { + tag = tagMapFunc(tag) + } + + // finally, split the options from the name + parts := strings.Split(tag, ",") + fieldName = parts[0] + + return tag, fieldName +} + +// parseOptions parses options out of a tag string, skipping the name +func parseOptions(tag string) map[string]string { + parts := strings.Split(tag, ",") + options := make(map[string]string, len(parts)) + if len(parts) > 1 { + for _, opt := range parts[1:] { + // short circuit potentially expensive split op + if strings.Contains(opt, "=") { + kv := strings.Split(opt, "=") + options[kv[0]] = kv[1] + continue + } + options[opt] = "" + } + } + return options +} + +// getMapping returns a mapping for the t type, using the tagName, mapFunc and +// tagMapFunc to determine the canonical names of fields. +func getMapping(t reflect.Type, tagName string, mapFunc, tagMapFunc mapf) *StructMap { + m := []*FieldInfo{} + + root := &FieldInfo{} + queue := []typeQueue{} + queue = append(queue, typeQueue{Deref(t), root, ""}) + +QueueLoop: + for len(queue) != 0 { + // pop the first item off of the queue + tq := queue[0] + queue = queue[1:] + + // ignore recursive field + for p := tq.fi.Parent; p != nil; p = p.Parent { + if tq.fi.Field.Type == p.Field.Type { + continue QueueLoop + } + } + + nChildren := 0 + if tq.t.Kind() == reflect.Struct { + nChildren = tq.t.NumField() + } + tq.fi.Children = make([]*FieldInfo, nChildren) + + // iterate through all of its fields + for fieldPos := 0; fieldPos < nChildren; fieldPos++ { + + f := tq.t.Field(fieldPos) + + // parse the tag and the target name using the mapping options for this field + tag, name := parseName(f, tagName, mapFunc, tagMapFunc) + + // if the name is "-", disabled via a tag, skip it + if name == "-" { + continue + } + + fi := FieldInfo{ + Field: f, + Name: name, + Zero: reflect.New(f.Type).Elem(), + Options: parseOptions(tag), + } + + // if the path is empty this path is just the name + if tq.pp == "" { + fi.Path = fi.Name + } else { + fi.Path = tq.pp + "." + fi.Name + } + + // skip unexported fields + if len(f.PkgPath) != 0 && !f.Anonymous { + continue + } + + // bfs search of anonymous embedded structs + if f.Anonymous { + pp := tq.pp + if tag != "" { + pp = fi.Path + } + + fi.Embedded = true + fi.Index = apnd(tq.fi.Index, fieldPos) + nChildren := 0 + ft := Deref(f.Type) + if ft.Kind() == reflect.Struct { + nChildren = ft.NumField() + } + fi.Children = make([]*FieldInfo, nChildren) + queue = append(queue, typeQueue{Deref(f.Type), &fi, pp}) + } else if fi.Zero.Kind() == reflect.Struct || (fi.Zero.Kind() == reflect.Ptr && fi.Zero.Type().Elem().Kind() == reflect.Struct) { + fi.Index = apnd(tq.fi.Index, fieldPos) + fi.Children = make([]*FieldInfo, Deref(f.Type).NumField()) + queue = append(queue, typeQueue{Deref(f.Type), &fi, fi.Path}) + } + + fi.Index = apnd(tq.fi.Index, fieldPos) + fi.Parent = tq.fi + tq.fi.Children[fieldPos] = &fi + m = append(m, &fi) + } + } + + flds := &StructMap{Index: m, Tree: root, Paths: map[string]*FieldInfo{}, Names: map[string]*FieldInfo{}} + for _, fi := range flds.Index { + // check if nothing has already been pushed with the same path + // sometimes you can choose to override a type using embedded struct + fld, ok := flds.Paths[fi.Path] + if !ok || fld.Embedded { + flds.Paths[fi.Path] = fi + if fi.Name != "" && !fi.Embedded { + flds.Names[fi.Path] = fi + } + } + } + + return flds +} diff --git a/apps/visual/entity/data_set.go b/apps/visual/entity/data_set.go index fdb310a..b83005e 100644 --- a/apps/visual/entity/data_set.go +++ b/apps/visual/entity/data_set.go @@ -18,7 +18,7 @@ func (VisualDataSetGroup) TableName() string { type VisualDataSetTable struct { model.BaseModelD TableId string `gorm:"primary_key;tableId;comment:表id" json:"tableId"` - DataSourceId string `gorm:"dataSourceId;type:varchar(64);comment:数据圆ID" json:"dataSourceId"` + DataSourceId string `gorm:"dataSourceId;type:varchar(64);comment:数据源ID" json:"dataSourceId"` TableType string `gorm:"tableType;type:varchar(64);comment:db,sql,excel,custom,api" json:"tableType"` //'db,sql,excel,custom', Mode string `gorm:"mode;type:varchar(1);comment:原始表信息" json:"mode"` //'连接模式:0-直连,1-定时同步', Info string `gorm:"info;type:TEXT;comment:原始表信息" json:"info"` diff --git a/apps/visual/entity/data_source.go b/apps/visual/entity/data_source.go index b61fbe0..539eb10 100644 --- a/apps/visual/entity/data_source.go +++ b/apps/visual/entity/data_source.go @@ -1,35 +1,49 @@ package entity -import "github.com/XM-GO/PandaKit/model" +import ( + "database/sql/driver" + "encoding/json" + "errors" + "github.com/XM-GO/PandaKit/model" +) + +const ( + DbTypeMysql = "MySQL" + DbTypePostgres = "PostgreSQL" +) type VisualDataSource struct { model.BaseModel - SourceId string `gorm:"primary_key;source_id;comment:数据源Id" json:"sourceId"` // 数据源Id - SourceType string `gorm:"source_type;type:varchar(50);comment:数据源类型" json:"sourceType"` // 数据源类型 - SourceName string `gorm:"source_name;type:varchar(50);comment:数据源名称" json:"sourceName"` // 原名称 - SourceComment string `gorm:"source_comment;type:varchar(50);comment:数据源描述" json:"source_comment"` // 描述 - Status string `gorm:"status;type:varchar(1);comment:数据源状态" json:"status"` - Configuration string `gorm:"configuration;type:text;comment:详细信息" json:"configuration"` - CreateBy int64 `gorm:"api" json:"createBy"` //创建人ID + SourceId string `gorm:"primary_key;source_id;comment:数据源Id" json:"sourceId"` // 数据源Id + SourceType string `gorm:"source_type;type:varchar(50);comment:数据源类型" json:"sourceType"` // 数据源类型 + SourceName string `gorm:"source_name;type:varchar(50);comment:数据源名称" json:"sourceName"` // 原名称 + SourceComment string `gorm:"source_comment;type:varchar(50);comment:数据源描述" json:"sourceComment"` // 描述 + Status string `gorm:"status;type:varchar(1);comment:数据源状态" json:"status"` + Db VisualDb `gorm:"db;type:text;comment:详细信息" json:"db"` + CreateBy int64 `gorm:"api" json:"createBy"` //创建人ID } type VisualDb struct { - DbIp string `gorm:"db_ip" json:"dbIp"` - DbPort string `gorm:"db_port" json:"dbPort"` - DbName string `gorm:"db_name" json:"dbName"` - DbUsername string `gorm:"db_username" json:"dbUsername"` - DbPassword string `gorm:"db_password" json:"dbPassword"` - DbJointParam string `gorm:"db_joint_param" json:"dbJointParam"` //额外的链接参数 -} - -type VisualApi struct { - Method string `gorm:"method" json:"method"` - Url string `gorm:"url" json:"url"` - Headers map[string]interface{} `gorm:"headers" json:"headers"` - RequestBody string `gorm:"db_username" json:"dbUsername"` - Auth string `gorm:"db_password" json:"dbPassword"` + Host string `gorm:"host" json:"host"` + Port int64 `gorm:"port" json:"port"` + Dbname string `gorm:"dbname" json:"dbname"` + Username string `gorm:"username" json:"username"` + Password string `gorm:"password" json:"password"` + Config string `gorm:"config" json:"config"` //额外的链接参数 + Schema string `gorm:"schema" json:"schema"` } func (VisualDataSource) TableName() string { return "visual_data_source" } + +func (a VisualDb) Value() (driver.Value, error) { + return json.Marshal(a) +} +func (a *VisualDb) Scan(value interface{}) error { + b, ok := value.([]byte) + if !ok { + return errors.New("type assertion to []byte failed") + } + return json.Unmarshal(b, &a) +} diff --git a/apps/visual/router/data_source.go b/apps/visual/router/data_source.go index c0f74ed..bac14bd 100644 --- a/apps/visual/router/data_source.go +++ b/apps/visual/router/data_source.go @@ -26,7 +26,7 @@ func InitVisualDataSourceRouter(container *restful.Container) { tags := []string{"datasource"} ws.Route(ws.GET("/list").To(func(request *restful.Request, response *restful.Response) { - restfulx.NewReqCtx(request, response).WithLog("获取DataSource分页列表").Handle(s.GetVisualDataSourceList) + restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("获取DataSource分页列表").Handle(s.GetVisualDataSourceList) }). Doc("获取DataSource分页列表"). Param(ws.QueryParameter("pageNum", "页数").Required(true).DataType("int")). @@ -46,25 +46,46 @@ func InitVisualDataSourceRouter(container *restful.Container) { Returns(404, "Not Found", nil)) ws.Route(ws.POST("").To(func(request *restful.Request, response *restful.Response) { - restfulx.NewReqCtx(request, response).WithLog("添加DataSource信息").Handle(s.InsertVisualDataSource) + restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("添加DataSource信息").Handle(s.InsertVisualDataSource) }). Doc("添加DataSource信息"). Metadata(restfulspec.KeyOpenAPITags, tags). Reads(entity.VisualDataSource{})) ws.Route(ws.PUT("").To(func(request *restful.Request, response *restful.Response) { - restfulx.NewReqCtx(request, response).WithLog("修改DataSource信息").Handle(s.UpdateVisualDataSource) + restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("修改DataSource信息").Handle(s.UpdateVisualDataSource) }). Doc("修改DataSource信息"). Metadata(restfulspec.KeyOpenAPITags, tags). Reads(entity.VisualDataSource{})) ws.Route(ws.DELETE("/{sourceId}").To(func(request *restful.Request, response *restful.Response) { - restfulx.NewReqCtx(request, response).WithLog("删除DataSource信息").Handle(s.DeleteVisualDataSource) + restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("删除DataSource信息").Handle(s.DeleteVisualDataSource) }). Doc("删除DataSource信息"). Metadata(restfulspec.KeyOpenAPITags, tags). Param(ws.PathParameter("sourceId", "多id 1,2,3").DataType("string"))) + ws.Route(ws.POST("/test").To(func(request *restful.Request, response *restful.Response) { + restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("验证数据源").Handle(s.GetDataSourceTest) + }). + Doc("验证数据源"). + Metadata(restfulspec.KeyOpenAPITags, tags)) + + ws.Route(ws.GET("/tables/{sourceId}").To(func(request *restful.Request, response *restful.Response) { + restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("验证数据源下所有表").Handle(s.GetDataSourceTables) + }). + Doc("验证数据源下所有表"). + Param(ws.PathParameter("sourceId", "sourceId").DataType("string")). + Metadata(restfulspec.KeyOpenAPITags, tags)) + + ws.Route(ws.GET("/tablesDetails/{sourceId}").To(func(request *restful.Request, response *restful.Response) { + restfulx.NewReqCtx(request, response).WithNeedCasbin(false).WithLog("验证数据源下所有表").Handle(s.GetDataSourceTableDetails) + }). + Doc("验证数据源下所有表"). + Param(ws.PathParameter("sourceId", "sourceId").DataType("string")). + Param(ws.QueryParameter("tableName", "表明").Required(true).DataType("string")). + Metadata(restfulspec.KeyOpenAPITags, tags)) + container.Add(ws) } diff --git a/apps/visual/services/data_source.go b/apps/visual/services/data_source.go index 09d2c94..356efa4 100644 --- a/apps/visual/services/data_source.go +++ b/apps/visual/services/data_source.go @@ -55,9 +55,6 @@ func (m *datasourceModelImpl) FindListPage(page, pageSize int, data entity.Visua db = db.Where("source_id = ?", data.SourceId) } db.Where("delete_time IS NULL") - if data.SourceComment != "" { - db = db.Where("source_comment = ?", data.SourceComment) - } if data.SourceType != "" { db = db.Where("source_type = ?", data.SourceType) } @@ -67,9 +64,6 @@ func (m *datasourceModelImpl) FindListPage(page, pageSize int, data entity.Visua if data.Status != "" { db = db.Where("status = ?", data.Status) } - if data.Configuration != "" { - db = db.Where("configuration = ?", data.Configuration) - } err := db.Count(&total).Error err = db.Order("create_time").Limit(pageSize).Offset(offset).Find(&list).Error biz.ErrIsNil(err, "查询数据源分页列表失败") @@ -84,9 +78,6 @@ func (m *datasourceModelImpl) FindList(data entity.VisualDataSource) *[]entity.V db = db.Where("source_id = ?", data.SourceId) } db.Where("delete_time IS NULL") - if data.SourceComment != "" { - db = db.Where("source_comment = ?", data.SourceComment) - } if data.SourceType != "" { db = db.Where("source_type = ?", data.SourceType) } @@ -96,9 +87,6 @@ func (m *datasourceModelImpl) FindList(data entity.VisualDataSource) *[]entity.V if data.Status != "" { db = db.Where("status = ?", data.Status) } - if data.Configuration != "" { - db = db.Where("configuration = ?", data.Configuration) - } biz.ErrIsNil(db.Order("create_time").Find(&list).Error, "查询数据源列表失败") return &list } diff --git a/go.mod b/go.mod index f40b9d9..6c470e5 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/go-openapi/swag v0.19.15 // indirect github.com/go-redis/redis v6.15.9+incompatible // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect - github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.2 // indirect diff --git a/go.sum b/go.sum index 3cbb952..a1c0cb2 100644 --- a/go.sum +++ b/go.sum @@ -113,6 +113,8 @@ github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5Nq github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= diff --git a/pkg/initialize/router.go b/pkg/initialize/router.go index 8bfe170..e5fa3ba 100644 --- a/pkg/initialize/router.go +++ b/pkg/initialize/router.go @@ -57,6 +57,9 @@ func InitRouter() *transport.HttpServer { visualRouter.InitRuleChainRouter(container) visualRouter.InitVisualScreenGroupRouter(container) visualRouter.InitVisualScreenRouter(container) + visualRouter.InitVisualDataSourceRouter(container) + visualRouter.InitVisualDataSetTableRouter(container) + visualRouter.InitVisualDataSetFieldRouter(container) } // 任务 { diff --git a/pkg/rule_engine/tool/email.go b/pkg/rule_engine/tool/email.go deleted file mode 100644 index 05b1676..0000000 --- a/pkg/rule_engine/tool/email.go +++ /dev/null @@ -1 +0,0 @@ -package tool diff --git a/pkg/rule_engine/tool/kafka.go b/pkg/rule_engine/tool/kafka.go deleted file mode 100644 index 05b1676..0000000 --- a/pkg/rule_engine/tool/kafka.go +++ /dev/null @@ -1 +0,0 @@ -package tool diff --git a/pkg/rule_engine/tool/mq.go b/pkg/rule_engine/tool/mq.go deleted file mode 100644 index 05b1676..0000000 --- a/pkg/rule_engine/tool/mq.go +++ /dev/null @@ -1 +0,0 @@ -package tool diff --git a/pkg/rule_engine/tool/mqtt.go b/pkg/rule_engine/tool/mqtt.go deleted file mode 100644 index 05b1676..0000000 --- a/pkg/rule_engine/tool/mqtt.go +++ /dev/null @@ -1 +0,0 @@ -package tool diff --git a/pkg/rule_engine/tool/rest_api.go b/pkg/rule_engine/tool/rest_api.go deleted file mode 100644 index 05b1676..0000000 --- a/pkg/rule_engine/tool/rest_api.go +++ /dev/null @@ -1 +0,0 @@ -package tool diff --git a/pkg/rule_engine/tool/sms.go b/pkg/rule_engine/tool/sms.go deleted file mode 100644 index 05b1676..0000000 --- a/pkg/rule_engine/tool/sms.go +++ /dev/null @@ -1 +0,0 @@ -package tool diff --git a/uploads/file/47bce5c74f589f4867dbd57e9ca9f808_20230414135050.jpg b/uploads/file/47bce5c74f589f4867dbd57e9ca9f808_20230414135050.jpg deleted file mode 100644 index dc07a48..0000000 Binary files a/uploads/file/47bce5c74f589f4867dbd57e9ca9f808_20230414135050.jpg and /dev/null differ