public/PandaX
1
0
Fork 0
mirror of https://gitee.com/XM-GO/PandaX.git synced 2026-04-23 02:48:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

支持tdengine

Browse Source
This commit is contained in:
XM-GO
2023-06-19 17:06:38 +08:00
parent 997e0862fa
commit 8b5a0206b9
12 changed files with 186 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/visual/api/data_source.go
View File

@@ -99,7 +99,7 @@ func (p *VisualDataSourceApi) GetDataSourceTables(rc *restfulx.ReqCtx) {
one := p.VisualDataSourceApp.FindOne(sourceId)
instance := driver.NewDbInstance(one)
biz.IsTrue(instance != nil, "获取数据源下所有表失败")
rc.ResData = instance.GetMeta().GetTableInfos()
rc.ResData = instance.GetMeta().GetTables()
}
// GetDataSourceTableDetails 获取表下面的所有字段

19
apps/visual/driver/DbInstance.go
View File

@@ -6,6 +6,7 @@ import (
"github.com/XM-GO/PandaKit/cache"
"pandax/apps/visual/entity"
"pandax/pkg/global"
"pandax/pkg/tool"
"reflect"
"strconv"
"strings"
@@ -74,6 +75,12 @@ func (di *DbInstance) GetMeta() DbMetadata {
if dbType == entity.DbTypePostgres {
return &PgsqlMetadata{di: di}
}
if dbType == entity.DbTypeTdengine {
return &TDMetadata{di: di}
}
if dbType == entity.DbTypeClickHouse {
return &ClickHouseMetadata{di: di}
}
return nil
}
@@ -121,11 +128,10 @@ func AddDbInstanceToCache(id string, di *DbInstance) {
func TestConnection(d *entity.VisualDataSource) error {
// 验证第一个库是否可以连接即可
DB, err := GetDbConn(d)
_, err := GetDbConn(d)
if err != nil {
return err
} else {
DB.Close()
return nil
}
}
@@ -138,8 +144,11 @@ func GetDbConn(d *entity.VisualDataSource) (*sql.DB, error) {
DB, err = getMysqlDB(d)
} else if d.SourceType == entity.DbTypePostgres {
DB, err = getPgsqlDB(d)
} else if d.SourceType == entity.DbTypeTdengine {
DB, err = getTdDB(d)
} else if d.SourceType == entity.DbTypeClickHouse {
DB, err = getClickHouseDB(d)
}
if err != nil {
return nil, err
}
@@ -148,7 +157,6 @@ func GetDbConn(d *entity.VisualDataSource) (*sql.DB, error) {
DB.Close()
return nil, err
}
return DB, nil
}
@@ -195,6 +203,9 @@ func SelectDataByDb(db *sql.DB, selectSql string) ([]string, []map[string]interf
if isFirst {
colNames = append(colNames, colName)
}
if strings.Contains(colName, "_") {
colName = tool.ToCamelCase(colName)
}
rowData[colName] = valueConvert(v, colType)
}
// 放入结果集

65
apps/visual/driver/click_house_meta.go Normal file
View File

@@ -0,0 +1,65 @@
package driver
import (
"database/sql"
"fmt"
"github.com/XM-GO/PandaKit/biz"
_ "github.com/go-sql-driver/mysql"
"pandax/apps/visual/entity"
)
func getClickHouseDB(d *entity.VisualDataSource) (*sql.DB, error) {
// 设置dataSourceName -> 更多参数参考：https://github.com/go-sql-driver/mysql#dsn-data-source-name
dsn := fmt.Sprintf("%s:%s@%s(%s:%d)/%s?timeout=8s",
d.Db.Username, d.Db.Password, "tcp", d.Db.Host, d.Db.Port, d.Db.Dbname)
if d.Db.Config != "" {
dsn = fmt.Sprintf("%s&%s", dsn, d.Db.Config)
}
return sql.Open("mysql", dsn)
}
// ---------------------------------- mysql元数据 -----------------------------------
const (
// 表信息元数据
ClickHouse_TABLE_MA = `SELECT name tableName, comment tableComment from system.tables WHERE database = '%s'`
// 表信息
ClickHouse_TABLE_INFO = `SELECT name tableName, comment tableComment, total_rows tableRows
FROM system.tables WHERE database = '%s'`
// 列信息元数据
ClickHouse_COLUMN_MA = `SELECT table tableName, name columnName, type columnType, comment columnComment from system.columns
WHERE database='%s' AND table in (%s) ORDER BY table`
)
type ClickHouseMetadata struct {
di *DbInstance
}
// 获取表基础元信息, 如表名等
func (mm *ClickHouseMetadata) GetTables() []map[string]interface{} {
res, err := mm.di.innerSelect(fmt.Sprintf(ClickHouse_TABLE_MA, mm.di.Info.Db.Dbname))
biz.ErrIsNilAppendErr(err, "获取表基本信息失败: %s")
return res
}
// 获取列元信息, 如列名等
func (mm *ClickHouseMetadata) GetColumns(tableNames ...string) []map[string]interface{} {
tableName := ""
for i := 0; i < len(tableNames); i++ {
if i != 0 {
tableName = tableName + ", "
}
tableName = tableName + "'" + tableNames[i] + "'"
}
result, err := mm.di.innerSelect(fmt.Sprintf(ClickHouse_COLUMN_MA, mm.di.Info.Db.Dbname, tableName))
biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s")
return result
}
// 获取表信息，比GetTableMetedatas获取更详细的表信息
func (mm *ClickHouseMetadata) GetTableInfos() []map[string]interface{} {
res, err := mm.di.innerSelect(fmt.Sprintf(ClickHouse_TABLE_INFO, mm.di.Info.Db.Dbname))
biz.ErrIsNilAppendErr(err, "获取表信息失败: %s")
return res
}

2
apps/visual/driver/meta.go
View File

@@ -3,7 +3,5 @@ package driver
type DbMetadata interface {
GetTables() []map[string]interface{}
GetColumns(tableNames ...string) []map[string]interface{}
GetPrimaryKey(tableName string) string
GetTableInfos() []map[string]interface{}
GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error)
}

62
apps/visual/driver/mysql_meta.go
View File

@@ -4,14 +4,11 @@ import (
"database/sql"
"fmt"
"github.com/XM-GO/PandaKit/biz"
"pandax/apps/visual/entity"
"strings"
_ "github.com/go-sql-driver/mysql"
"pandax/apps/visual/entity"
)
func getMysqlDB(d *entity.VisualDataSource) (*sql.DB, error) {
// SSH Conect
// 设置dataSourceName -> 更多参数参考：https://github.com/go-sql-driver/mysql#dsn-data-source-name
dsn := fmt.Sprintf("%s:%s@%s(%s:%d)/%s?timeout=8s",
d.Db.Username, d.Db.Password, "tcp", d.Db.Host, d.Db.Port, d.Db.Dbname)
@@ -32,25 +29,12 @@ const (
FROM information_schema.tables
WHERE table_schema = (SELECT database())`
// mysql 索引信息
MYSQL_INDEX_INFO = `SELECT index_name indexName, column_name columnName, index_type indexType, non_unique nonUnique,
SEQ_IN_INDEX seqInIndex, INDEX_COMMENT indexComment
FROM information_schema.STATISTICS
WHERE table_schema = (SELECT database()) AND table_name = '%s' ORDER BY index_name asc , SEQ_IN_INDEX asc`
// mysql 列信息元数据
MYSQL_COLUMN_MA = `SELECT table_name tableName, column_name columnName, column_type columnType, column_default columnDefault,
column_comment columnComment, column_key columnKey, extra, is_nullable nullable from information_schema.columns
MYSQL_COLUMN_MA = `SELECT table_name tableName, column_name columnName, column_type columnType,column_comment columnComment,
column_key columnKey, is_nullable nullable from information_schema.columns
WHERE table_schema = (SELECT database()) AND table_name in (%s) ORDER BY tableName, ordinal_position`
)
func sqlFix(sql string) string {
if strings.LastIndex(sql, ";") == (len(sql) - 1) {
sql = sql[0 : len(sql)-1]
}
return sql
}
type MysqlMetadata struct {
di *DbInstance
}
@@ -76,49 +60,9 @@ func (mm *MysqlMetadata) GetColumns(tableNames ...string) []map[string]interface
return result
}
// 获取表主键字段名，不存在主键标识则默认第一个字段
func (mm *MysqlMetadata) GetPrimaryKey(tablename string) string {
columns := mm.GetColumns(tablename)
biz.IsTrue(len(columns) > 0, "[%s] 表不存在", tablename)
for _, v := range columns {
if v["columnKey"].(string) == "PRI" {
return v["columnName"].(string)
}
}
return columns[0]["columnName"].(string)
}
// 获取表信息，比GetTableMetedatas获取更详细的表信息
func (mm *MysqlMetadata) GetTableInfos() []map[string]interface{} {
res, err := mm.di.innerSelect(MYSQL_TABLE_INFO)
biz.ErrIsNilAppendErr(err, "获取表信息失败: %s")
return res
}
// 获取表索引信息
func (mm *MysqlMetadata) GetTableIndex(tableName string) []map[string]interface{} {
res, err := mm.di.innerSelect(fmt.Sprintf(MYSQL_INDEX_INFO, tableName))
biz.ErrIsNilAppendErr(err, "获取表索引信息失败: %s")
// 把查询结果以索引名分组，索引字段以逗号连接
result := make([]map[string]interface{}, 0)
key := ""
for _, v := range res {
// 当前的索引名
in := v["indexName"].(string)
if key == in {
// 索引字段已根据名称和顺序排序，故取最后一个即可
i := len(result) - 1
// 同索引字段以逗号连接
result[i]["columnName"] = result[i]["columnName"].(string) + "," + v["columnName"].(string)
} else {
key = in
result = append(result, v)
}
}
return result
}
func (mm *MysqlMetadata) GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error) {
return mm.di.SelectData(fmt.Sprintf("SELECT * FROM %s LIMIT %d, %d", tableName, (pageNum-1)*pageSize, pageSize))
}

26
apps/visual/driver/pgsql_meta.go
View File

@@ -28,9 +28,6 @@ const (
PGSQL_TABLE_INFO = `SELECT obj_description(c.oid) AS "tableComment", c.relname AS "tableName" FROM pg_class c
JOIN pg_namespace n ON c.relnamespace = n.oid WHERE n.nspname = (select current_schema()) AND c.reltype > 0`
PGSQL_INDEX_INFO = `SELECT indexname AS "indexName", indexdef AS "indexComment"
FROM pg_indexes WHERE schemaname = (select current_schema()) AND tablename = '%s'`
PGSQL_COLUMN_MA = `SELECT
C.relname AS "tableName",
A.attname AS "columnName",
@@ -80,18 +77,6 @@ func (pm *PgsqlMetadata) GetColumns(tableNames ...string) []map[string]interface
return result
}
func (pm *PgsqlMetadata) GetPrimaryKey(tablename string) string {
columns := pm.GetColumns(tablename)
biz.IsTrue(len(columns) > 0, "[%s] 表不存在", tablename)
for _, v := range columns {
if v["columnKey"].(string) == "PRI" {
return v["columnName"].(string)
}
}
return columns[0]["columnName"].(string)
}
// 获取表信息，比GetTables获取更详细的表信息
func (pm *PgsqlMetadata) GetTableInfos() []map[string]interface{} {
res, err := pm.di.innerSelect(PGSQL_TABLE_INFO)
@@ -99,17 +84,6 @@ func (pm *PgsqlMetadata) GetTableInfos() []map[string]interface{} {
return res
}
// 获取表索引信息
func (pm *PgsqlMetadata) GetTableIndex(tableName string) []map[string]interface{} {
res, err := pm.di.innerSelect(fmt.Sprintf(PGSQL_INDEX_INFO, tableName))
biz.ErrIsNilAppendErr(err, "获取表索引信息失败: %s")
return res
}
func (pm *PgsqlMetadata) GetTableRecord(tableName string, pageNum, pageSize int) ([]string, []map[string]interface{}, error) {
return pm.di.SelectData(fmt.Sprintf("SELECT * FROM %s OFFSET %d LIMIT %d", tableName, (pageNum-1)*pageSize, pageSize))
}
// 获取所有Schema
func (pm *PgsqlMetadata) GetSchemas() ([]string, []map[string]interface{}, error) {
return pm.di.SelectData("SELECT schema_name FROM information_schema.schemata")

64
apps/visual/driver/td_meta.go Normal file
View File

@@ -0,0 +1,64 @@
package driver
import (
"database/sql"
"fmt"
"github.com/XM-GO/PandaKit/biz"
_ "github.com/taosdata/driver-go/v3/taosRestful"
"pandax/apps/visual/entity"
)
func getTdDB(d *entity.VisualDataSource) (*sql.DB, error) {
dsn := fmt.Sprintf("%s:%s@%s(%s:%d)/%s",
d.Db.Username, d.Db.Password, "http", d.Db.Host, d.Db.Port, d.Db.Dbname)
if d.Db.Config != "" {
dsn = fmt.Sprintf("%s&%s", dsn, d.Db.Config)
}
return sql.Open("taosRestful", dsn)
}
// ---------------------------------- mysql元数据 -----------------------------------
const (
// 表信息元数据
TD_TABLE_MA = `SELECT table_name table_name,table_comment table_comment from information_schema.INS_TABLES WHERE db_name = '%s'`
// 表信息
TD_TABLE_INFO = `SELECT table_name table_name, table_comment table_comment, columns table_rows, create_time create_time
FROM information_schema.INS_TABLES WHERE db_name = '%s'`
//列信息元数据
TD_COLUMN_MA = `SELECT table_name table_name, col_name column_name, col_type column_type,col_nullable nullable from information_schema.INS_COLUMNS
WHERE db_name = '%s' AND table_name in (%s) ORDER BY table_name`
)
type TDMetadata struct {
di *DbInstance
}
// 获取表基础元信息, 如表名等
func (mm *TDMetadata) GetTables() []map[string]interface{} {
res, err := mm.di.innerSelect(fmt.Sprintf(TD_TABLE_MA, mm.di.Info.Db.Dbname))
biz.ErrIsNilAppendErr(err, "获取表基本信息失败: %s")
return res
}
// 获取列元信息, 如列名等
func (mm *TDMetadata) GetColumns(tableNames ...string) []map[string]interface{} {
tableName := ""
for i := 0; i < len(tableNames); i++ {
if i != 0 {
tableName = tableName + ", "
}
tableName = tableName + "'" + tableNames[i] + "'"
}
result, err := mm.di.innerSelect(fmt.Sprintf(TD_COLUMN_MA, mm.di.Info.Db.Dbname, tableName))
biz.ErrIsNilAppendErr(err, "获取数据库列信息失败: %s")
return result
}
// 获取表信息，比GetTableMetedatas获取更详细的表信息
func (mm *TDMetadata) GetTableInfos() []map[string]interface{} {
res, err := mm.di.innerSelect(fmt.Sprintf(TD_TABLE_INFO, mm.di.Info.Db.Dbname))
biz.ErrIsNilAppendErr(err, "获取表信息失败: %s")
return res
}

6
apps/visual/entity/data_source.go
View File

@@ -8,8 +8,10 @@ import (
)
const (
DbTypeMysql = "MySQL"
DbTypePostgres = "PostgreSQL"
DbTypeMysql = "MySQL"
DbTypePostgres = "PostgreSQL"
DbTypeTdengine = "Tdengine"
DbTypeClickHouse = "ClickHouse"
)
type VisualDataSource struct {

5
go.mod
View File

@@ -4,7 +4,7 @@ go 1.18
require (
github.com/Shopify/sarama v1.38.1
github.com/XM-GO/PandaKit v0.0.0-20230527040642-b5fb5258717c
github.com/XM-GO/PandaKit v0.0.0-20230619032950-ff62bb372352
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/didip/tollbooth v4.0.2+incompatible
github.com/dop251/goja v0.0.0-20230304130813-e2f543bf4b4c
@@ -14,7 +14,7 @@ require (
github.com/go-openapi/spec v0.20.6
github.com/go-sql-driver/mysql v1.7.0
github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.4.2
github.com/gorilla/websocket v1.5.0
github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible
github.com/kakuilan/kgo v0.1.8
github.com/lib/pq v1.10.4
@@ -24,6 +24,7 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.5.0
github.com/taosdata/driver-go/v3 v3.5.0
github.com/xuri/excelize/v2 v2.7.1
golang.org/x/crypto v0.8.0
google.golang.org/grpc v1.48.0

9
go.sum
View File

@@ -12,8 +12,8 @@ github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSd
github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc=
github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY=
github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/XM-GO/PandaKit v0.0.0-20230527040642-b5fb5258717c h1:YdIbF5bPNHW6VSEiZNDJE5XzsqQlMgL67G869CEeg+Q=
github.com/XM-GO/PandaKit v0.0.0-20230527040642-b5fb5258717c/go.mod h1:JvzBc3WObQ35hOBnsrhI5/tcoFmAXeDnIiL24x2t0bk=
github.com/XM-GO/PandaKit v0.0.0-20230619032950-ff62bb372352 h1:h7nxqPw7Dhx/rKbHIIXa/fCIT7liWffOWuXlnItMBAs=
github.com/XM-GO/PandaKit v0.0.0-20230619032950-ff62bb372352/go.mod h1:JvzBc3WObQ35hOBnsrhI5/tcoFmAXeDnIiL24x2t0bk=
github.com/aliyun/aliyun-oss-go-sdk v2.2.0+incompatible h1:ht2+VfbXtNLGhCsnTMc6/N26nSTBK6qdhktjYyjJQkk=
github.com/aliyun/aliyun-oss-go-sdk v2.2.0+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -159,8 +159,9 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -378,6 +379,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/taosdata/driver-go/v3 v3.5.0 h1:30crN+E+ACURmq28kn3Y8B3jfL5knaC1fc1rLvgyXqs=
github.com/taosdata/driver-go/v3 v3.5.0/go.mod h1:H2vo/At+rOPY1aMzUV9P49SVX7NlXb3LAbKw+MCLrmU=
github.com/xuri/efp v0.0.0-20220603152613-6918739fd470 h1:6932x8ltq1w4utjmfMPVj09jdMlkY0aiA6+Skbtl3/c=
github.com/xuri/efp v0.0.0-20220603152613-6918739fd470/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
github.com/xuri/excelize/v2 v2.7.1 h1:gm8q0UCAyaTt3MEF5wWMjVdmthm2EHAWesGSKS9tdVI=

17
pkg/tool/base.go Normal file
View File

@@ -0,0 +1,17 @@
package tool
import (
"regexp"
"strings"
)
func ToCamelCase(s string) string {
re := regexp.MustCompile(`[_\W]+`)
words := re.Split(s, -1)
for i := range words {
if i != 0 {
words[i] = strings.Title(words[i])
}
}
return strings.Join(words, "")
}

8
pkg/tool/base_test.go Normal file
View File

@@ -0,0 +1,8 @@
package tool
import "testing"
func TestToCamelCase(t *testing.T) {
camelCase := ToCamelCase("hello_world")
t.Log(camelCase)
}