[优化]设备数据上传并发处理，添加队列，以及并发数控制

iothub/client/updclient/udp.go

@@ -12,7 +12,6 @@ type UdpClientT struct {
 	Addr *net.UDPAddr
 }
 
-// var UdpClient = make(map[string]*UdpClientT)
 var UdpClient sync.Map
 
 func Send(deviceId, msg string) error {

iothub/hook_message_work/hook_service.go

@@ -1,6 +1,10 @@
 package hook_message_work
 
 import (
+	"context"
+	"encoding/json"
+	"github.com/golang-queue/queue"
+	"github.com/golang-queue/queue/core"
 	"pandax/iothub/netbase"
 	"pandax/pkg/global"
 	"sync"
@@ -8,19 +12,28 @@ import (
 
 type HookService struct {
 	Cache     sync.Map
-	Wg        sync.WaitGroup //  优雅关闭
+	Queue     *queue.Queue
 	Ch        chan struct{}  //  并发限制
+	Wg        sync.WaitGroup //  优雅关闭
 	MessageCh chan *netbase.DeviceEventInfo
 }
 
 func NewHookService() *HookService {
 	hs := &HookService{
 		Cache:     sync.Map{},
-		MessageCh: make(chan *netbase.DeviceEventInfo),
-	}
-	// 并发限制，代表服务器处理能力
-	if global.Conf.Queue.Enable && global.Conf.Queue.Num > 0 {
-		hs.Ch = make(chan struct{}, global.Conf.Queue.Num)
+		Ch:        make(chan struct{}, global.Conf.Queue.ChNum),
+		MessageCh: make(chan *netbase.DeviceEventInfo, global.Conf.Queue.TaskNum),
 	}
+	pool := queue.NewPool(int(global.Conf.Queue.QueuePool), queue.WithFn(func(ctx context.Context, m core.QueuedMessage) error {
+		v, ok := m.(*netbase.DeviceEventInfo)
+		if !ok {
+			if err := json.Unmarshal(m.Bytes(), &v); err != nil {
+				return err
+			}
+		}
+		hs.MessageCh <- v
+		return nil
+	}))
+	hs.Queue = pool
 	return hs
 }

iothub/netbase/iothub_session.go

@@ -1,6 +1,7 @@
 package netbase
 
 import (
+	"encoding/json"
 	"pandax/pkg/global/model"
 )
 
@@ -11,3 +12,11 @@ type DeviceEventInfo struct {
 	Type       string            `json:"type"`
 	RequestId  string            `json:"requestId"`
 }
+
+func (j *DeviceEventInfo) Bytes() []byte {
+	b, err := json.Marshal(j)
+	if err != nil {
+		panic(err)
+	}
+	return b
+}

iothub/server/emqxserver/hook.go

@@ -281,9 +281,8 @@ func (s *HookGrpcService) OnMessagePublish(ctx context.Context, in *exhook2.Mess
 		data.RequestId = id
 	}
 
-	//TODO 如果设备消息；量过大，推荐采用NATS队列处理
-	s.HookService.MessageCh <- data
-
+	//将数据放到队列中
+	s.HookService.Queue.Queue(data)
 	res.Value = &exhook2.ValuedResponse_Message{Message: in.Message}
 	return res, nil
 }