hw: add draft rockchip-mpp hw video encoding

http/route/nanokvm_ui.go

@@ -5,7 +5,6 @@ import (
 	"rkkvm/external/ffmpeg"
 	"rkkvm/http/middleware"
 	"rkkvm/http/reqrsp"
-	"time"
 
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
@@ -71,14 +70,14 @@ func SetScreen(c *gin.Context) {
 		})
 		return
 	}
-
-	log.Debug("Stopping ffmpeg SetScreen")
-	ffmpeg.Stop()
-	time.Sleep(100 * time.Millisecond)
-	ffmpeg.ApplyOptions()
-	log.Debug("Starting ffmpeg SetScreen")
-	ffmpeg.Start()
-
+	/*
+		log.Debug("Stopping ffmpeg SetScreen")
+		ffmpeg.Stop()
+		time.Sleep(100 * time.Millisecond)
+		ffmpeg.ApplyOptions()
+		log.Debug("Starting ffmpeg SetScreen")
+		ffmpeg.Start()
+	*/
 	log.Debugf("update screen: %+v", req)
 	c.JSON(http.StatusOK, reqrsp.NanoKVMRsp{
 		Msg: reqrsp.MsgSuccess,

http/rtc/listener.go

@@ -26,11 +26,6 @@ func initUDPListener(host string, port int) (*net.UDPConn, error) {
 }
 
 func InitListener(host string, port int, aPort int) (*RTC, error) {
-	vl, err := initUDPListener(host, port)
-	if err != nil {
-		return nil, err
-	}
-
 	al, err := initUDPListener(host, aPort)
 	if err != nil {
 		return nil, err
@@ -40,16 +35,13 @@ func InitListener(host string, port int, aPort int) (*RTC, error) {
 	switch config.Get().Video.Codec {
 	case config.StreamSourceH264:
 		mimeType = webrtc.MimeTypeH264
-	case config.StreamSourceHevc: // WebRTC currently has no official support for H265
-		mimeType = webrtc.MimeTypeH265
 	default:
 		return nil, ErrWebRTCParam("unknown video codec: %s", config.Get().Video.Codec)
 	}
 
-	video, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
+	video, _ := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
 	audio, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "rkkvm")
 	rtc = &RTC{
-		videoListener: vl,
 		audioListener: al,
 		peers:         make(map[string]*webrtc.PeerConnection),
 		videoTrack:    video,

http/rtc/webrtc.go

@@ -4,13 +4,15 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"rkkvm/external/ffmpeg"
+	"rkkvm/external/mpp"
 	"sync"
+	"time"
 
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/pion/webrtc/v4"
+	"github.com/pion/webrtc/v4/pkg/media"
 )
 
 var rtc *RTC
@@ -27,9 +29,8 @@ var ErrPeerClosedConn = ErrWebRTCParam("peer closed conn")
 
 type RTC struct {
 	peers         map[string]*webrtc.PeerConnection
-	videoListener *net.UDPConn
 	audioListener *net.UDPConn
-	videoTrack    *webrtc.TrackLocalStaticRTP
+	videoTrack    *webrtc.TrackLocalStaticSample
 	audioTrack    *webrtc.TrackLocalStaticRTP
 	m             sync.Mutex
 }
@@ -38,10 +39,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 	peerID := uuid.New().String()
 	r.m.Lock()
 	r.peers[peerID] = p
-	if len(r.peers) == 1 {
+	/*if len(r.peers) == 1 {
 		ffmpeg.GetFFmpeg().Start()
 		log.Info("FFmpeg process started")
-	}
+	}*/
 	r.m.Unlock()
 
 	p.OnConnectionStateChange(func(connState webrtc.PeerConnectionState) {
@@ -49,10 +50,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 			r.m.Lock()
 			defer r.m.Unlock()
 			delete(r.peers, peerID)
-			if len(r.peers) == 0 {
+			/*if len(r.peers) == 0 {
 				ffmpeg.GetFFmpeg().Stop()
 				log.Info("No clients anymore, stop ffmpeg process")
-			}
+			}*/
 			p.Close()
 
 			peers := make([]string, 0, len(r.peers))
@@ -93,7 +94,42 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 }
 
 func (r *RTC) VideoListenerRead() {
-	listenerRead(r.videoListener, r.videoTrack)
+	duration := time.Second / time.Duration(60)
+	ticker := time.NewTicker(duration)
+	defer ticker.Stop()
+
+	// Retrieve SPS and PPS once at the start
+	sps, err := mpp.GetSPS()
+	if err != nil {
+		log.Fatalf("Failed to retrieve SPS: %v", err)
+	}
+
+	firstFrame := true
+	for {
+		select {
+		case <-ticker.C:
+			frame, err := mpp.GetInstance().CaptureAndEncode()
+			if err != nil {
+				log.Errorf("failed to capture frame: %v", err)
+				continue
+			}
+
+			// If this is the first frame or an IDR frame, prepend SPS and PPS
+			if firstFrame || isIDRFrame(frame) {
+				firstFrame = false
+				frame = append(sps, frame...)
+			}
+
+			sample := media.Sample{
+				Data:     frame,
+				Duration: duration,
+			}
+			err = r.videoTrack.WriteSample(sample)
+			if err != nil {
+				log.Errorf("failed to write sample: %v", err)
+			}
+		}
+	}
 }
 
 func (r *RTC) AudioListenerRead() {
@@ -101,7 +137,6 @@ func (r *RTC) AudioListenerRead() {
 }
 
 func (r *RTC) Close() error {
-	r.videoListener.Close()
 	r.audioListener.Close()
 
 	return nil
@@ -146,3 +181,16 @@ func processRTCP(rtpSender *webrtc.RTPSender) {
 		}
 	}()
 }
+
+func isIDRFrame(frame []byte) bool {
+	// Check for NAL unit type 5 (IDR)
+	for i := 0; i < len(frame)-4; i++ {
+		if frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x00 && frame[i+3] == 0x01 {
+			nalType := frame[i+4] & 0x1F
+			if nalType == 5 { // IDR frame
+				return true
+			}
+		}
+	}
+	return false
+}