hw: add draft rockchip-mpp hw video encoding

Dockerfile

@@ -12,11 +12,26 @@ RUN mv /web/dist /static
 #RUN mv nanokvm/web/dist /static
 
 FROM golang:alpine as backend
-RUN apk add --no-cache git
+RUN apk add --no-cache git v4l-utils-dev build-base musl-dev gcc g++ make libc-dev linux-headers \
+    git cmake
+RUN mkdir -p ~/dev && cd ~/dev && \
+    #git clone -b jellyfin-mpp --depth=1 https://github.com/nyanmisaka/mpp.git rkmpp && \
+    git clone --depth=1 https://github.com/rockchip-linux/mpp.git rkmpp && \
+    cd rkmpp && mkdir rkmpp_build && cd rkmpp_build && \
+    sed -i '/#include <errno.h>/a #include <cstdint>' ~/dev/rkmpp/mpp/vproc/vdpp/test/hwpq_test.cpp && \ 
+    cmake \
+        -DCMAKE_INSTALL_PREFIX=/usr \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DBUILD_SHARED_LIBS=ON \
+        -DBUILD_TEST=OFF \
+        -DVDPP_TEST=OFF \
+        .. && \
+    make -j $(nproc) && \
+    make install
 COPY . /app
 WORKDIR /app
 RUN go mod download
-RUN go build -o ./bin/ ./cmd/...
+RUN CGO_ENABLED=1 go build -o ./bin/ ./cmd/...
 
 # we would use ffmpeg-rockchip from nyanmisaka with rockchip's mpp support
 FROM alpine as ffmpeg
@@ -74,7 +89,6 @@ RUN mkdir -p ~/dev && cd ~/dev && \
     && \
     make -j $(nproc) && make install
 
-
 FROM pikvm/ustreamer:latest as ustreamer
 
 FROM alpine

cmd/kvm/main.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"rkkvm/config"
 	"rkkvm/external/ffmpeg"
+	"rkkvm/external/mpp"
 	"rkkvm/external/process"
 	"rkkvm/http/route"
 	"rkkvm/http/rtc"
@@ -31,6 +32,12 @@ func main() {
 		log.Println("Failed to parse log level, use default level: info")
 	}
 
+	// FIXME: hardcoded device
+	if err := mpp.NewTest("/dev/video0"); err != nil {
+		log.Fatalf("Failed to open device: %v", err)
+	}
+	defer mpp.GetInstance().Close()
+
 	webrtc, err := rtc.InitListener(cfg.WebRtc.Host, cfg.WebRtc.VideoPort, cfg.WebRtc.AudioPort)
 	if err != nil {
 		log.Fatal(err)

external/mpp/mpp.c

@@ -0,0 +1,245 @@
+#include "mpp.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <linux/videodev2.h>
+#include <rockchip/mpp_buffer.h>
+#include <rockchip/mpp_err.h>
+#include <rockchip/mpp_frame.h>
+#include <rockchip/mpp_packet.h>
+#include <rockchip/rk_mpi.h>
+
+#define WIDTH 1920
+#define HEIGHT 1080
+#define NUM_PLANES 1
+#define BUFFER_COUNT 4
+
+#define SPS_MAX_SIZE 64
+
+static unsigned char sps_buffer[SPS_MAX_SIZE];
+static int sps_size = 0;
+
+static int video_fd = -1;
+static void *buffers[BUFFER_COUNT];
+static struct v4l2_buffer v4l2_buf;
+static MppCtx mpp_ctx = NULL;
+static MppApi *mpp_api = NULL;
+static MppEncCfg cfg;
+
+int video_init(const char *device) {
+    video_fd = open(device, O_RDWR);
+    if (video_fd < 0) {
+        perror("Failed to open video device");
+        return -1;
+    }
+
+    // Set video format
+    struct v4l2_format fmt;
+    memset(&fmt, 0, sizeof(fmt));
+    fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    fmt.fmt.pix_mp.width = WIDTH;
+    fmt.fmt.pix_mp.height = HEIGHT;
+    fmt.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_BGR24;
+    fmt.fmt.pix_mp.num_planes = NUM_PLANES;
+
+    if (ioctl(video_fd, VIDIOC_S_FMT, &fmt) < 0) {
+        perror("Failed to set video format");
+        close(video_fd);
+        return -1;
+    }
+
+    // Request buffers
+    struct v4l2_requestbuffers req;
+    memset(&req, 0, sizeof(req));
+    req.count = BUFFER_COUNT;
+    req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    req.memory = V4L2_MEMORY_MMAP;
+
+    if (ioctl(video_fd, VIDIOC_REQBUFS, &req) < 0) {
+        perror("Failed to request buffers");
+        close(video_fd);
+        return -1;
+    }
+
+    // Map buffers
+    struct v4l2_plane planes[1];
+    for (int i = 0; i < BUFFER_COUNT; i++) {
+        memset(&v4l2_buf, 0, sizeof(v4l2_buf));
+        memset(planes, 0, sizeof(planes));
+        v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+        v4l2_buf.memory = V4L2_MEMORY_MMAP;
+        v4l2_buf.index = i;
+        v4l2_buf.m.planes = planes;
+        v4l2_buf.length = 1;
+
+        if (ioctl(video_fd, VIDIOC_QUERYBUF, &v4l2_buf) < 0) {
+            perror("Failed to query buffer");
+            close(video_fd);
+            return -1;
+        }
+
+        buffers[i] = mmap(NULL, planes[0].length, PROT_READ | PROT_WRITE, MAP_SHARED, video_fd, planes[0].m.mem_offset);
+        if (buffers[i] == MAP_FAILED) {
+            perror("Failed to map buffer");
+            close(video_fd);
+            return -1;
+        }
+
+        // Queue buffer
+        if (ioctl(video_fd, VIDIOC_QBUF, &v4l2_buf) < 0) {
+            perror("Failed to queue buffer");
+            close(video_fd);
+            return -1;
+        }
+    }
+
+    // Start streaming
+    int type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    if (ioctl(video_fd, VIDIOC_STREAMON, &type) < 0) {
+        perror("Failed to start streaming");
+        close(video_fd);
+        return -1;
+    }
+
+    return 0;
+}
+
+MppEncCfg mpp_enc_cfg(int fps, int bitrate) {
+    MppEncCfg cfg;
+    mpp_enc_cfg_init(&cfg);
+    mpp_enc_cfg_set_s32(cfg, "prep:width", WIDTH);
+    mpp_enc_cfg_set_s32(cfg, "prep:height", HEIGHT);
+    mpp_enc_cfg_set_s32(cfg, "prep:format", MPP_FMT_BGR888);
+    mpp_enc_cfg_set_s32(cfg, "prep:hor_stride", WIDTH * 3);
+    mpp_enc_cfg_set_s32(cfg, "prep:ver_stride", HEIGHT);
+
+    mpp_enc_cfg_set_s32(cfg, "rc:mode", MPP_ENC_RC_MODE_CBR);
+
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_in_flex", 0);            // Fixed input FPS
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_in_num", fps);            // Input FPS numerator
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_in_denorm", 1);          // Input FPS denominator
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_out_flex", 0);           // Fixed output FPS
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_out_num", fps);           // Output FPS numerator
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_out_denorm", 1);         // Output FPS denominator
+
+    mpp_enc_cfg_set_s32(cfg, "rc:gop", 5);
+
+    mpp_enc_cfg_set_u32(cfg, "rc:drop_mode", MPP_ENC_RC_DROP_FRM_DISABLED);
+    mpp_enc_cfg_set_u32(cfg, "rc:drop_thd", 20);        /* 20% of max bps */
+    mpp_enc_cfg_set_u32(cfg, "rc:drop_gap", 1);         /* Do not continuous drop frame */
+
+    mpp_enc_cfg_set_s32(cfg, "rc:bps_target", bitrate);
+    mpp_enc_cfg_set_s32(cfg, "rc:bps_max", bitrate * 17 / 16);
+    mpp_enc_cfg_set_s32(cfg, "rc:bps_min", bitrate * 15 / 16);
+
+    return cfg;
+}
+
+int mpp_init_wrapper(int fps, int bitrate) {
+    MPP_RET ret = mpp_create(&mpp_ctx, &mpp_api);
+    if (ret != MPP_OK) 
+        return -1;
+
+    ret = mpp_init(mpp_ctx, MPP_CTX_ENC, MPP_VIDEO_CodingAVC);
+    if (ret != MPP_OK)
+        return -1;
+    
+    MppPollType poll_type = MPP_POLL_BLOCK;
+	ret = mpp_api->control(mpp_ctx, MPP_SET_OUTPUT_TIMEOUT,(MppParam)&poll_type);
+    if (ret != MPP_OK) {
+        mpp_destroy(mpp_ctx);
+        return -1;
+    }
+
+    cfg = mpp_enc_cfg(fps, bitrate);
+    ret = mpp_api->control(mpp_ctx, MPP_ENC_SET_CFG, cfg);
+    if (ret != MPP_OK) {
+        mpp_destroy(mpp_ctx);
+        return -1;
+    }
+
+    // Get SPS and PPS information
+    MppPacket extra_pkt = NULL;
+    ret = mpp_api->control(mpp_ctx, MPP_ENC_GET_EXTRA_INFO, &extra_pkt);
+    if (ret == MPP_OK && extra_pkt) {
+        void *extra_data = mpp_packet_get_pos(extra_pkt);
+        size_t extra_size = mpp_packet_get_length(extra_pkt);
+
+        memcpy(sps_buffer, extra_data, extra_size);
+        sps_size = extra_size;
+
+        mpp_packet_deinit(&extra_pkt);
+    } else {
+        fprintf(stderr, "Failed to get SPS/PPS data from encoder\n");
+    }
+
+    return 0;
+}
+
+int get_sps(unsigned char *output, int *output_len) {
+    if (sps_size == 0) {
+        fprintf(stderr, "SPS data not available\n");
+        return -1;
+    }
+
+    memcpy(output, sps_buffer, sps_size);
+    *output_len = sps_size;
+
+    return 0;
+}
+
+int capture_and_encode(unsigned char *output, int *output_len) {
+    // Dequeue buffer
+    struct v4l2_plane planes[1];
+    memset(&v4l2_buf, 0, sizeof(v4l2_buf));
+    memset(planes, 0, sizeof(planes));
+    v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    v4l2_buf.memory = V4L2_MEMORY_MMAP;
+    v4l2_buf.m.planes = planes;
+    v4l2_buf.length = NUM_PLANES;
+
+    if (ioctl(video_fd, VIDIOC_DQBUF, &v4l2_buf) < 0) {
+        perror("Failed to dequeue buffer");
+        return -1;
+    }
+
+    // Encode frame
+    MppFrame frame;
+    mpp_frame_init(&frame);
+    mpp_frame_set_width(frame, WIDTH);
+    mpp_frame_set_height(frame, HEIGHT);
+    mpp_frame_set_fmt(frame, MPP_FMT_BGR888);
+
+    MppBuffer frm_buf;
+    mpp_buffer_get(NULL, &frm_buf, planes[0].length);
+    memcpy(mpp_buffer_get_ptr(frm_buf), buffers[v4l2_buf.index], planes->bytesused);
+    mpp_frame_set_buffer(frame, frm_buf);
+
+    MPP_RET ret = mpp_api->encode_put_frame(mpp_ctx, frame);
+    if (ret == MPP_OK) {
+        MppPacket packet;
+        ret = mpp_api->encode_get_packet(mpp_ctx, &packet);
+        if (ret == MPP_OK) {
+            *output_len = mpp_packet_get_length(packet);
+            memcpy(output, mpp_packet_get_data(packet), *output_len);
+            mpp_packet_deinit(&packet);
+        }
+    }
+
+    mpp_frame_deinit(&frame);
+    mpp_buffer_put(frm_buf);
+
+    // Requeue buffer
+    ioctl(video_fd, VIDIOC_QBUF, &v4l2_buf);
+    return ret == MPP_OK ? 0 : -1;
+}
+
+void cleanup() {
+    //if (cfg) mpp_enc_cfg_deinit(cfg);
+    if (mpp_ctx) mpp_destroy(mpp_ctx);
+    if (video_fd >= 0) close(video_fd);
+}

external/mpp/mpp.go

@@ -0,0 +1,82 @@
+package mpp
+
+/*
+#cgo CFLAGS: -I/usr/include/rockchip/mpp
+#cgo LDFLAGS: -L/usr/lib -lv4l2 -lrockchip_mpp
+#include "mpp.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <linux/videodev2.h>
+#include <rockchip/mpp_buffer.h>
+#include <rockchip/mpp_err.h>
+#include <rockchip/mpp_frame.h>
+#include <rockchip/mpp_packet.h>
+#include <rockchip/rk_mpi.h>
+*/
+import "C"
+import (
+	"errors"
+	"fmt"
+	"unsafe"
+)
+
+var mpp *MPP
+
+type MPP struct {
+}
+
+func New(device string, fps int, bitrate int) (*MPP, error) {
+	cDevice := C.CString(device)
+	defer C.free(unsafe.Pointer(cDevice))
+
+	if C.video_init(cDevice) != 0 {
+		return nil, errors.New("failed to initialize video capture")
+	}
+	if C.mpp_init_wrapper(C.int(fps), C.int(bitrate)) != 0 {
+		return nil, errors.New("failed to initialize MPP")
+	}
+	return &MPP{}, nil
+}
+
+func GetSPS() ([]byte, error) {
+	output := make([]byte, 64) // Allocate buffer for SPS
+	var outputLen C.int
+
+	ret := C.get_sps((*C.uchar)(unsafe.Pointer(&output[0])), &outputLen)
+	if ret != 0 {
+		return nil, fmt.Errorf("failed to retrieve SPS: %v", ret)
+	}
+
+	return output[:outputLen], nil
+}
+
+func (m *MPP) CaptureAndEncode() ([]byte, error) {
+	output := make([]byte, 1024*1024) // 1MB buffer
+	var outputLen C.int
+
+	ret := C.capture_and_encode((*C.uchar)(unsafe.Pointer(&output[0])), &outputLen)
+	if ret != 0 {
+		return nil, errors.New("failed to capture and encode frame")
+	}
+
+	return output[:outputLen], nil
+}
+
+func (m *MPP) Close() {
+	C.cleanup()
+}
+
+func NewTest(device string) error {
+	var err error
+	mpp, err = New(device, 60, 10000*1000)
+	return err
+}
+
+func GetInstance() *MPP {
+	return mpp
+}

external/mpp/mpp.h

@@ -0,0 +1,10 @@
+#ifndef MPP_H
+#define MPP_H
+
+int video_init(const char *device);
+int mpp_init_wrapper(int fps, int bitrate);
+int get_sps(unsigned char *output, int *output_len);
+int capture_and_encode(unsigned char *output, int *output_len);
+void cleanup();
+
+#endif

http/route/nanokvm_ui.go

@@ -5,7 +5,6 @@ import (
 	"rkkvm/external/ffmpeg"
 	"rkkvm/http/middleware"
 	"rkkvm/http/reqrsp"
-	"time"
 
 	"github.com/gin-gonic/gin"
 	log "github.com/sirupsen/logrus"
@@ -71,14 +70,14 @@ func SetScreen(c *gin.Context) {
 		})
 		return
 	}
-
+	/*
 		log.Debug("Stopping ffmpeg SetScreen")
 		ffmpeg.Stop()
 		time.Sleep(100 * time.Millisecond)
 		ffmpeg.ApplyOptions()
 		log.Debug("Starting ffmpeg SetScreen")
 		ffmpeg.Start()
-
+	*/
 	log.Debugf("update screen: %+v", req)
 	c.JSON(http.StatusOK, reqrsp.NanoKVMRsp{
 		Msg: reqrsp.MsgSuccess,

http/rtc/listener.go

@@ -26,11 +26,6 @@ func initUDPListener(host string, port int) (*net.UDPConn, error) {
 }
 
 func InitListener(host string, port int, aPort int) (*RTC, error) {
-	vl, err := initUDPListener(host, port)
-	if err != nil {
-		return nil, err
-	}
-
 	al, err := initUDPListener(host, aPort)
 	if err != nil {
 		return nil, err
@@ -40,16 +35,13 @@ func InitListener(host string, port int, aPort int) (*RTC, error) {
 	switch config.Get().Video.Codec {
 	case config.StreamSourceH264:
 		mimeType = webrtc.MimeTypeH264
-	case config.StreamSourceHevc: // WebRTC currently has no official support for H265
-		mimeType = webrtc.MimeTypeH265
 	default:
 		return nil, ErrWebRTCParam("unknown video codec: %s", config.Get().Video.Codec)
 	}
 
-	video, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
+	video, _ := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
 	audio, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "rkkvm")
 	rtc = &RTC{
-		videoListener: vl,
 		audioListener: al,
 		peers:         make(map[string]*webrtc.PeerConnection),
 		videoTrack:    video,

http/rtc/webrtc.go

@@ -4,13 +4,15 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"rkkvm/external/ffmpeg"
+	"rkkvm/external/mpp"
 	"sync"
+	"time"
 
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/pion/webrtc/v4"
+	"github.com/pion/webrtc/v4/pkg/media"
 )
 
 var rtc *RTC
@@ -27,9 +29,8 @@ var ErrPeerClosedConn = ErrWebRTCParam("peer closed conn")
 
 type RTC struct {
 	peers         map[string]*webrtc.PeerConnection
-	videoListener *net.UDPConn
 	audioListener *net.UDPConn
-	videoTrack    *webrtc.TrackLocalStaticRTP
+	videoTrack    *webrtc.TrackLocalStaticSample
 	audioTrack    *webrtc.TrackLocalStaticRTP
 	m             sync.Mutex
 }
@@ -38,10 +39,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 	peerID := uuid.New().String()
 	r.m.Lock()
 	r.peers[peerID] = p
-	if len(r.peers) == 1 {
+	/*if len(r.peers) == 1 {
 		ffmpeg.GetFFmpeg().Start()
 		log.Info("FFmpeg process started")
-	}
+	}*/
 	r.m.Unlock()
 
 	p.OnConnectionStateChange(func(connState webrtc.PeerConnectionState) {
@@ -49,10 +50,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 			r.m.Lock()
 			defer r.m.Unlock()
 			delete(r.peers, peerID)
-			if len(r.peers) == 0 {
+			/*if len(r.peers) == 0 {
 				ffmpeg.GetFFmpeg().Stop()
 				log.Info("No clients anymore, stop ffmpeg process")
-			}
+			}*/
 			p.Close()
 
 			peers := make([]string, 0, len(r.peers))
@@ -93,7 +94,42 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 }
 
 func (r *RTC) VideoListenerRead() {
-	listenerRead(r.videoListener, r.videoTrack)
+	duration := time.Second / time.Duration(60)
+	ticker := time.NewTicker(duration)
+	defer ticker.Stop()
+
+	// Retrieve SPS and PPS once at the start
+	sps, err := mpp.GetSPS()
+	if err != nil {
+		log.Fatalf("Failed to retrieve SPS: %v", err)
+	}
+
+	firstFrame := true
+	for {
+		select {
+		case <-ticker.C:
+			frame, err := mpp.GetInstance().CaptureAndEncode()
+			if err != nil {
+				log.Errorf("failed to capture frame: %v", err)
+				continue
+			}
+
+			// If this is the first frame or an IDR frame, prepend SPS and PPS
+			if firstFrame || isIDRFrame(frame) {
+				firstFrame = false
+				frame = append(sps, frame...)
+			}
+
+			sample := media.Sample{
+				Data:     frame,
+				Duration: duration,
+			}
+			err = r.videoTrack.WriteSample(sample)
+			if err != nil {
+				log.Errorf("failed to write sample: %v", err)
+			}
+		}
+	}
 }
 
 func (r *RTC) AudioListenerRead() {
@@ -101,7 +137,6 @@ func (r *RTC) AudioListenerRead() {
 }
 
 func (r *RTC) Close() error {
-	r.videoListener.Close()
 	r.audioListener.Close()
 
 	return nil
@@ -146,3 +181,16 @@ func processRTCP(rtpSender *webrtc.RTPSender) {
 		}
 	}()
 }
+
+func isIDRFrame(frame []byte) bool {
+	// Check for NAL unit type 5 (IDR)
+	for i := 0; i < len(frame)-4; i++ {
+		if frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x00 && frame[i+3] == 0x01 {
+			nalType := frame[i+4] & 0x1F
+			if nalType == 5 { // IDR frame
+				return true
+			}
+		}
+	}
+	return false
+}

stream/video.go

@@ -193,6 +193,6 @@ func WebRTCSettings(c *gin.Context) {
 		ffmpeg.SetFPS(fps)
 	}
 
-	ffmpeg.Stop()
+	//ffmpeg.Stop()
-	ffmpeg.Start()
+	//ffmpeg.Start()
 }