Artem/rkKVM
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hw: add draft rockchip-mpp hw video encoding

Browse Source
This commit is contained in:
Artem
2024-11-19 22:08:12 +01:00
parent c5f77df6b0
commit fc2273d40b
9 changed files with 429 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
Dockerfile
View File

@@ -12,11 +12,26 @@ RUN mv /web/dist /static
#RUN mv nanokvm/web/dist /static
FROM golang:alpine as backend
RUN apk add --no-cache git
RUN apk add --no-cache git v4l-utils-dev build-base musl-dev gcc g++ make libc-dev linux-headers \
git cmake
RUN mkdir -p ~/dev && cd ~/dev && \
#git clone -b jellyfin-mpp --depth=1 https://github.com/nyanmisaka/mpp.git rkmpp && \
git clone --depth=1 https://github.com/rockchip-linux/mpp.git rkmpp && \
cd rkmpp && mkdir rkmpp_build && cd rkmpp_build && \
sed -i '/#include <errno.h>/a #include <cstdint>' ~/dev/rkmpp/mpp/vproc/vdpp/test/hwpq_test.cpp && \
cmake \
-DCMAKE_INSTALL_PREFIX=/usr \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_TEST=OFF \
-DVDPP_TEST=OFF \
.. && \
make -j $(nproc) && \
make install
COPY . /app
WORKDIR /app
RUN go mod download
RUN go build -o ./bin/ ./cmd/...
RUN CGO_ENABLED=1 go build -o ./bin/ ./cmd/...
# we would use ffmpeg-rockchip from nyanmisaka with rockchip's mpp support
FROM alpine as ffmpeg
@@ -74,7 +89,6 @@ RUN mkdir -p ~/dev && cd ~/dev && \
&& \
make -j $(nproc) && make install
FROM pikvm/ustreamer:latest as ustreamer
FROM alpine

7
cmd/kvm/main.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"rkkvm/config"
"rkkvm/external/ffmpeg"
"rkkvm/external/mpp"
"rkkvm/external/process"
"rkkvm/http/route"
"rkkvm/http/rtc"
@@ -31,6 +32,12 @@ func main() {
log.Println("Failed to parse log level, use default level: info")
}
// FIXME: hardcoded device
if err := mpp.NewTest("/dev/video0"); err != nil {
log.Fatalf("Failed to open device: %v", err)
}
defer mpp.GetInstance().Close()
webrtc, err := rtc.InitListener(cfg.WebRtc.Host, cfg.WebRtc.VideoPort, cfg.WebRtc.AudioPort)
if err != nil {
log.Fatal(err)

245
external/mpp/mpp.c vendored Normal file
View File

@@ -0,0 +1,245 @@
#include "mpp.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <linux/videodev2.h>
#include <rockchip/mpp_buffer.h>
#include <rockchip/mpp_err.h>
#include <rockchip/mpp_frame.h>
#include <rockchip/mpp_packet.h>
#include <rockchip/rk_mpi.h>
#define WIDTH 1920
#define HEIGHT 1080
#define NUM_PLANES 1
#define BUFFER_COUNT 4
#define SPS_MAX_SIZE 64
static unsigned char sps_buffer[SPS_MAX_SIZE];
static int sps_size = 0;
static int video_fd = -1;
static void *buffers[BUFFER_COUNT];
static struct v4l2_buffer v4l2_buf;
static MppCtx mpp_ctx = NULL;
static MppApi *mpp_api = NULL;
static MppEncCfg cfg;
int video_init(const char *device) {
video_fd = open(device, O_RDWR);
if (video_fd < 0) {
perror("Failed to open video device");
return -1;
}
// Set video format
struct v4l2_format fmt;
memset(&fmt, 0, sizeof(fmt));
fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
fmt.fmt.pix_mp.width = WIDTH;
fmt.fmt.pix_mp.height = HEIGHT;
fmt.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_BGR24;
fmt.fmt.pix_mp.num_planes = NUM_PLANES;
if (ioctl(video_fd, VIDIOC_S_FMT, &fmt) < 0) {
perror("Failed to set video format");
close(video_fd);
return -1;
}
// Request buffers
struct v4l2_requestbuffers req;
memset(&req, 0, sizeof(req));
req.count = BUFFER_COUNT;
req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
req.memory = V4L2_MEMORY_MMAP;
if (ioctl(video_fd, VIDIOC_REQBUFS, &req) < 0) {
perror("Failed to request buffers");
close(video_fd);
return -1;
}
// Map buffers
struct v4l2_plane planes[1];
for (int i = 0; i < BUFFER_COUNT; i++) {
memset(&v4l2_buf, 0, sizeof(v4l2_buf));
memset(planes, 0, sizeof(planes));
v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
v4l2_buf.memory = V4L2_MEMORY_MMAP;
v4l2_buf.index = i;
v4l2_buf.m.planes = planes;
v4l2_buf.length = 1;
if (ioctl(video_fd, VIDIOC_QUERYBUF, &v4l2_buf) < 0) {
perror("Failed to query buffer");
close(video_fd);
return -1;
}
buffers[i] = mmap(NULL, planes[0].length, PROT_READ | PROT_WRITE, MAP_SHARED, video_fd, planes[0].m.mem_offset);
if (buffers[i] == MAP_FAILED) {
perror("Failed to map buffer");
close(video_fd);
return -1;
}
// Queue buffer
if (ioctl(video_fd, VIDIOC_QBUF, &v4l2_buf) < 0) {
perror("Failed to queue buffer");
close(video_fd);
return -1;
}
}
// Start streaming
int type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
if (ioctl(video_fd, VIDIOC_STREAMON, &type) < 0) {
perror("Failed to start streaming");
close(video_fd);
return -1;
}
return 0;
}
MppEncCfg mpp_enc_cfg(int fps, int bitrate) {
MppEncCfg cfg;
mpp_enc_cfg_init(&cfg);
mpp_enc_cfg_set_s32(cfg, "prep:width", WIDTH);
mpp_enc_cfg_set_s32(cfg, "prep:height", HEIGHT);
mpp_enc_cfg_set_s32(cfg, "prep:format", MPP_FMT_BGR888);
mpp_enc_cfg_set_s32(cfg, "prep:hor_stride", WIDTH * 3);
mpp_enc_cfg_set_s32(cfg, "prep:ver_stride", HEIGHT);
mpp_enc_cfg_set_s32(cfg, "rc:mode", MPP_ENC_RC_MODE_CBR);
mpp_enc_cfg_set_s32(cfg, "rc:fps_in_flex", 0); // Fixed input FPS
mpp_enc_cfg_set_s32(cfg, "rc:fps_in_num", fps); // Input FPS numerator
mpp_enc_cfg_set_s32(cfg, "rc:fps_in_denorm", 1); // Input FPS denominator
mpp_enc_cfg_set_s32(cfg, "rc:fps_out_flex", 0); // Fixed output FPS
mpp_enc_cfg_set_s32(cfg, "rc:fps_out_num", fps); // Output FPS numerator
mpp_enc_cfg_set_s32(cfg, "rc:fps_out_denorm", 1); // Output FPS denominator
mpp_enc_cfg_set_s32(cfg, "rc:gop", 5);
mpp_enc_cfg_set_u32(cfg, "rc:drop_mode", MPP_ENC_RC_DROP_FRM_DISABLED);
mpp_enc_cfg_set_u32(cfg, "rc:drop_thd", 20); /* 20% of max bps */
mpp_enc_cfg_set_u32(cfg, "rc:drop_gap", 1); /* Do not continuous drop frame */
mpp_enc_cfg_set_s32(cfg, "rc:bps_target", bitrate);
mpp_enc_cfg_set_s32(cfg, "rc:bps_max", bitrate * 17 / 16);
mpp_enc_cfg_set_s32(cfg, "rc:bps_min", bitrate * 15 / 16);
return cfg;
}
int mpp_init_wrapper(int fps, int bitrate) {
MPP_RET ret = mpp_create(&mpp_ctx, &mpp_api);
if (ret != MPP_OK)
return -1;
ret = mpp_init(mpp_ctx, MPP_CTX_ENC, MPP_VIDEO_CodingAVC);
if (ret != MPP_OK)
return -1;
MppPollType poll_type = MPP_POLL_BLOCK;
ret = mpp_api->control(mpp_ctx, MPP_SET_OUTPUT_TIMEOUT,(MppParam)&poll_type);
if (ret != MPP_OK) {
mpp_destroy(mpp_ctx);
return -1;
}
cfg = mpp_enc_cfg(fps, bitrate);
ret = mpp_api->control(mpp_ctx, MPP_ENC_SET_CFG, cfg);
if (ret != MPP_OK) {
mpp_destroy(mpp_ctx);
return -1;
}
// Get SPS and PPS information
MppPacket extra_pkt = NULL;
ret = mpp_api->control(mpp_ctx, MPP_ENC_GET_EXTRA_INFO, &extra_pkt);
if (ret == MPP_OK && extra_pkt) {
void *extra_data = mpp_packet_get_pos(extra_pkt);
size_t extra_size = mpp_packet_get_length(extra_pkt);
memcpy(sps_buffer, extra_data, extra_size);
sps_size = extra_size;
mpp_packet_deinit(&extra_pkt);
} else {
fprintf(stderr, "Failed to get SPS/PPS data from encoder\n");
}
return 0;
}
int get_sps(unsigned char *output, int *output_len) {
if (sps_size == 0) {
fprintf(stderr, "SPS data not available\n");
return -1;
}
memcpy(output, sps_buffer, sps_size);
*output_len = sps_size;
return 0;
}
int capture_and_encode(unsigned char *output, int *output_len) {
// Dequeue buffer
struct v4l2_plane planes[1];
memset(&v4l2_buf, 0, sizeof(v4l2_buf));
memset(planes, 0, sizeof(planes));
v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
v4l2_buf.memory = V4L2_MEMORY_MMAP;
v4l2_buf.m.planes = planes;
v4l2_buf.length = NUM_PLANES;
if (ioctl(video_fd, VIDIOC_DQBUF, &v4l2_buf) < 0) {
perror("Failed to dequeue buffer");
return -1;
}
// Encode frame
MppFrame frame;
mpp_frame_init(&frame);
mpp_frame_set_width(frame, WIDTH);
mpp_frame_set_height(frame, HEIGHT);
mpp_frame_set_fmt(frame, MPP_FMT_BGR888);
MppBuffer frm_buf;
mpp_buffer_get(NULL, &frm_buf, planes[0].length);
memcpy(mpp_buffer_get_ptr(frm_buf), buffers[v4l2_buf.index], planes->bytesused);
mpp_frame_set_buffer(frame, frm_buf);
MPP_RET ret = mpp_api->encode_put_frame(mpp_ctx, frame);
if (ret == MPP_OK) {
MppPacket packet;
ret = mpp_api->encode_get_packet(mpp_ctx, &packet);
if (ret == MPP_OK) {
*output_len = mpp_packet_get_length(packet);
memcpy(output, mpp_packet_get_data(packet), *output_len);
mpp_packet_deinit(&packet);
}
}
mpp_frame_deinit(&frame);
mpp_buffer_put(frm_buf);
// Requeue buffer
ioctl(video_fd, VIDIOC_QBUF, &v4l2_buf);
return ret == MPP_OK ? 0 : -1;
}
void cleanup() {
//if (cfg) mpp_enc_cfg_deinit(cfg);
if (mpp_ctx) mpp_destroy(mpp_ctx);
if (video_fd >= 0) close(video_fd);
}

82
external/mpp/mpp.go vendored Normal file
View File

@@ -0,0 +1,82 @@
package mpp
/*
#cgo CFLAGS: -I/usr/include/rockchip/mpp
#cgo LDFLAGS: -L/usr/lib -lv4l2 -lrockchip_mpp
#include "mpp.h"
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <linux/videodev2.h>
#include <rockchip/mpp_buffer.h>
#include <rockchip/mpp_err.h>
#include <rockchip/mpp_frame.h>
#include <rockchip/mpp_packet.h>
#include <rockchip/rk_mpi.h>
*/
import "C"
import (
"errors"
"fmt"
"unsafe"
)
var mpp *MPP
type MPP struct {
}
func New(device string, fps int, bitrate int) (*MPP, error) {
cDevice := C.CString(device)
defer C.free(unsafe.Pointer(cDevice))
if C.video_init(cDevice) != 0 {
return nil, errors.New("failed to initialize video capture")
}
if C.mpp_init_wrapper(C.int(fps), C.int(bitrate)) != 0 {
return nil, errors.New("failed to initialize MPP")
}
return &MPP{}, nil
}
func GetSPS() ([]byte, error) {
output := make([]byte, 64) // Allocate buffer for SPS
var outputLen C.int
ret := C.get_sps((*C.uchar)(unsafe.Pointer(&output[0])), &outputLen)
if ret != 0 {
return nil, fmt.Errorf("failed to retrieve SPS: %v", ret)
}
return output[:outputLen], nil
}
func (m *MPP) CaptureAndEncode() ([]byte, error) {
output := make([]byte, 1024*1024) // 1MB buffer
var outputLen C.int
ret := C.capture_and_encode((*C.uchar)(unsafe.Pointer(&output[0])), &outputLen)
if ret != 0 {
return nil, errors.New("failed to capture and encode frame")
}
return output[:outputLen], nil
}
func (m *MPP) Close() {
C.cleanup()
}
func NewTest(device string) error {
var err error
mpp, err = New(device, 60, 10000*1000)
return err
}
func GetInstance() *MPP {
return mpp
}

10
external/mpp/mpp.h vendored Normal file
View File

@@ -0,0 +1,10 @@
#ifndef MPP_H
#define MPP_H
int video_init(const char *device);
int mpp_init_wrapper(int fps, int bitrate);
int get_sps(unsigned char *output, int *output_len);
int capture_and_encode(unsigned char *output, int *output_len);
void cleanup();
#endif

5
http/route/nanokvm_ui.go
View File

@@ -5,7 +5,6 @@ import (
"rkkvm/external/ffmpeg"
"rkkvm/http/middleware"
"rkkvm/http/reqrsp"
"time"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
@@ -71,14 +70,14 @@ func SetScreen(c *gin.Context) {
})
return
}
/*
log.Debug("Stopping ffmpeg SetScreen")
ffmpeg.Stop()
time.Sleep(100 * time.Millisecond)
ffmpeg.ApplyOptions()
log.Debug("Starting ffmpeg SetScreen")
ffmpeg.Start()
*/
log.Debugf("update screen: %+v", req)
c.JSON(http.StatusOK, reqrsp.NanoKVMRsp{
Msg: reqrsp.MsgSuccess,

10
http/rtc/listener.go
View File

@@ -26,11 +26,6 @@ func initUDPListener(host string, port int) (*net.UDPConn, error) {
}
func InitListener(host string, port int, aPort int) (*RTC, error) {
vl, err := initUDPListener(host, port)
if err != nil {
return nil, err
}
al, err := initUDPListener(host, aPort)
if err != nil {
return nil, err
@@ -40,16 +35,13 @@ func InitListener(host string, port int, aPort int) (*RTC, error) {
switch config.Get().Video.Codec {
case config.StreamSourceH264:
mimeType = webrtc.MimeTypeH264
case config.StreamSourceHevc: // WebRTC currently has no official support for H265
mimeType = webrtc.MimeTypeH265
default:
return nil, ErrWebRTCParam("unknown video codec: %s", config.Get().Video.Codec)
}
video, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
video, _ := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
audio, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "rkkvm")
rtc = &RTC{
videoListener: vl,
audioListener: al,
peers: make(map[string]*webrtc.PeerConnection),
videoTrack: video,

66
http/rtc/webrtc.go
View File

@@ -4,13 +4,15 @@ import (
"errors"
"fmt"
"net"
"rkkvm/external/ffmpeg"
"rkkvm/external/mpp"
"sync"
"time"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/pion/webrtc/v4"
"github.com/pion/webrtc/v4/pkg/media"
)
var rtc *RTC
@@ -27,9 +29,8 @@ var ErrPeerClosedConn = ErrWebRTCParam("peer closed conn")
type RTC struct {
peers map[string]*webrtc.PeerConnection
videoListener *net.UDPConn
audioListener *net.UDPConn
videoTrack *webrtc.TrackLocalStaticRTP
videoTrack *webrtc.TrackLocalStaticSample
audioTrack *webrtc.TrackLocalStaticRTP
m sync.Mutex
}
@@ -38,10 +39,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
peerID := uuid.New().String()
r.m.Lock()
r.peers[peerID] = p
if len(r.peers) == 1 {
/*if len(r.peers) == 1 {
ffmpeg.GetFFmpeg().Start()
log.Info("FFmpeg process started")
}
}*/
r.m.Unlock()
p.OnConnectionStateChange(func(connState webrtc.PeerConnectionState) {
@@ -49,10 +50,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
r.m.Lock()
defer r.m.Unlock()
delete(r.peers, peerID)
if len(r.peers) == 0 {
/*if len(r.peers) == 0 {
ffmpeg.GetFFmpeg().Stop()
log.Info("No clients anymore, stop ffmpeg process")
}
}*/
p.Close()
peers := make([]string, 0, len(r.peers))
@@ -93,7 +94,42 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
}
func (r *RTC) VideoListenerRead() {
listenerRead(r.videoListener, r.videoTrack)
duration := time.Second / time.Duration(60)
ticker := time.NewTicker(duration)
defer ticker.Stop()
// Retrieve SPS and PPS once at the start
sps, err := mpp.GetSPS()
if err != nil {
log.Fatalf("Failed to retrieve SPS: %v", err)
}
firstFrame := true
for {
select {
case <-ticker.C:
frame, err := mpp.GetInstance().CaptureAndEncode()
if err != nil {
log.Errorf("failed to capture frame: %v", err)
continue
}
// If this is the first frame or an IDR frame, prepend SPS and PPS
if firstFrame || isIDRFrame(frame) {
firstFrame = false
frame = append(sps, frame...)
}
sample := media.Sample{
Data: frame,
Duration: duration,
}
err = r.videoTrack.WriteSample(sample)
if err != nil {
log.Errorf("failed to write sample: %v", err)
}
}
}
}
func (r *RTC) AudioListenerRead() {
@@ -101,7 +137,6 @@ func (r *RTC) AudioListenerRead() {
}
func (r *RTC) Close() error {
r.videoListener.Close()
r.audioListener.Close()
return nil
@@ -146,3 +181,16 @@ func processRTCP(rtpSender *webrtc.RTPSender) {
}
}()
}
func isIDRFrame(frame []byte) bool {
// Check for NAL unit type 5 (IDR)
for i := 0; i < len(frame)-4; i++ {
if frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x00 && frame[i+3] == 0x01 {
nalType := frame[i+4] & 0x1F
if nalType == 5 { // IDR frame
return true
}
}
}
return false
}

4
stream/video.go
View File

@@ -193,6 +193,6 @@ func WebRTCSettings(c *gin.Context) {
ffmpeg.SetFPS(fps)
}
ffmpeg.Stop()
ffmpeg.Start()
//ffmpeg.Stop()
//ffmpeg.Start()
}