From fc2273d40b343793b0cc613532e4dd671164ff49 Mon Sep 17 00:00:00 2001 From: Artem Date: Tue, 19 Nov 2024 22:08:12 +0100 Subject: [PATCH] hw: add draft rockchip-mpp hw video encoding --- Dockerfile | 20 +++- cmd/kvm/main.go | 7 ++ external/mpp/mpp.c | 245 +++++++++++++++++++++++++++++++++++++++ external/mpp/mpp.go | 82 +++++++++++++ external/mpp/mpp.h | 10 ++ http/route/nanokvm_ui.go | 17 ++- http/rtc/listener.go | 10 +- http/rtc/webrtc.go | 66 +++++++++-- stream/video.go | 4 +- 9 files changed, 429 insertions(+), 32 deletions(-) create mode 100644 external/mpp/mpp.c create mode 100644 external/mpp/mpp.go create mode 100644 external/mpp/mpp.h diff --git a/Dockerfile b/Dockerfile index c6f1384..85b0067 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,11 +12,26 @@ RUN mv /web/dist /static #RUN mv nanokvm/web/dist /static FROM golang:alpine as backend -RUN apk add --no-cache git +RUN apk add --no-cache git v4l-utils-dev build-base musl-dev gcc g++ make libc-dev linux-headers \ + git cmake +RUN mkdir -p ~/dev && cd ~/dev && \ + #git clone -b jellyfin-mpp --depth=1 https://github.com/nyanmisaka/mpp.git rkmpp && \ + git clone --depth=1 https://github.com/rockchip-linux/mpp.git rkmpp && \ + cd rkmpp && mkdir rkmpp_build && cd rkmpp_build && \ + sed -i '/#include /a #include ' ~/dev/rkmpp/mpp/vproc/vdpp/test/hwpq_test.cpp && \ + cmake \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TEST=OFF \ + -DVDPP_TEST=OFF \ + .. && \ + make -j $(nproc) && \ + make install COPY . /app WORKDIR /app RUN go mod download -RUN go build -o ./bin/ ./cmd/... +RUN CGO_ENABLED=1 go build -o ./bin/ ./cmd/... # we would use ffmpeg-rockchip from nyanmisaka with rockchip's mpp support FROM alpine as ffmpeg @@ -74,7 +89,6 @@ RUN mkdir -p ~/dev && cd ~/dev && \ && \ make -j $(nproc) && make install - FROM pikvm/ustreamer:latest as ustreamer FROM alpine diff --git a/cmd/kvm/main.go b/cmd/kvm/main.go index 968dd96..dd8ce5c 100644 --- a/cmd/kvm/main.go +++ b/cmd/kvm/main.go @@ -4,6 +4,7 @@ import ( "fmt" "rkkvm/config" "rkkvm/external/ffmpeg" + "rkkvm/external/mpp" "rkkvm/external/process" "rkkvm/http/route" "rkkvm/http/rtc" @@ -31,6 +32,12 @@ func main() { log.Println("Failed to parse log level, use default level: info") } + // FIXME: hardcoded device + if err := mpp.NewTest("/dev/video0"); err != nil { + log.Fatalf("Failed to open device: %v", err) + } + defer mpp.GetInstance().Close() + webrtc, err := rtc.InitListener(cfg.WebRtc.Host, cfg.WebRtc.VideoPort, cfg.WebRtc.AudioPort) if err != nil { log.Fatal(err) diff --git a/external/mpp/mpp.c b/external/mpp/mpp.c new file mode 100644 index 0000000..e90bb9b --- /dev/null +++ b/external/mpp/mpp.c @@ -0,0 +1,245 @@ +#include "mpp.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define WIDTH 1920 +#define HEIGHT 1080 +#define NUM_PLANES 1 +#define BUFFER_COUNT 4 + +#define SPS_MAX_SIZE 64 + +static unsigned char sps_buffer[SPS_MAX_SIZE]; +static int sps_size = 0; + +static int video_fd = -1; +static void *buffers[BUFFER_COUNT]; +static struct v4l2_buffer v4l2_buf; +static MppCtx mpp_ctx = NULL; +static MppApi *mpp_api = NULL; +static MppEncCfg cfg; + +int video_init(const char *device) { + video_fd = open(device, O_RDWR); + if (video_fd < 0) { + perror("Failed to open video device"); + return -1; + } + + // Set video format + struct v4l2_format fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + fmt.fmt.pix_mp.width = WIDTH; + fmt.fmt.pix_mp.height = HEIGHT; + fmt.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_BGR24; + fmt.fmt.pix_mp.num_planes = NUM_PLANES; + + if (ioctl(video_fd, VIDIOC_S_FMT, &fmt) < 0) { + perror("Failed to set video format"); + close(video_fd); + return -1; + } + + // Request buffers + struct v4l2_requestbuffers req; + memset(&req, 0, sizeof(req)); + req.count = BUFFER_COUNT; + req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + req.memory = V4L2_MEMORY_MMAP; + + if (ioctl(video_fd, VIDIOC_REQBUFS, &req) < 0) { + perror("Failed to request buffers"); + close(video_fd); + return -1; + } + + // Map buffers + struct v4l2_plane planes[1]; + for (int i = 0; i < BUFFER_COUNT; i++) { + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, sizeof(planes)); + v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + v4l2_buf.memory = V4L2_MEMORY_MMAP; + v4l2_buf.index = i; + v4l2_buf.m.planes = planes; + v4l2_buf.length = 1; + + if (ioctl(video_fd, VIDIOC_QUERYBUF, &v4l2_buf) < 0) { + perror("Failed to query buffer"); + close(video_fd); + return -1; + } + + buffers[i] = mmap(NULL, planes[0].length, PROT_READ | PROT_WRITE, MAP_SHARED, video_fd, planes[0].m.mem_offset); + if (buffers[i] == MAP_FAILED) { + perror("Failed to map buffer"); + close(video_fd); + return -1; + } + + // Queue buffer + if (ioctl(video_fd, VIDIOC_QBUF, &v4l2_buf) < 0) { + perror("Failed to queue buffer"); + close(video_fd); + return -1; + } + } + + // Start streaming + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + if (ioctl(video_fd, VIDIOC_STREAMON, &type) < 0) { + perror("Failed to start streaming"); + close(video_fd); + return -1; + } + + return 0; +} + +MppEncCfg mpp_enc_cfg(int fps, int bitrate) { + MppEncCfg cfg; + mpp_enc_cfg_init(&cfg); + mpp_enc_cfg_set_s32(cfg, "prep:width", WIDTH); + mpp_enc_cfg_set_s32(cfg, "prep:height", HEIGHT); + mpp_enc_cfg_set_s32(cfg, "prep:format", MPP_FMT_BGR888); + mpp_enc_cfg_set_s32(cfg, "prep:hor_stride", WIDTH * 3); + mpp_enc_cfg_set_s32(cfg, "prep:ver_stride", HEIGHT); + + mpp_enc_cfg_set_s32(cfg, "rc:mode", MPP_ENC_RC_MODE_CBR); + + mpp_enc_cfg_set_s32(cfg, "rc:fps_in_flex", 0); // Fixed input FPS + mpp_enc_cfg_set_s32(cfg, "rc:fps_in_num", fps); // Input FPS numerator + mpp_enc_cfg_set_s32(cfg, "rc:fps_in_denorm", 1); // Input FPS denominator + mpp_enc_cfg_set_s32(cfg, "rc:fps_out_flex", 0); // Fixed output FPS + mpp_enc_cfg_set_s32(cfg, "rc:fps_out_num", fps); // Output FPS numerator + mpp_enc_cfg_set_s32(cfg, "rc:fps_out_denorm", 1); // Output FPS denominator + + mpp_enc_cfg_set_s32(cfg, "rc:gop", 5); + + mpp_enc_cfg_set_u32(cfg, "rc:drop_mode", MPP_ENC_RC_DROP_FRM_DISABLED); + mpp_enc_cfg_set_u32(cfg, "rc:drop_thd", 20); /* 20% of max bps */ + mpp_enc_cfg_set_u32(cfg, "rc:drop_gap", 1); /* Do not continuous drop frame */ + + mpp_enc_cfg_set_s32(cfg, "rc:bps_target", bitrate); + mpp_enc_cfg_set_s32(cfg, "rc:bps_max", bitrate * 17 / 16); + mpp_enc_cfg_set_s32(cfg, "rc:bps_min", bitrate * 15 / 16); + + return cfg; +} + +int mpp_init_wrapper(int fps, int bitrate) { + MPP_RET ret = mpp_create(&mpp_ctx, &mpp_api); + if (ret != MPP_OK) + return -1; + + ret = mpp_init(mpp_ctx, MPP_CTX_ENC, MPP_VIDEO_CodingAVC); + if (ret != MPP_OK) + return -1; + + MppPollType poll_type = MPP_POLL_BLOCK; + ret = mpp_api->control(mpp_ctx, MPP_SET_OUTPUT_TIMEOUT,(MppParam)&poll_type); + if (ret != MPP_OK) { + mpp_destroy(mpp_ctx); + return -1; + } + + cfg = mpp_enc_cfg(fps, bitrate); + ret = mpp_api->control(mpp_ctx, MPP_ENC_SET_CFG, cfg); + if (ret != MPP_OK) { + mpp_destroy(mpp_ctx); + return -1; + } + + // Get SPS and PPS information + MppPacket extra_pkt = NULL; + ret = mpp_api->control(mpp_ctx, MPP_ENC_GET_EXTRA_INFO, &extra_pkt); + if (ret == MPP_OK && extra_pkt) { + void *extra_data = mpp_packet_get_pos(extra_pkt); + size_t extra_size = mpp_packet_get_length(extra_pkt); + + memcpy(sps_buffer, extra_data, extra_size); + sps_size = extra_size; + + mpp_packet_deinit(&extra_pkt); + } else { + fprintf(stderr, "Failed to get SPS/PPS data from encoder\n"); + } + + return 0; +} + +int get_sps(unsigned char *output, int *output_len) { + if (sps_size == 0) { + fprintf(stderr, "SPS data not available\n"); + return -1; + } + + memcpy(output, sps_buffer, sps_size); + *output_len = sps_size; + + return 0; +} + +int capture_and_encode(unsigned char *output, int *output_len) { + // Dequeue buffer + struct v4l2_plane planes[1]; + memset(&v4l2_buf, 0, sizeof(v4l2_buf)); + memset(planes, 0, sizeof(planes)); + v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; + v4l2_buf.memory = V4L2_MEMORY_MMAP; + v4l2_buf.m.planes = planes; + v4l2_buf.length = NUM_PLANES; + + if (ioctl(video_fd, VIDIOC_DQBUF, &v4l2_buf) < 0) { + perror("Failed to dequeue buffer"); + return -1; + } + + // Encode frame + MppFrame frame; + mpp_frame_init(&frame); + mpp_frame_set_width(frame, WIDTH); + mpp_frame_set_height(frame, HEIGHT); + mpp_frame_set_fmt(frame, MPP_FMT_BGR888); + + MppBuffer frm_buf; + mpp_buffer_get(NULL, &frm_buf, planes[0].length); + memcpy(mpp_buffer_get_ptr(frm_buf), buffers[v4l2_buf.index], planes->bytesused); + mpp_frame_set_buffer(frame, frm_buf); + + MPP_RET ret = mpp_api->encode_put_frame(mpp_ctx, frame); + if (ret == MPP_OK) { + MppPacket packet; + ret = mpp_api->encode_get_packet(mpp_ctx, &packet); + if (ret == MPP_OK) { + *output_len = mpp_packet_get_length(packet); + memcpy(output, mpp_packet_get_data(packet), *output_len); + mpp_packet_deinit(&packet); + } + } + + mpp_frame_deinit(&frame); + mpp_buffer_put(frm_buf); + + // Requeue buffer + ioctl(video_fd, VIDIOC_QBUF, &v4l2_buf); + return ret == MPP_OK ? 0 : -1; +} + +void cleanup() { + //if (cfg) mpp_enc_cfg_deinit(cfg); + if (mpp_ctx) mpp_destroy(mpp_ctx); + if (video_fd >= 0) close(video_fd); +} diff --git a/external/mpp/mpp.go b/external/mpp/mpp.go new file mode 100644 index 0000000..9dc611d --- /dev/null +++ b/external/mpp/mpp.go @@ -0,0 +1,82 @@ +package mpp + +/* +#cgo CFLAGS: -I/usr/include/rockchip/mpp +#cgo LDFLAGS: -L/usr/lib -lv4l2 -lrockchip_mpp +#include "mpp.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +*/ +import "C" +import ( + "errors" + "fmt" + "unsafe" +) + +var mpp *MPP + +type MPP struct { +} + +func New(device string, fps int, bitrate int) (*MPP, error) { + cDevice := C.CString(device) + defer C.free(unsafe.Pointer(cDevice)) + + if C.video_init(cDevice) != 0 { + return nil, errors.New("failed to initialize video capture") + } + if C.mpp_init_wrapper(C.int(fps), C.int(bitrate)) != 0 { + return nil, errors.New("failed to initialize MPP") + } + return &MPP{}, nil +} + +func GetSPS() ([]byte, error) { + output := make([]byte, 64) // Allocate buffer for SPS + var outputLen C.int + + ret := C.get_sps((*C.uchar)(unsafe.Pointer(&output[0])), &outputLen) + if ret != 0 { + return nil, fmt.Errorf("failed to retrieve SPS: %v", ret) + } + + return output[:outputLen], nil +} + +func (m *MPP) CaptureAndEncode() ([]byte, error) { + output := make([]byte, 1024*1024) // 1MB buffer + var outputLen C.int + + ret := C.capture_and_encode((*C.uchar)(unsafe.Pointer(&output[0])), &outputLen) + if ret != 0 { + return nil, errors.New("failed to capture and encode frame") + } + + return output[:outputLen], nil +} + +func (m *MPP) Close() { + C.cleanup() +} + +func NewTest(device string) error { + var err error + mpp, err = New(device, 60, 10000*1000) + return err +} + +func GetInstance() *MPP { + return mpp +} diff --git a/external/mpp/mpp.h b/external/mpp/mpp.h new file mode 100644 index 0000000..3d04a85 --- /dev/null +++ b/external/mpp/mpp.h @@ -0,0 +1,10 @@ +#ifndef MPP_H +#define MPP_H + +int video_init(const char *device); +int mpp_init_wrapper(int fps, int bitrate); +int get_sps(unsigned char *output, int *output_len); +int capture_and_encode(unsigned char *output, int *output_len); +void cleanup(); + +#endif diff --git a/http/route/nanokvm_ui.go b/http/route/nanokvm_ui.go index 529b7c6..44ae11f 100644 --- a/http/route/nanokvm_ui.go +++ b/http/route/nanokvm_ui.go @@ -5,7 +5,6 @@ import ( "rkkvm/external/ffmpeg" "rkkvm/http/middleware" "rkkvm/http/reqrsp" - "time" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -71,14 +70,14 @@ func SetScreen(c *gin.Context) { }) return } - - log.Debug("Stopping ffmpeg SetScreen") - ffmpeg.Stop() - time.Sleep(100 * time.Millisecond) - ffmpeg.ApplyOptions() - log.Debug("Starting ffmpeg SetScreen") - ffmpeg.Start() - + /* + log.Debug("Stopping ffmpeg SetScreen") + ffmpeg.Stop() + time.Sleep(100 * time.Millisecond) + ffmpeg.ApplyOptions() + log.Debug("Starting ffmpeg SetScreen") + ffmpeg.Start() + */ log.Debugf("update screen: %+v", req) c.JSON(http.StatusOK, reqrsp.NanoKVMRsp{ Msg: reqrsp.MsgSuccess, diff --git a/http/rtc/listener.go b/http/rtc/listener.go index 7404901..a8d436a 100644 --- a/http/rtc/listener.go +++ b/http/rtc/listener.go @@ -26,11 +26,6 @@ func initUDPListener(host string, port int) (*net.UDPConn, error) { } func InitListener(host string, port int, aPort int) (*RTC, error) { - vl, err := initUDPListener(host, port) - if err != nil { - return nil, err - } - al, err := initUDPListener(host, aPort) if err != nil { return nil, err @@ -40,16 +35,13 @@ func InitListener(host string, port int, aPort int) (*RTC, error) { switch config.Get().Video.Codec { case config.StreamSourceH264: mimeType = webrtc.MimeTypeH264 - case config.StreamSourceHevc: // WebRTC currently has no official support for H265 - mimeType = webrtc.MimeTypeH265 default: return nil, ErrWebRTCParam("unknown video codec: %s", config.Get().Video.Codec) } - video, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm") + video, _ := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm") audio, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "rkkvm") rtc = &RTC{ - videoListener: vl, audioListener: al, peers: make(map[string]*webrtc.PeerConnection), videoTrack: video, diff --git a/http/rtc/webrtc.go b/http/rtc/webrtc.go index a063644..bcc2180 100644 --- a/http/rtc/webrtc.go +++ b/http/rtc/webrtc.go @@ -4,13 +4,15 @@ import ( "errors" "fmt" "net" - "rkkvm/external/ffmpeg" + "rkkvm/external/mpp" "sync" + "time" "github.com/google/uuid" log "github.com/sirupsen/logrus" "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" ) var rtc *RTC @@ -27,9 +29,8 @@ var ErrPeerClosedConn = ErrWebRTCParam("peer closed conn") type RTC struct { peers map[string]*webrtc.PeerConnection - videoListener *net.UDPConn audioListener *net.UDPConn - videoTrack *webrtc.TrackLocalStaticRTP + videoTrack *webrtc.TrackLocalStaticSample audioTrack *webrtc.TrackLocalStaticRTP m sync.Mutex } @@ -38,10 +39,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription) peerID := uuid.New().String() r.m.Lock() r.peers[peerID] = p - if len(r.peers) == 1 { + /*if len(r.peers) == 1 { ffmpeg.GetFFmpeg().Start() log.Info("FFmpeg process started") - } + }*/ r.m.Unlock() p.OnConnectionStateChange(func(connState webrtc.PeerConnectionState) { @@ -49,10 +50,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription) r.m.Lock() defer r.m.Unlock() delete(r.peers, peerID) - if len(r.peers) == 0 { + /*if len(r.peers) == 0 { ffmpeg.GetFFmpeg().Stop() log.Info("No clients anymore, stop ffmpeg process") - } + }*/ p.Close() peers := make([]string, 0, len(r.peers)) @@ -93,7 +94,42 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription) } func (r *RTC) VideoListenerRead() { - listenerRead(r.videoListener, r.videoTrack) + duration := time.Second / time.Duration(60) + ticker := time.NewTicker(duration) + defer ticker.Stop() + + // Retrieve SPS and PPS once at the start + sps, err := mpp.GetSPS() + if err != nil { + log.Fatalf("Failed to retrieve SPS: %v", err) + } + + firstFrame := true + for { + select { + case <-ticker.C: + frame, err := mpp.GetInstance().CaptureAndEncode() + if err != nil { + log.Errorf("failed to capture frame: %v", err) + continue + } + + // If this is the first frame or an IDR frame, prepend SPS and PPS + if firstFrame || isIDRFrame(frame) { + firstFrame = false + frame = append(sps, frame...) + } + + sample := media.Sample{ + Data: frame, + Duration: duration, + } + err = r.videoTrack.WriteSample(sample) + if err != nil { + log.Errorf("failed to write sample: %v", err) + } + } + } } func (r *RTC) AudioListenerRead() { @@ -101,7 +137,6 @@ func (r *RTC) AudioListenerRead() { } func (r *RTC) Close() error { - r.videoListener.Close() r.audioListener.Close() return nil @@ -146,3 +181,16 @@ func processRTCP(rtpSender *webrtc.RTPSender) { } }() } + +func isIDRFrame(frame []byte) bool { + // Check for NAL unit type 5 (IDR) + for i := 0; i < len(frame)-4; i++ { + if frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x00 && frame[i+3] == 0x01 { + nalType := frame[i+4] & 0x1F + if nalType == 5 { // IDR frame + return true + } + } + } + return false +} diff --git a/stream/video.go b/stream/video.go index fcf552c..794f78a 100644 --- a/stream/video.go +++ b/stream/video.go @@ -193,6 +193,6 @@ func WebRTCSettings(c *gin.Context) { ffmpeg.SetFPS(fps) } - ffmpeg.Stop() - ffmpeg.Start() + //ffmpeg.Stop() + //ffmpeg.Start() }