hw: add draft rockchip-mpp hw video encoding

webrtc: start/stop ffmpeg procees if webrtc has clients
project structure refactoring
2024-11-19 22:08:12 +01:00 · 2024-11-05 23:49:26 +01:00 · 2024-11-05 23:47:00 +01:00 · 2024-11-05 18:49:21 +01:00
23 changed files with 655 additions and 404 deletions
--- a/20
+++ b/20
@@ -12,11 +12,26 @@ RUN mv /web/dist /static
 #RUN mv nanokvm/web/dist /static

 FROM golang:alpine as backend
-RUN apk add --no-cache git
+RUN apk add --no-cache git v4l-utils-dev build-base musl-dev gcc g++ make libc-dev linux-headers \
+    git cmake
+RUN mkdir -p ~/dev && cd ~/dev && \
+    #git clone -b jellyfin-mpp --depth=1 https://github.com/nyanmisaka/mpp.git rkmpp && \
+    git clone --depth=1 https://github.com/rockchip-linux/mpp.git rkmpp && \
+    cd rkmpp && mkdir rkmpp_build && cd rkmpp_build && \
+    sed -i '/#include <errno.h>/a #include <cstdint>' ~/dev/rkmpp/mpp/vproc/vdpp/test/hwpq_test.cpp && \ 
+    cmake \
+        -DCMAKE_INSTALL_PREFIX=/usr \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DBUILD_SHARED_LIBS=ON \
+        -DBUILD_TEST=OFF \
+        -DVDPP_TEST=OFF \
+        .. && \
+    make -j $(nproc) && \
+    make install
 COPY . /app
 WORKDIR /app
 RUN go mod download
-RUN go build -o ./bin/ ./cmd/...
+RUN CGO_ENABLED=1 go build -o ./bin/ ./cmd/...

 # we would use ffmpeg-rockchip from nyanmisaka with rockchip's mpp support
 FROM alpine as ffmpeg
@@ -74,7 +89,6 @@ RUN mkdir -p ~/dev && cd ~/dev && \
    && \
    make -j $(nproc) && make install

-
 FROM pikvm/ustreamer:latest as ustreamer

 FROM alpine
--- a/cmd/kvm/main.go
+++ b/cmd/kvm/main.go
@@ -3,9 +3,11 @@ package main
 import (
 	"fmt"
 	"rkkvm/config"
-	"rkkvm/http/hw/rtc"
-	"rkkvm/http/hw/stream"
+	"rkkvm/external/ffmpeg"
+	"rkkvm/external/mpp"
+	"rkkvm/external/process"
 	"rkkvm/http/route"
+	"rkkvm/http/rtc"
 	"strings"

 	"github.com/gin-gonic/gin"
@@ -30,21 +32,13 @@ func main() {
 		log.Println("Failed to parse log level, use default level: info")
 	}

-	if cfg.Stream.Source == config.StreamSourceMjpeg {
-		ustreamer := stream.Init(cfg.UStreamer.Path, strings.Split(cfg.UStreamer.Args, " "))
-		ustreamer.Start()
-		//go ustreamer.Watch()
-	} else if cfg.Stream.Source == config.StreamSourceH264 {
-		ffmpeg := stream.InitFFmpeg(cfg.Video.Path, cfg.Video.FormatArgs())
-		ffmpeg.Start()
-
-		audio := stream.InitPipedCmd(config.Get().Audio)
-		audio.Start()
-	} else {
-		log.Fatalf("unsupported stream source type: %v", cfg.Stream.Source)
+	// FIXME: hardcoded device
+	if err := mpp.NewTest("/dev/video0"); err != nil {
+		log.Fatalf("Failed to open device: %v", err)
 	}
+	defer mpp.GetInstance().Close()

-	webrtc, err := rtc.InitListener(cfg.WebRtc.Host, cfg.WebRtc.Port, 5006)
+	webrtc, err := rtc.InitListener(cfg.WebRtc.Host, cfg.WebRtc.VideoPort, cfg.WebRtc.AudioPort)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -52,6 +46,16 @@ func main() {
 	go webrtc.VideoListenerRead()
 	go webrtc.AudioListenerRead()

+	if cfg.Stream.Source == config.StreamSourceMjpeg {
+		ustreamer := process.Init(cfg.UStreamer.Path, strings.Split(cfg.UStreamer.Args, " "))
+		ustreamer.Start()
+		//go ustreamer.Watch()
+	} else if cfg.Stream.Source == config.StreamSourceH264 {
+		ffmpeg.InitFFmpeg()
+	} else {
+		log.Fatalf("unsupported stream source type: %v", cfg.Stream.Source)
+	}
+
 	r := gin.Default()
 	r.Use(gin.Recovery())
 	route.Api(r)
--- a/config/config.go
+++ b/config/config.go
@@ -3,7 +3,6 @@ package config
 import (
 	"fmt"
 	"os"
-	"strings"

 	"gopkg.in/yaml.v3"
 )
@@ -15,7 +14,6 @@ type Config struct {
 	Port       int       `yaml:"port"`
 	UStreamer  UStreamer `yaml:"ustreamer"`
 	Video      FFmpeg    `yaml:"video"`
-	Audio      []string  `yaml:"audio"`
 	WebRtc     WebRtc    `yaml:"webrtc"`
 	Stream     Stream    `yaml:"stream"`

@@ -24,8 +22,9 @@ type Config struct {
 }

 type WebRtc struct {
-	Host string `yaml:"host"`
-	Port int    `yaml:"port"`
+	Host      string `yaml:"host"`
+	VideoPort int    `yaml:"video_port"`
+	AudioPort int    `yaml:"audio_port"`
 }

 type ExtProcess struct {
@@ -35,6 +34,7 @@ type ExtProcess struct {

 type Stream struct {
 	Source string `yaml:"source"` // mjpeg or h264
+
 }

 const (
@@ -43,17 +43,24 @@ const (
 	StreamSourceHevc  = "hevc"
 )

+type StreamInput string
+
+const (
+	StreamInputVideo      = "v"
+	StreamInputVideoAudio = "va"
+)
+
 type FFmpeg struct {
-	ExtProcess
-	FPS     int    `yaml:"fps"`
-	Bitrate int    `yaml:"bitrate"`
-	Height  int    `yaml:"height"`
-	GOP     int    `yaml:"gop"`
-	Codec   string `yaml:"codec"`
+	Commands map[StreamInput]string `yaml:"commands"`
+	FPS      int                    `yaml:"fps"`
+	Bitrate  int                    `yaml:"bitrate"`
+	Height   int                    `yaml:"height"`
+	GOP      int                    `yaml:"gop"`
+	Codec    string                 `yaml:"codec"`
 }

-func (f FFmpeg) FormatArgs() []string {
-	return strings.Split(fmt.Sprintf(f.Args, f.Height, f.Codec, f.Bitrate*1000, f.FPS, f.GOP), " ")
+func (f FFmpeg) FormatCmd(cmd string) string {
+	return fmt.Sprintf(cmd, f.Height, f.Codec, f.Bitrate*1000, f.FPS, f.GOP)
 }

 type UStreamer struct {
@@ -68,8 +75,6 @@ func Get() Config {
 	return c
 }

-// arecord -D hw:0,0 -f cd -r 44100 -c 2 | /app/ffmpeg -re -init_hw_device rkmpp=hw -filter_hw_device hw -f wav -i pipe:0 -f v4l2 -c:a aac -b:a 128k -ar 44100 -ac 2 -f rtp rtp://127.0.0.1:5006?pkt_size=1200 -i /dev/video0 -vf hwupload,scale_rkrga=h=720:force_original_aspect_ratio=1 -c:v h264_rkmpp -flags +low_delay -b:v 6000000 -framerate 60 -g 10 -f rtp rtp://127.0.0.1:5004?pkt_size=1200
-
 func Init() {
 	c = Config{
 		LogLevel: "debug",
@@ -84,24 +89,26 @@ func Init() {
 			},
 		},
 		Video: FFmpeg{
-			ExtProcess: ExtProcess{
-				Path: "/app/ffmpeg",
-				Args: "-hide_banner -loglevel error -re -init_hw_device rkmpp=hw -filter_hw_device hw -i /dev/video0 -vf hwupload,scale_rkrga=h=%d:force_original_aspect_ratio=1 -c:v %s_rkmpp -flags +low_delay -b:v %d -framerate %d -g %d -f rtp rtp://127.0.0.1:5004?pkt_size=1200",
-				//Args: "-re -i /dev/video0 -c:v h264_rkmpp -b:v %d -framerate %d -bsf:v h264_mp4toannexb -g 10 -f rtp rtp://127.0.0.1:5004?pkt_size=1200",
+			Commands: map[StreamInput]string{
+				StreamInputVideoAudio: "/usr/bin/arecord -D hw:0,0 -f dat -r 48000 -c 2 --buffer-size=150 | /app/ffmpeg -re -init_hw_device rkmpp=hw -filter_hw_device hw" +
+					" -f wav -i pipe:0 -map 0:a -c:a libopus -b:a 48000 -sample_fmt s16 -ssrc 1 -payload_type 111 -f rtp -max_delay 0 -application lowdelay" +
+					" -f rtp rtp://127.0.0.1:5006?pkt_size=1200" +
+					" -i /dev/video0 -map 1:v -vf hwupload,scale_rkrga=h=%d:force_original_aspect_ratio=1 -c:v %s_rkmpp -flags +low_delay -b:v %d -framerate %d -g %d" +
+					" -f rtp rtp://127.0.0.1:5004?pkt_size=1200",
+				StreamInputVideo: "/app/ffmpeg -re -hide_banner -loglevel error -init_hw_device rkmpp=hw -filter_hw_device hw" +
+					" -i /dev/video0 -vf hwupload,scale_rkrga=h=%d:force_original_aspect_ratio=1 -c:v %s_rkmpp -flags +low_delay -b:v %d -framerate %d -g %d" +
+					" -f rtp rtp://127.0.0.1:5004?pkt_size=1200",
 			},
 			FPS:     60,
 			Bitrate: 6000,
 			Height:  720,
 			GOP:     5,
-			Codec:   "h264",
-		},
-		Audio: []string{
-			"/usr/bin/arecord -D hw:0,0 -f dat -r 48000 -c 2 --buffer-size=60",
-			"/app/ffmpeg -hide_banner -loglevel error -re -f wav -i pipe:0 -c:a libopus -b:a 48000 -sample_fmt s16 -ssrc 1 -payload_type 111 -f rtp -max_delay 0 -application lowdelay -f rtp rtp://127.0.0.1:5006?pkt_size=1200",
+			Codec:   StreamSourceH264,
 		},
 		WebRtc: WebRtc{
-			Host: "127.0.0.1",
-			Port: 5004,
+			Host:      "127.0.0.1",
+			VideoPort: 5004,
+			AudioPort: 5006,
 		},
 		Stream: Stream{
 			Source: StreamSourceH264,
--- a/external/ffmpeg/ffmpeg.go
+++ b/external/ffmpeg/ffmpeg.go
@@ -1,7 +1,8 @@
-package stream
+package ffmpeg

 import (
 	"rkkvm/config"
+	"rkkvm/external/process"
 )

 //h264
@@ -23,6 +24,12 @@ arecord -D hw:0,0 -f cd -r 44100 -c 2 | /app/ffmpeg -re -init_hw_device rkmpp=hw


 arecord -D hw:0,0 -f cd -r 44100 -c 2 | /app/ffmpeg -re -f wav -i pipe:0 -c:a aac -b:a 128k -ar 44100 -ac 2 -f rtp rtp://127.0.0.1:5006?pkt_size=1200
+
+arecord -D hw:0,0 -f dat -r 48000 -c 2 --buffer-size=60 | /app/ffmpeg -re -init_hw_device rkmpp=hw -filter_hw_device hw \
+-f wav -i pipe:0 -map 0:a -c:a libopus -b:a 48000 -sample_fmt s16 -ssrc 1 -payload_type 111 -f rtp -max_delay 0 -application lowdelay \
+-f rtp rtp://127.0.0.1:5006?pkt_size=1200 \
+-i /dev/video0 -map 1:v -vf hwupload,scale_rkrga=h=720:force_original_aspect_ratio=1 -c:v h264_rkmpp -flags +low_delay -b:v 6000000 -framerate 60 -g 10 \
+-f rtp rtp://127.0.0.1:5004?pkt_size=1200
 */

 // https://jsfiddle.net/z7ms3u5r/
@@ -30,50 +37,57 @@ arecord -D hw:0,0 -f cd -r 44100 -c 2 | /app/ffmpeg -re -f wav -i pipe:0 -c:a aa
 var ffmpeg *FFmpeg

 type FFmpeg struct {
-	*ExtProcess
+	*process.PipedCmd
 	config.FFmpeg
 }

+func (f *FFmpeg) getCmd() string {
+	return f.Commands[config.StreamInputVideoAudio]
+}
+
 func (f *FFmpeg) SetBitrate(b int) {
 	f.Bitrate = b
 	if f.Bitrate < 0 {
 		f.Bitrate = 6000
 	}
-
-	f.ChangeArgs(f.FormatArgs())
 }

 func (f *FFmpeg) SetFPS(fps int) {
 	f.FPS = fps
 	if f.FPS < 0 {
 		f.FPS = 30
+	} else if f.FPS > 60 {
+		f.FPS = 60
 	}
-
-	f.ChangeArgs(f.FormatArgs())
 }

 func (f *FFmpeg) SetResolution(height int) {
 	f.Height = height
 	if f.Height <= 0 {
 		f.Height = 1080
+	} else if f.Height > 2060 {
+		f.Height = 2060
 	}
-
-	f.ChangeArgs(f.FormatArgs())
 }

 func (f *FFmpeg) SetGOP(gop int) {
 	f.GOP = gop
 	if f.GOP < 0 {
-		f.GOP = 0
+		f.GOP = 2
 	}
-
-	f.ChangeArgs(f.FormatArgs())
 }

-func InitFFmpeg(path string, args []string) *FFmpeg {
+func (f *FFmpeg) ApplyOptions() {
+	f.ChangeCmd(f.FormatCmd(f.getCmd()))
+}
+
+func InitFFmpeg() *FFmpeg {
+	cfg := config.Get().Video
+	cmd := cfg.Commands[config.StreamInputVideoAudio]
+
 	ffmpeg = &FFmpeg{
-		ExtProcess: Init(path, args),
-		FFmpeg:     config.Get().Video,
+		PipedCmd: process.InitPipedCmd(cfg.FormatCmd(cmd)),
+		FFmpeg:   cfg,
 	}
 	return ffmpeg
 }
--- a/external/mpp/mpp.c
+++ b/external/mpp/mpp.c
@@ -0,0 +1,245 @@
+#include "mpp.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <linux/videodev2.h>
+#include <rockchip/mpp_buffer.h>
+#include <rockchip/mpp_err.h>
+#include <rockchip/mpp_frame.h>
+#include <rockchip/mpp_packet.h>
+#include <rockchip/rk_mpi.h>
+
+#define WIDTH 1920
+#define HEIGHT 1080
+#define NUM_PLANES 1
+#define BUFFER_COUNT 4
+
+#define SPS_MAX_SIZE 64
+
+static unsigned char sps_buffer[SPS_MAX_SIZE];
+static int sps_size = 0;
+
+static int video_fd = -1;
+static void *buffers[BUFFER_COUNT];
+static struct v4l2_buffer v4l2_buf;
+static MppCtx mpp_ctx = NULL;
+static MppApi *mpp_api = NULL;
+static MppEncCfg cfg;
+
+int video_init(const char *device) {
+    video_fd = open(device, O_RDWR);
+    if (video_fd < 0) {
+        perror("Failed to open video device");
+        return -1;
+    }
+
+    // Set video format
+    struct v4l2_format fmt;
+    memset(&fmt, 0, sizeof(fmt));
+    fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    fmt.fmt.pix_mp.width = WIDTH;
+    fmt.fmt.pix_mp.height = HEIGHT;
+    fmt.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_BGR24;
+    fmt.fmt.pix_mp.num_planes = NUM_PLANES;
+
+    if (ioctl(video_fd, VIDIOC_S_FMT, &fmt) < 0) {
+        perror("Failed to set video format");
+        close(video_fd);
+        return -1;
+    }
+
+    // Request buffers
+    struct v4l2_requestbuffers req;
+    memset(&req, 0, sizeof(req));
+    req.count = BUFFER_COUNT;
+    req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    req.memory = V4L2_MEMORY_MMAP;
+
+    if (ioctl(video_fd, VIDIOC_REQBUFS, &req) < 0) {
+        perror("Failed to request buffers");
+        close(video_fd);
+        return -1;
+    }
+
+    // Map buffers
+    struct v4l2_plane planes[1];
+    for (int i = 0; i < BUFFER_COUNT; i++) {
+        memset(&v4l2_buf, 0, sizeof(v4l2_buf));
+        memset(planes, 0, sizeof(planes));
+        v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+        v4l2_buf.memory = V4L2_MEMORY_MMAP;
+        v4l2_buf.index = i;
+        v4l2_buf.m.planes = planes;
+        v4l2_buf.length = 1;
+
+        if (ioctl(video_fd, VIDIOC_QUERYBUF, &v4l2_buf) < 0) {
+            perror("Failed to query buffer");
+            close(video_fd);
+            return -1;
+        }
+
+        buffers[i] = mmap(NULL, planes[0].length, PROT_READ | PROT_WRITE, MAP_SHARED, video_fd, planes[0].m.mem_offset);
+        if (buffers[i] == MAP_FAILED) {
+            perror("Failed to map buffer");
+            close(video_fd);
+            return -1;
+        }
+
+        // Queue buffer
+        if (ioctl(video_fd, VIDIOC_QBUF, &v4l2_buf) < 0) {
+            perror("Failed to queue buffer");
+            close(video_fd);
+            return -1;
+        }
+    }
+
+    // Start streaming
+    int type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    if (ioctl(video_fd, VIDIOC_STREAMON, &type) < 0) {
+        perror("Failed to start streaming");
+        close(video_fd);
+        return -1;
+    }
+
+    return 0;
+}
+
+MppEncCfg mpp_enc_cfg(int fps, int bitrate) {
+    MppEncCfg cfg;
+    mpp_enc_cfg_init(&cfg);
+    mpp_enc_cfg_set_s32(cfg, "prep:width", WIDTH);
+    mpp_enc_cfg_set_s32(cfg, "prep:height", HEIGHT);
+    mpp_enc_cfg_set_s32(cfg, "prep:format", MPP_FMT_BGR888);
+    mpp_enc_cfg_set_s32(cfg, "prep:hor_stride", WIDTH * 3);
+    mpp_enc_cfg_set_s32(cfg, "prep:ver_stride", HEIGHT);
+
+    mpp_enc_cfg_set_s32(cfg, "rc:mode", MPP_ENC_RC_MODE_CBR);
+
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_in_flex", 0);            // Fixed input FPS
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_in_num", fps);            // Input FPS numerator
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_in_denorm", 1);          // Input FPS denominator
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_out_flex", 0);           // Fixed output FPS
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_out_num", fps);           // Output FPS numerator
+    mpp_enc_cfg_set_s32(cfg, "rc:fps_out_denorm", 1);         // Output FPS denominator
+
+    mpp_enc_cfg_set_s32(cfg, "rc:gop", 5);
+
+    mpp_enc_cfg_set_u32(cfg, "rc:drop_mode", MPP_ENC_RC_DROP_FRM_DISABLED);
+    mpp_enc_cfg_set_u32(cfg, "rc:drop_thd", 20);        /* 20% of max bps */
+    mpp_enc_cfg_set_u32(cfg, "rc:drop_gap", 1);         /* Do not continuous drop frame */
+
+    mpp_enc_cfg_set_s32(cfg, "rc:bps_target", bitrate);
+    mpp_enc_cfg_set_s32(cfg, "rc:bps_max", bitrate * 17 / 16);
+    mpp_enc_cfg_set_s32(cfg, "rc:bps_min", bitrate * 15 / 16);
+
+    return cfg;
+}
+
+int mpp_init_wrapper(int fps, int bitrate) {
+    MPP_RET ret = mpp_create(&mpp_ctx, &mpp_api);
+    if (ret != MPP_OK) 
+        return -1;
+
+    ret = mpp_init(mpp_ctx, MPP_CTX_ENC, MPP_VIDEO_CodingAVC);
+    if (ret != MPP_OK)
+        return -1;
+    
+    MppPollType poll_type = MPP_POLL_BLOCK;
+	ret = mpp_api->control(mpp_ctx, MPP_SET_OUTPUT_TIMEOUT,(MppParam)&poll_type);
+    if (ret != MPP_OK) {
+        mpp_destroy(mpp_ctx);
+        return -1;
+    }
+
+    cfg = mpp_enc_cfg(fps, bitrate);
+    ret = mpp_api->control(mpp_ctx, MPP_ENC_SET_CFG, cfg);
+    if (ret != MPP_OK) {
+        mpp_destroy(mpp_ctx);
+        return -1;
+    }
+
+    // Get SPS and PPS information
+    MppPacket extra_pkt = NULL;
+    ret = mpp_api->control(mpp_ctx, MPP_ENC_GET_EXTRA_INFO, &extra_pkt);
+    if (ret == MPP_OK && extra_pkt) {
+        void *extra_data = mpp_packet_get_pos(extra_pkt);
+        size_t extra_size = mpp_packet_get_length(extra_pkt);
+
+        memcpy(sps_buffer, extra_data, extra_size);
+        sps_size = extra_size;
+
+        mpp_packet_deinit(&extra_pkt);
+    } else {
+        fprintf(stderr, "Failed to get SPS/PPS data from encoder\n");
+    }
+
+    return 0;
+}
+
+int get_sps(unsigned char *output, int *output_len) {
+    if (sps_size == 0) {
+        fprintf(stderr, "SPS data not available\n");
+        return -1;
+    }
+
+    memcpy(output, sps_buffer, sps_size);
+    *output_len = sps_size;
+
+    return 0;
+}
+
+int capture_and_encode(unsigned char *output, int *output_len) {
+    // Dequeue buffer
+    struct v4l2_plane planes[1];
+    memset(&v4l2_buf, 0, sizeof(v4l2_buf));
+    memset(planes, 0, sizeof(planes));
+    v4l2_buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE;
+    v4l2_buf.memory = V4L2_MEMORY_MMAP;
+    v4l2_buf.m.planes = planes;
+    v4l2_buf.length = NUM_PLANES;
+
+    if (ioctl(video_fd, VIDIOC_DQBUF, &v4l2_buf) < 0) {
+        perror("Failed to dequeue buffer");
+        return -1;
+    }
+
+    // Encode frame
+    MppFrame frame;
+    mpp_frame_init(&frame);
+    mpp_frame_set_width(frame, WIDTH);
+    mpp_frame_set_height(frame, HEIGHT);
+    mpp_frame_set_fmt(frame, MPP_FMT_BGR888);
+
+    MppBuffer frm_buf;
+    mpp_buffer_get(NULL, &frm_buf, planes[0].length);
+    memcpy(mpp_buffer_get_ptr(frm_buf), buffers[v4l2_buf.index], planes->bytesused);
+    mpp_frame_set_buffer(frame, frm_buf);
+
+    MPP_RET ret = mpp_api->encode_put_frame(mpp_ctx, frame);
+    if (ret == MPP_OK) {
+        MppPacket packet;
+        ret = mpp_api->encode_get_packet(mpp_ctx, &packet);
+        if (ret == MPP_OK) {
+            *output_len = mpp_packet_get_length(packet);
+            memcpy(output, mpp_packet_get_data(packet), *output_len);
+            mpp_packet_deinit(&packet);
+        }
+    }
+
+    mpp_frame_deinit(&frame);
+    mpp_buffer_put(frm_buf);
+
+    // Requeue buffer
+    ioctl(video_fd, VIDIOC_QBUF, &v4l2_buf);
+    return ret == MPP_OK ? 0 : -1;
+}
+
+void cleanup() {
+    //if (cfg) mpp_enc_cfg_deinit(cfg);
+    if (mpp_ctx) mpp_destroy(mpp_ctx);
+    if (video_fd >= 0) close(video_fd);
+}
--- a/external/mpp/mpp.go
+++ b/external/mpp/mpp.go
@@ -0,0 +1,82 @@
+package mpp
+
+/*
+#cgo CFLAGS: -I/usr/include/rockchip/mpp
+#cgo LDFLAGS: -L/usr/lib -lv4l2 -lrockchip_mpp
+#include "mpp.h"
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <linux/videodev2.h>
+#include <rockchip/mpp_buffer.h>
+#include <rockchip/mpp_err.h>
+#include <rockchip/mpp_frame.h>
+#include <rockchip/mpp_packet.h>
+#include <rockchip/rk_mpi.h>
+*/
+import "C"
+import (
+	"errors"
+	"fmt"
+	"unsafe"
+)
+
+var mpp *MPP
+
+type MPP struct {
+}
+
+func New(device string, fps int, bitrate int) (*MPP, error) {
+	cDevice := C.CString(device)
+	defer C.free(unsafe.Pointer(cDevice))
+
+	if C.video_init(cDevice) != 0 {
+		return nil, errors.New("failed to initialize video capture")
+	}
+	if C.mpp_init_wrapper(C.int(fps), C.int(bitrate)) != 0 {
+		return nil, errors.New("failed to initialize MPP")
+	}
+	return &MPP{}, nil
+}
+
+func GetSPS() ([]byte, error) {
+	output := make([]byte, 64) // Allocate buffer for SPS
+	var outputLen C.int
+
+	ret := C.get_sps((*C.uchar)(unsafe.Pointer(&output[0])), &outputLen)
+	if ret != 0 {
+		return nil, fmt.Errorf("failed to retrieve SPS: %v", ret)
+	}
+
+	return output[:outputLen], nil
+}
+
+func (m *MPP) CaptureAndEncode() ([]byte, error) {
+	output := make([]byte, 1024*1024) // 1MB buffer
+	var outputLen C.int
+
+	ret := C.capture_and_encode((*C.uchar)(unsafe.Pointer(&output[0])), &outputLen)
+	if ret != 0 {
+		return nil, errors.New("failed to capture and encode frame")
+	}
+
+	return output[:outputLen], nil
+}
+
+func (m *MPP) Close() {
+	C.cleanup()
+}
+
+func NewTest(device string) error {
+	var err error
+	mpp, err = New(device, 60, 10000*1000)
+	return err
+}
+
+func GetInstance() *MPP {
+	return mpp
+}
--- a/external/mpp/mpp.h
+++ b/external/mpp/mpp.h
@@ -0,0 +1,10 @@
+#ifndef MPP_H
+#define MPP_H
+
+int video_init(const char *device);
+int mpp_init_wrapper(int fps, int bitrate);
+int get_sps(unsigned char *output, int *output_len);
+int capture_and_encode(unsigned char *output, int *output_len);
+void cleanup();
+
+#endif
--- a/external/process/extprocess.go
+++ b/external/process/extprocess.go
@@ -1,4 +1,4 @@
-package stream
+package process

 import (
 	"io"
@@ -17,7 +17,6 @@ type ExtProcess struct {
 	running  bool
 	stopChan chan struct{}
 	finished chan struct{}
-	state    State

 	stdin io.WriteCloser
 }
--- a/external/process/pipedprocess.go
+++ b/external/process/pipedprocess.go
@@ -0,0 +1,136 @@
+package process
+
+import (
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// PipedCmd struct manages a sequence of piped commands.
+type PipedCmd struct {
+	cmds    []*exec.Cmd
+	mu      sync.Mutex
+	running bool
+}
+
+// InitPipedCmd initializes a PipedCmd instance with a sequence of commands.
+func InitPipedCmd(cmd string) *PipedCmd {
+	pipedCmd := &PipedCmd{}
+	pipedCmd.ChangeCmd(cmd)
+	return pipedCmd
+}
+
+func (p *PipedCmd) ChangeCmd(cmd string) {
+	cmds := strings.Split(cmd, "|")
+	for i, c := range cmds {
+		cmds[i] = strings.TrimSpace(c)
+	}
+
+	log.Debugf("Cmds: %+v", cmds)
+
+	pipedCmds := make([]*exec.Cmd, len(cmds))
+
+	// Initialize each command in the sequence
+	for i, cmd := range cmds {
+		if len(cmd) == 0 {
+			continue
+		}
+		cmdArgs := strings.Split(cmd, " ")
+		pipedCmds[i] = exec.Command(cmdArgs[0], cmdArgs[1:]...)
+	}
+
+	p.cmds = pipedCmds
+}
+
+// Start begins execution of all commands in the piped sequence.
+func (p *PipedCmd) Start() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.running {
+		log.Debugf("Process is already running.")
+		return nil
+	}
+
+	// Pipe each command's output to the next command's input
+	for i := 0; i < len(p.cmds)-1; i++ {
+		stdout, err := p.cmds[i].StdoutPipe()
+		if err != nil {
+			return err
+		}
+		p.cmds[i+1].Stdin = stdout
+	}
+	// pipe stdout and stderr of the last command into os
+	p.cmds[len(p.cmds)-1].Stdout = os.Stdout
+	p.cmds[len(p.cmds)-1].Stderr = os.Stderr
+
+	// Start each command in the sequence
+	for _, cmd := range p.cmds {
+		if err := cmd.Start(); err != nil {
+			p.terminateAll()
+			return err
+		} else {
+			log.Debugf("Started process: (%d) %v", cmd.Process.Pid, cmd.Args)
+		}
+	}
+
+	p.running = true
+
+	// Monitor commands in a goroutine to handle failures
+	go p.monitorCommands()
+
+	return nil
+}
+
+// monitorCommands waits for each command to finish and checks for errors.
+func (p *PipedCmd) monitorCommands() {
+	var wg sync.WaitGroup
+	wg.Add(len(p.cmds))
+
+	for _, cmd := range p.cmds {
+		go func(cmd *exec.Cmd) {
+			defer wg.Done()
+			err := cmd.Wait()
+			if err != nil && err.Error() != "signal: terminated" {
+				log.Debugf("Command failed: %v", err)
+				p.terminateAll() // Terminate all if any command fails
+			}
+		}(cmd)
+	}
+
+	// Wait for all commands to complete or terminate
+	wg.Wait()
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.running = false
+}
+
+// terminateAll sends a termination signal to all running commands.
+func (p *PipedCmd) terminateAll() {
+	for _, cmd := range p.cmds {
+		if cmd.Process != nil {
+			log.Debugf("Sending SIGTERM to: (%d) %v", cmd.Process.Pid, cmd.Args)
+			_ = cmd.Process.Signal(syscall.SIGTERM) // Send SIGTERM to allow graceful termination
+		}
+	}
+
+	p.running = false
+}
+
+// Stop manually stops all commands in the sequence.
+func (p *PipedCmd) Stop() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if !p.running {
+		log.Debug("Process is not running.")
+		return
+	}
+
+	log.Debug("Stopping process...")
+	p.terminateAll()
+}
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/pion/webrtc/v4 v4.0.1
 	github.com/sirupsen/logrus v1.9.3
+	gopkg.in/yaml.v3 v3.0.1
 )

 require (
@@ -54,5 +55,4 @@ require (
 	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/text v0.19.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
--- a/http/hw/hid/hid.go
+++ b/http/hw/hid/hid.go
--- a/http/hw/hid/keyboard.go
+++ b/http/hw/hid/keyboard.go
--- a/http/hw/hid/mouse.go
+++ b/http/hw/hid/mouse.go
--- a/http/hw/hid/op.go
+++ b/http/hw/hid/op.go
--- a/http/hw/stream/audio.go
+++ b/http/hw/stream/audio.go
@@ -1,47 +0,0 @@
-package stream
-
-import (
-	"io"
-	"net/http"
-	"os/exec"
-
-	"github.com/gin-gonic/gin"
-	log "github.com/sirupsen/logrus"
-)
-
-type Audio struct {
-}
-
-func AudioHandler(c *gin.Context) {
-	cmd := exec.Command("ffmpeg",
-		"-f", "alsa",
-		"-i", "hw:0,0",
-		"-acodec", "aac",
-		"-f", "mp4", "-")
-
-	//c := "arecord -D hw:0,0 -f cd -r 44100 -c 2 | /app/ffmpeg -re -f wav -i pipe:0 -c:a aac -b:a 128k -ar 44100 -ac 2 -f rtp rtp://127.0.0.1:5006?pkt_size=1200"
-	//cmd := exec.Command("sh", "-c", c)
-	// ffmpeg -f alsa -i hw:0,0 -acodec aac -f mp4 -
-	stdout, err := cmd.StdoutPipe()
-	if err != nil {
-		c.String(http.StatusInternalServerError, "Failed to capture audio: %v", err)
-		log.Errorf("Failed to capture audio: %v", err)
-		return
-	}
-	if err := cmd.Start(); err != nil {
-		c.String(http.StatusInternalServerError, "Failed to start ffmpeg: %v", err)
-		log.Errorf("Failed to start ffmpeg: %v", err)
-		return
-	}
-	defer cmd.Wait()
-
-	//c.Header("Content-Type", "audio/aac")
-	c.Header("Content-Type", "audio/wav")
-	c.Header("Transfer-Encoding", "chunked")
-
-	if _, err := io.Copy(c.Writer, stdout); err != nil {
-		c.String(http.StatusInternalServerError, "Failed to stream audio: %v", err)
-		log.Errorf("Failed to stream audio: %v", err)
-		return
-	}
-}
--- a/http/hw/stream/pipedprocess.go
+++ b/http/hw/stream/pipedprocess.go
@@ -1,269 +0,0 @@
-package stream
-
-import (
-	"log"
-	"os/exec"
-	"strings"
-	"sync"
-	"syscall"
-)
-
-var pipedCmd *PipedCmd
-
-// PipedCmd struct manages a sequence of piped commands.
-type PipedCmd struct {
-	cmds     []*exec.Cmd
-	mu       sync.Mutex
-	running  bool
-	finished chan struct{}
-}
-
-// InitPipedCmd initializes a PipedCmd instance with a sequence of commands.
-func InitPipedCmd(cmds []string) *PipedCmd {
-	pipedCmds := make([]*exec.Cmd, len(cmds))
-
-	// Initialize each command in the sequence
-	for i, cmd := range cmds {
-		if len(cmd) == 0 {
-			continue
-		}
-		cmdArgs := strings.Split(cmd, " ")
-		pipedCmds[i] = exec.Command(cmdArgs[0], cmdArgs[1:]...)
-	}
-
-	pipedCmd = &PipedCmd{
-		cmds:     pipedCmds,
-		finished: make(chan struct{}),
-	}
-	return pipedCmd
-}
-
-// Start begins execution of all commands in the piped sequence.
-func (p *PipedCmd) Start() error {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	if p.running {
-		log.Println("Process is already running.")
-		return nil
-	}
-
-	// Pipe each command's output to the next command's input
-	for i := 0; i < len(p.cmds)-1; i++ {
-		stdout, err := p.cmds[i].StdoutPipe()
-		if err != nil {
-			return err
-		}
-		p.cmds[i+1].Stdin = stdout
-	}
-
-	// Start each command in the sequence
-	for _, cmd := range p.cmds {
-		if err := cmd.Start(); err != nil {
-			p.terminateAll() // Clean up if any command fails to start
-			return err
-		}
-	}
-
-	p.running = true
-
-	// Monitor commands in a goroutine to handle failures
-	go p.monitorCommands()
-
-	return nil
-}
-
-// monitorCommands waits for each command to finish and checks for errors.
-func (p *PipedCmd) monitorCommands() {
-	var wg sync.WaitGroup
-	wg.Add(len(p.cmds))
-
-	for _, cmd := range p.cmds {
-		go func(cmd *exec.Cmd) {
-			defer wg.Done()
-			err := cmd.Wait()
-			if err != nil {
-				log.Printf("Command failed: %v", err)
-				p.terminateAll() // Terminate all if any command fails
-			}
-		}(cmd)
-	}
-
-	// Wait for all commands to complete or terminate
-	wg.Wait()
-	p.mu.Lock()
-	p.running = false
-	close(p.finished)
-	p.mu.Unlock()
-}
-
-// terminateAll sends a termination signal to all running commands.
-func (p *PipedCmd) terminateAll() {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	for _, cmd := range p.cmds {
-		if cmd.Process != nil {
-			_ = cmd.Process.Signal(syscall.SIGTERM) // Send SIGTERM to allow graceful termination
-		}
-	}
-}
-
-// Stop manually stops all commands in the sequence.
-func (p *PipedCmd) Stop() {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	if !p.running {
-		log.Println("Process is not running.")
-		return
-	}
-
-	log.Println("Stopping process...")
-	p.terminateAll()
-	p.running = false
-	close(p.finished)
-}
-
-/*
-import (
-	"os"
-	"os/exec"
-	"strings"
-	"sync"
-
-	log "github.com/sirupsen/logrus"
-)
-
-type PipedCmd struct {
-	cmds     []string
-	cmdsExec []*exec.Cmd
-	mu       sync.Mutex
-	running  bool
-	stopChan chan struct{}
-	finished chan struct{}
-}
-
-// Init initializes the PipedCmd with a slice of command strings
-func InitPipedCmd(cmds []string) *PipedCmd {
-	return &PipedCmd{
-		cmds: cmds,
-	}
-}
-
-// Start initializes and starts the piped commands
-func (p *PipedCmd) Start() {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	if p.running {
-		log.Debug("process is already running.")
-		return
-	}
-
-	p.stopChan = make(chan struct{})
-	p.finished = make(chan struct{})
-
-	log.Debugf("Starting piped commands: <%s>", strings.Join(p.cmds, " | "))
-
-	// Create commands and set up pipes
-	for i, cmdStr := range p.cmds {
-		// Split command string into command and arguments
-		cmdParts := strings.Fields(cmdStr)
-		if len(cmdParts) == 0 {
-			log.Errorf("Empty command string at index %d", i)
-			continue
-		}
-
-		cmd := exec.Command(cmdParts[0], cmdParts[1:]...)
-
-		// Set up pipes for stdin/stdout
-		if i > 0 {
-			stdin, err := p.cmdsExec[i-1].StdoutPipe()
-			if err != nil {
-				log.Errorf("Couldn't set up stdout pipe for command %s: %v", p.cmdsExec[i-1].Path, err)
-				return
-			}
-			cmd.Stdin = stdin
-		}
-
-		cmd.Stderr = os.Stderr // Log stderr to standard error output
-
-		p.cmdsExec = append(p.cmdsExec, cmd)
-	}
-
-	// Start the first command
-	if err := p.cmdsExec[0].Start(); err != nil {
-		log.Errorf("Failed to start command: %v", err)
-		return
-	}
-	p.running = true
-
-	// Start remaining commands
-	for _, cmd := range p.cmdsExec[1:] {
-		if err := cmd.Start(); err != nil {
-			log.Errorf("Failed to start command: %v", err)
-			return
-		}
-	}
-
-	go func() {
-		// Wait for the last command to finish
-		err := p.cmdsExec[len(p.cmdsExec)-1].Wait()
-		p.running = false
-		log.Errorf("process exited with error: %v", err)
-
-		// Signal that the process has finished
-		close(p.finished)
-		close(p.stopChan)
-	}()
-}
-
-// Stop terminates the piped commands gracefully
-func (p *PipedCmd) Stop() {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	if !p.running {
-		log.Debug("process is not running.")
-		return
-	}
-
-	log.Warn("Stopping process...")
-
-	for _, cmd := range p.cmdsExec {
-		if err := cmd.Process.Kill(); err != nil {
-			log.Errorf("Failed to kill process: %v", err)
-		}
-	}
-
-	select {
-	case <-p.finished:
-		log.Info("stopped as expected")
-	case <-p.stopChan:
-		log.Info("was killed")
-	}
-
-	p.running = false
-}
-
-// ChangeArgs updates the command arguments
-func (p *PipedCmd) ChangeArgs(newCmds []string) {
-	p.mu.Lock()
-	defer p.mu.Unlock()
-
-	p.cmds = newCmds
-	log.Printf("Updated process commands: %v", p.cmds)
-}
-
-// Watch monitors the process and restarts if it stops unexpectedly
-func (p *PipedCmd) Watch() {
-	for {
-		select {
-		case <-p.stopChan:
-			log.Errorf("process stopped unexpectedly. Restarting...")
-			p.Start()
-		}
-	}
-}
-*/
--- a/http/route/api.go
+++ b/http/route/api.go
@@ -6,11 +6,11 @@ import (
 	"os/exec"
 	"path/filepath"
 	"rkkvm/config"
-	"rkkvm/http/hw/hid"
-	"rkkvm/http/hw/stream"
+	"rkkvm/hid"
 	"rkkvm/http/middleware"
 	"rkkvm/http/reqrsp"
 	"rkkvm/http/ws"
+	"rkkvm/stream"
 	"strings"

 	"github.com/gin-gonic/gin"
@@ -21,7 +21,6 @@ func Api(e *gin.Engine) {
 	api := e.Group("/api").Use(middleware.CheckToken())

 	api.GET("/stream/mjpeg", stream.MjpegHandler)
-	api.GET("/stream/audio", stream.AudioHandler)
 	api.GET("/ws", ws.ConnHandler)

 	api.POST("/stream/webrtc", stream.WebRTCPeerConnect)
--- a/http/route/nanokvm_ui.go
+++ b/http/route/nanokvm_ui.go
@@ -2,7 +2,7 @@ package route

 import (
 	"net/http"
-	"rkkvm/http/hw/stream"
+	"rkkvm/external/ffmpeg"
 	"rkkvm/http/middleware"
 	"rkkvm/http/reqrsp"

@@ -55,7 +55,7 @@ func SetScreen(c *gin.Context) {
 		return
 	}

-	ffmpeg := stream.GetFFmpeg()
+	ffmpeg := ffmpeg.GetFFmpeg()
 	switch req.Type {
 	case "fps":
 		ffmpeg.SetFPS(req.Value)
@@ -70,9 +70,14 @@ func SetScreen(c *gin.Context) {
 		})
 		return
 	}
-	ffmpeg.Stop()
-	ffmpeg.Start()
-
+	/*
+		log.Debug("Stopping ffmpeg SetScreen")
+		ffmpeg.Stop()
+		time.Sleep(100 * time.Millisecond)
+		ffmpeg.ApplyOptions()
+		log.Debug("Starting ffmpeg SetScreen")
+		ffmpeg.Start()
+	*/
 	log.Debugf("update screen: %+v", req)
 	c.JSON(http.StatusOK, reqrsp.NanoKVMRsp{
 		Msg: reqrsp.MsgSuccess,
--- a/http/hw/rtc/listener.go
+++ b/http/hw/rtc/listener.go
@@ -26,11 +26,6 @@ func initUDPListener(host string, port int) (*net.UDPConn, error) {
 }

 func InitListener(host string, port int, aPort int) (*RTC, error) {
-	vl, err := initUDPListener(host, port)
-	if err != nil {
-		return nil, err
-	}
-
 	al, err := initUDPListener(host, aPort)
 	if err != nil {
 		return nil, err
@@ -40,16 +35,13 @@ func InitListener(host string, port int, aPort int) (*RTC, error) {
 	switch config.Get().Video.Codec {
 	case config.StreamSourceH264:
 		mimeType = webrtc.MimeTypeH264
-	case config.StreamSourceHevc: // WebRTC currently has no official support for H265
-		mimeType = webrtc.MimeTypeH265
 	default:
 		return nil, ErrWebRTCParam("unknown video codec: %s", config.Get().Video.Codec)
 	}

-	video, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
+	video, _ := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
 	audio, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "rkkvm")
 	rtc = &RTC{
-		videoListener: vl,
 		audioListener: al,
 		peers:         make(map[string]*webrtc.PeerConnection),
 		videoTrack:    video,
--- a/http/hw/rtc/webrtc.go
+++ b/http/hw/rtc/webrtc.go
@@ -4,12 +4,15 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"rkkvm/external/mpp"
 	"sync"
+	"time"

 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"

 	"github.com/pion/webrtc/v4"
+	"github.com/pion/webrtc/v4/pkg/media"
 )

 var rtc *RTC
@@ -26,9 +29,8 @@ var ErrPeerClosedConn = ErrWebRTCParam("peer closed conn")

 type RTC struct {
 	peers         map[string]*webrtc.PeerConnection
-	videoListener *net.UDPConn
 	audioListener *net.UDPConn
-	videoTrack    *webrtc.TrackLocalStaticRTP
+	videoTrack    *webrtc.TrackLocalStaticSample
 	audioTrack    *webrtc.TrackLocalStaticRTP
 	m             sync.Mutex
 }
@@ -37,6 +39,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 	peerID := uuid.New().String()
 	r.m.Lock()
 	r.peers[peerID] = p
+	/*if len(r.peers) == 1 {
+		ffmpeg.GetFFmpeg().Start()
+		log.Info("FFmpeg process started")
+	}*/
 	r.m.Unlock()

 	p.OnConnectionStateChange(func(connState webrtc.PeerConnectionState) {
@@ -44,6 +50,10 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 			r.m.Lock()
 			defer r.m.Unlock()
 			delete(r.peers, peerID)
+			/*if len(r.peers) == 0 {
+				ffmpeg.GetFFmpeg().Stop()
+				log.Info("No clients anymore, stop ffmpeg process")
+			}*/
 			p.Close()

 			peers := make([]string, 0, len(r.peers))
@@ -84,7 +94,42 @@ func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription)
 }

 func (r *RTC) VideoListenerRead() {
-	listenerRead(r.videoListener, r.videoTrack)
+	duration := time.Second / time.Duration(60)
+	ticker := time.NewTicker(duration)
+	defer ticker.Stop()
+
+	// Retrieve SPS and PPS once at the start
+	sps, err := mpp.GetSPS()
+	if err != nil {
+		log.Fatalf("Failed to retrieve SPS: %v", err)
+	}
+
+	firstFrame := true
+	for {
+		select {
+		case <-ticker.C:
+			frame, err := mpp.GetInstance().CaptureAndEncode()
+			if err != nil {
+				log.Errorf("failed to capture frame: %v", err)
+				continue
+			}
+
+			// If this is the first frame or an IDR frame, prepend SPS and PPS
+			if firstFrame || isIDRFrame(frame) {
+				firstFrame = false
+				frame = append(sps, frame...)
+			}
+
+			sample := media.Sample{
+				Data:     frame,
+				Duration: duration,
+			}
+			err = r.videoTrack.WriteSample(sample)
+			if err != nil {
+				log.Errorf("failed to write sample: %v", err)
+			}
+		}
+	}
 }

 func (r *RTC) AudioListenerRead() {
@@ -92,7 +137,6 @@ func (r *RTC) AudioListenerRead() {
 }

 func (r *RTC) Close() error {
-	r.videoListener.Close()
 	r.audioListener.Close()

 	return nil
@@ -137,3 +181,16 @@ func processRTCP(rtpSender *webrtc.RTPSender) {
 		}
 	}()
 }
+
+func isIDRFrame(frame []byte) bool {
+	// Check for NAL unit type 5 (IDR)
+	for i := 0; i < len(frame)-4; i++ {
+		if frame[i] == 0x00 && frame[i+1] == 0x00 && frame[i+2] == 0x00 && frame[i+3] == 0x01 {
+			nalType := frame[i+4] & 0x1F
+			if nalType == 5 { // IDR frame
+				return true
+			}
+		}
+	}
+	return false
+}
--- a/http/ws/ws.go
+++ b/http/ws/ws.go
@@ -5,7 +5,7 @@ import (
 	"net"
 	"net/http"
 	"rkkvm/config"
-	"rkkvm/http/hw/hid"
+	"rkkvm/hid"

 	"github.com/gin-gonic/gin"
 	"github.com/gorilla/websocket"
--- a/http/hw/stream/ustreamer.go
+++ b/http/hw/stream/ustreamer.go
@@ -2,6 +2,7 @@ package stream

 import (
 	"rkkvm/config"
+	"rkkvm/external/process"
 	"time"

 	log "github.com/sirupsen/logrus"
@@ -10,13 +11,14 @@ import (
 var ustreamer *UStreamer

 type UStreamer struct {
-	*ExtProcess
+	*process.ExtProcess
 	config.UStreamer
+	state State
 }

 func InitUStreamer(path string, args []string) *UStreamer {
 	ustreamer = &UStreamer{
-		ExtProcess: Init(path, args),
+		ExtProcess: process.Init(path, args),
 		UStreamer:  config.Get().UStreamer,
 	}
 	return ustreamer
--- a/http/hw/stream/video.go
+++ b/http/hw/stream/video.go
@@ -6,8 +6,9 @@ import (
 	"io"
 	"net/http"
 	"rkkvm/config"
-	"rkkvm/http/hw/rtc"
+	"rkkvm/external/ffmpeg"
 	"rkkvm/http/reqrsp"
+	"rkkvm/http/rtc"
 	"strconv"

 	"github.com/gin-gonic/gin"
@@ -172,7 +173,7 @@ func WebRTCSettings(c *gin.Context) {
 		return
 	}

-	ffmpeg := GetFFmpeg()
+	ffmpeg := ffmpeg.GetFFmpeg()

 	if len(bitrateStr) > 0 {
 		bitrate, err := strconv.Atoi(bitrateStr)
@@ -192,6 +193,6 @@ func WebRTCSettings(c *gin.Context) {
 		ffmpeg.SetFPS(fps)
 	}

-	ffmpeg.Stop()
-	ffmpeg.Start()
+	//ffmpeg.Stop()
+	//ffmpeg.Start()
 }
Author	SHA1	Message	Date
Artem	fc2273d40b	hw: add draft rockchip-mpp hw video encoding	2024-11-19 22:08:12 +01:00
Artem	c5f77df6b0	webrtc: start/stop ffmpeg procees if webrtc has clients	2024-11-05 23:49:26 +01:00
Artem	c228921237	project structure refactoring	2024-11-05 23:47:00 +01:00
Artem	1f5a56cc5d	ffmpeg: use one command to stream video/audio	2024-11-05 18:49:21 +01:00