add audio support, trying to manage MSD device

This commit is contained in:
Artem
2024-11-03 11:44:40 +01:00
parent 68ba48a3a2
commit 1b60f43df1
22 changed files with 1059 additions and 263 deletions

View File

@@ -87,10 +87,10 @@ func (h *Hid) writeWithTimeout(file *os.File, data []byte) {
if err != nil {
switch {
case errors.Is(err, os.ErrClosed):
log.Debugf("hid already closed, reopen it...")
log.Tracef("hid already closed, reopen it...")
h.OpenNoLock()
case errors.Is(err, os.ErrDeadlineExceeded):
log.Debugf("write to hid timeout")
log.Tracef("write to hid timeout")
default:
log.Errorf("write to hid failed: %s", err)
}

View File

@@ -8,6 +8,7 @@ import (
)
func (h *Hid) OpenNoLock() {
log.Debug("OpenNoLock hid")
var err error
h.CloseNoLock()
@@ -39,6 +40,7 @@ func (h *Hid) Open() {
}
func (h *Hid) CloseNoLock() {
log.Debug("CloseNoLock")
for _, file := range []*os.File{h.g0, h.g1, h.g2} {
if file != nil {
_ = file.Sync()

77
http/hw/rtc/listener.go Normal file
View File

@@ -0,0 +1,77 @@
package rtc
import (
"net"
"rkkvm/config"
"github.com/pion/webrtc/v4"
log "github.com/sirupsen/logrus"
)
func initUDPListener(host string, port int) (*net.UDPConn, error) {
l, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP(host), Port: port})
if err != nil {
return nil, ErrWebRTCParam("failed to init webrtc listener: %v", err)
}
// Increase the UDP receive buffer size
// Default UDP buffer sizes vary on different operating systems
bufferSize := 300000 // 300KB
err = l.SetReadBuffer(bufferSize)
if err != nil {
return nil, ErrWebRTCParam("failed to set read buffer: %v", err)
}
return l, nil
}
func InitListener(host string, port int, aPort int) (*RTC, error) {
vl, err := initUDPListener(host, port)
if err != nil {
return nil, err
}
al, err := initUDPListener(host, aPort)
if err != nil {
return nil, err
}
mimeType := ""
switch config.Get().Video.Codec {
case config.StreamSourceH264:
mimeType = webrtc.MimeTypeH264
case config.StreamSourceHevc: // WebRTC currently has no official support for H265
mimeType = webrtc.MimeTypeH265
default:
return nil, ErrWebRTCParam("unknown video codec: %s", config.Get().Video.Codec)
}
video, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: mimeType}, "video", "rkkvm")
audio, _ := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "rkkvm")
rtc = &RTC{
videoListener: vl,
audioListener: al,
peers: make(map[string]*webrtc.PeerConnection),
videoTrack: video,
audioTrack: audio,
}
return rtc, nil
}
func listenerRead(l *net.UDPConn, track *webrtc.TrackLocalStaticRTP) {
buf := make([]byte, 1600) // Buffer to hold incoming RTP packets
for {
n, _, err := l.ReadFrom(buf)
if err != nil {
log.Errorf("error reading from UDP: %v\n", err)
continue
}
_, err = track.Write(buf[:n])
if err != nil {
log.Errorf("failed to send RTP to peer: %v", err)
}
}
}

View File

@@ -1,184 +1,139 @@
package rtc
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"sync"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/pion/webrtc/v4"
)
// https://github.com/pion/example-webrtc-applications/blob/master/sfu-ws/main.go
var rtc *RTC
func Get() *RTC {
return rtc
}
var ErrPeerClosedConn = errors.New("webrtc: peer closed conn")
var ErrWebRTC = errors.New("webrtc")
var ErrWebRTCParam = func(format string, args ...any) error {
return fmt.Errorf("%w: "+format, args...)
}
var ErrPeerClosedConn = ErrWebRTCParam("peer closed conn")
type RTC struct {
l *net.UDPConn
peer *webrtc.PeerConnection
track *webrtc.TrackLocalStaticRTP
sender *webrtc.RTPSender
localSession string
peers map[string]*webrtc.PeerConnection
videoListener *net.UDPConn
audioListener *net.UDPConn
videoTrack *webrtc.TrackLocalStaticRTP
audioTrack *webrtc.TrackLocalStaticRTP
m sync.Mutex
}
func (r *RTC) AddPeer(p *webrtc.PeerConnection, offer webrtc.SessionDescription) (*webrtc.SessionDescription, error) {
peerID := uuid.New().String()
r.m.Lock()
r.peers[peerID] = p
r.m.Unlock()
p.OnConnectionStateChange(func(connState webrtc.PeerConnectionState) {
if connState == webrtc.PeerConnectionStateFailed || connState == webrtc.PeerConnectionStateClosed {
r.m.Lock()
defer r.m.Unlock()
delete(r.peers, peerID)
p.Close()
peers := make([]string, 0, len(r.peers))
for p := range r.peers {
peers = append(peers, p)
}
log.WithField("peers", peers).Infof("Peer %s disconnected and resources cleaned up.", peerID)
}
})
vSender, err := p.AddTrack(r.videoTrack)
if err != nil {
return nil, ErrWebRTCParam("failed to add video track: %v", err)
}
processRTCP(vSender)
aSender, err := p.AddTrack(r.audioTrack)
if err != nil {
return nil, ErrWebRTCParam("failed to add audio track: %v", err)
}
processRTCP(aSender)
if err := p.SetRemoteDescription(offer); err != nil {
return nil, ErrWebRTCParam("failed to set remote description: %v", err)
}
answer, err := p.CreateAnswer(nil)
if err != nil {
return nil, ErrWebRTCParam("failed to create answer: %v", err)
}
gatherComplete := webrtc.GatheringCompletePromise(p)
if err := p.SetLocalDescription(answer); err != nil {
return nil, ErrWebRTCParam("failed to set local description: %v", err)
}
<-gatherComplete
return p.LocalDescription(), nil
}
func (r *RTC) VideoListenerRead() {
listenerRead(r.videoListener, r.videoTrack)
}
func (r *RTC) AudioListenerRead() {
listenerRead(r.audioListener, r.audioTrack)
}
func (r *RTC) Close() error {
return r.l.Close()
r.videoListener.Close()
r.audioListener.Close()
return nil
}
// Read incoming RTCP packets
// Before these packets are returned they are processed by interceptors. For things
// like NACK this needs to be called.
func (r *RTC) Read() {
rtcpBuf := make([]byte, 1500)
for {
if _, _, rtcpErr := r.sender.Read(rtcpBuf); rtcpErr != nil {
log.Errorf("failed to read RTCP packet: %v", rtcpErr)
return
}
}
}
func Init(host string, port int) (*RTC, error) {
peerConnection, err := webrtc.NewPeerConnection(webrtc.Configuration{
func NewPeer() (*webrtc.PeerConnection, error) {
peer, err := webrtc.NewPeerConnection(webrtc.Configuration{
ICEServers: []webrtc.ICEServer{
{
URLs: []string{"stun:stun.l.google.com:19302"},
},
},
})
if err != nil {
return nil, fmt.Errorf("failed to create peer connection: %v", err)
if err == nil {
// Set the handler for ICE connection state
// This will notify you when the peer has connected/disconnected
peer.OnICEConnectionStateChange(func(connState webrtc.ICEConnectionState) {
log.Infof("Connection State has changed %s", connState.String())
if connState == webrtc.ICEConnectionStateFailed {
if closeErr := peer.Close(); closeErr != nil {
panic(closeErr)
}
}
})
}
// Open a UDP Listener for RTP Packets on port 5004
l, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP(host), Port: port})
if err != nil {
return nil, fmt.Errorf("failed to init webrtc listener: %v", err)
}
// Increase the UDP receive buffer size
// Default UDP buffer sizes vary on different operating systems
bufferSize := 300000 // 300KB
err = l.SetReadBuffer(bufferSize)
if err != nil {
return nil, fmt.Errorf("failed to set read buffer: %v", err)
}
track, err := webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH264}, "video", "pion")
if err != nil { // it should never happens
panic(fmt.Sprintf("failed to create video track: %v", err))
}
rtpSender, err := peerConnection.AddTrack(track)
if err != nil {
return nil, fmt.Errorf("failed to add track to peer connection: %v", err)
}
r := &RTC{
peer: peerConnection,
sender: rtpSender,
track: track,
l: l,
}
rtc = r
return r, nil
return peer, err
}
func (r *RTC) Handshake(clientSession string) (string, error) {
// Set the handler for ICE connection state
// This will notify you when the peer has connected/disconnected
r.peer.OnICEConnectionStateChange(func(connState webrtc.ICEConnectionState) {
log.Infof("Connection State has changed %s", connState.String())
// Read incoming RTCP packets
// Before these packets are retuned they are processed by interceptors. For things
// like NACK this needs to be called.
func processRTCP(rtpSender *webrtc.RTPSender) {
go func() {
rtcpBuf := make([]byte, 1500)
if connState == webrtc.ICEConnectionStateFailed {
if closeErr := r.peer.Close(); closeErr != nil {
panic(closeErr)
for {
if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
return
}
}
})
// Wait for the offer to be pasted
offer := webrtc.SessionDescription{}
decode(clientSession, &offer)
fmt.Printf("Offer: %+v\n", offer)
// Set the remote SessionDescription
if err := r.peer.SetRemoteDescription(offer); err != nil {
return "", fmt.Errorf("failed to set remote session description: %v", err)
}
// Create answer
answer, err := r.peer.CreateAnswer(nil)
if err != nil {
return "", fmt.Errorf("failed to create answer: %v", err)
}
// Create channel that is blocked until ICE Gathering is complete
gatherComplete := webrtc.GatheringCompletePromise(r.peer)
// Sets the LocalDescription, and starts our UDP listeners
if err = r.peer.SetLocalDescription(answer); err != nil {
return "", fmt.Errorf("failed to set local description: %v", err)
}
// Block until ICE Gathering is complete, disabling trickle ICE
// we do this because we only can exchange one signaling message
// in a production application you should exchange ICE Candidates via OnICECandidate
<-gatherComplete
r.localSession = encode(r.peer.LocalDescription())
return r.localSession, nil
}
func (r *RTC) Listen() error {
// Read RTP packets forever and send them to the WebRTC Client
inboundRTPPacket := make([]byte, 1600) // UDP MTU
for {
n, _, err := r.l.ReadFrom(inboundRTPPacket)
if err != nil {
return fmt.Errorf("error during read: %v", err)
}
if _, err = r.track.Write(inboundRTPPacket[:n]); err != nil {
if errors.Is(err, io.ErrClosedPipe) {
// The peerConnection has been closed.
return ErrPeerClosedConn
}
return fmt.Errorf("failed to send RTP packet to client: %v", err)
}
}
}
// JSON encode + base64 a SessionDescription
func encode(obj *webrtc.SessionDescription) string {
b, err := json.Marshal(obj)
if err != nil {
panic(err)
}
return base64.StdEncoding.EncodeToString(b)
}
// Decode a base64 and unmarshal JSON into a SessionDescription
func decode(in string, obj *webrtc.SessionDescription) {
b, err := base64.StdEncoding.DecodeString(in)
if err != nil {
panic(err)
}
if err = json.Unmarshal(b, obj); err != nil {
panic(err)
}
}()
}

View File

@@ -1,33 +1,47 @@
package stream
import (
"github.com/gin-gonic/gin"
"io"
"net/http"
"os/exec"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
)
type Audio struct {
}
func AudioHandler(c *gin.Context) {
cmd := exec.Command("ffmpeg",
"-f", "alsa",
"-i", "hw:0,0",
"-acodec", "aac",
"-f", "mp4", "-")
//c := "arecord -D hw:0,0 -f cd -r 44100 -c 2 | /app/ffmpeg -re -f wav -i pipe:0 -c:a aac -b:a 128k -ar 44100 -ac 2 -f rtp rtp://127.0.0.1:5006?pkt_size=1200"
//cmd := exec.Command("sh", "-c", c)
// ffmpeg -f alsa -i hw:0,0 -acodec aac -f mp4 -
stdout, err := cmd.StdoutPipe()
if err != nil {
c.String(http.StatusInternalServerError, "Failed to capture audio: %v", err)
log.Errorf("Failed to capture audio: %v", err)
return
}
if err := cmd.Start(); err != nil {
c.String(http.StatusInternalServerError, "Failed to start ffmpeg: %v", err)
log.Errorf("Failed to start ffmpeg: %v", err)
return
}
defer cmd.Wait()
c.Header("Content-Type", "audio/mp4")
//c.Header("Content-Type", "audio/aac")
c.Header("Content-Type", "audio/wav")
c.Header("Transfer-Encoding", "chunked")
if _, err := io.Copy(c.Writer, stdout); err != nil {
c.String(http.StatusInternalServerError, "Failed to stream audio: %v", err)
log.Errorf("Failed to stream audio: %v", err)
return
}
}

View File

@@ -4,6 +4,7 @@ import (
"io"
"os"
"os/exec"
"rkkvm/config"
"sync"
log "github.com/sirupsen/logrus"
@@ -25,7 +26,7 @@ type ExtProcess struct {
func Init(path string, args []string) *ExtProcess {
return &ExtProcess{
args: args,
path: path,
path: config.RootFS + path,
}
}
@@ -41,7 +42,7 @@ func (u *ExtProcess) Start() {
u.stopChan = make(chan struct{})
u.finished = make(chan struct{})
log.Debug("Starting external process...")
log.Debugf("Starting external process: %s %v", u.path, u.args)
u.cmd = exec.Command(u.path, u.args...)
u.cmd.Stdout = os.Stdout
u.cmd.Stderr = os.Stderr
@@ -99,8 +100,6 @@ func (u *ExtProcess) Stop() {
log.Errorf("Failed to close stdin: %v", err)
}
log.Info("waiting for finish")
select {
case <-u.finished:
log.Info("stopped as expected")

View File

@@ -13,6 +13,18 @@ import (
// vp8
//ffmpeg -re -i /dev/video0 -c:v libvpx -crf 10 -b:v 3M -deadline 1 -g 10 -error-resilient 1 -auto-alt-ref 1 -an -f rtp rtp://127.0.0.1:5004?pkt_size=1200
/*
arecord -D hw:0,0 -f cd -r 44100 -c 2 | /app/ffmpeg -re -init_hw_device rkmpp=hw -filter_hw_device hw \
-f wav -thread_queue_size 4096 -i pipe:0 -f v4l2 -i /dev/video0 \
-vf hwupload,scale_rkrga=h=720:force_original_aspect_ratio=1 \
-c:v h264_rkmpp -flags +low_delay -b:v 6000000 -framerate 60 -g 10 \
-map 0:a -c:a aac -b:a 128k -ar 44100 -ac 2 -f rtp rtp://127.0.0.1:5006?pkt_size=1200 \
-map 1:v -f rtp rtp://127.0.0.1:5004?pkt_size=1200
arecord -D hw:0,0 -f cd -r 44100 -c 2 | /app/ffmpeg -re -f wav -i pipe:0 -c:a aac -b:a 128k -ar 44100 -ac 2 -f rtp rtp://127.0.0.1:5006?pkt_size=1200
*/
// https://jsfiddle.net/z7ms3u5r/
var ffmpeg *FFmpeg
@@ -40,10 +52,28 @@ func (f *FFmpeg) SetFPS(fps int) {
f.ChangeArgs(f.FormatArgs())
}
func (f *FFmpeg) SetResolution(height int) {
f.Height = height
if f.Height <= 0 {
f.Height = 1080
}
f.ChangeArgs(f.FormatArgs())
}
func (f *FFmpeg) SetGOP(gop int) {
f.GOP = gop
if f.GOP < 0 {
f.GOP = 0
}
f.ChangeArgs(f.FormatArgs())
}
func InitFFmpeg(path string, args []string) *FFmpeg {
ffmpeg = &FFmpeg{
ExtProcess: Init(path, args),
FFmpeg: config.Get().FFmpeg,
FFmpeg: config.Get().Video,
}
return ffmpeg
}

View File

@@ -0,0 +1,270 @@
package stream
import (
"log"
"os/exec"
"rkkvm/config"
"strings"
"sync"
"syscall"
)
var pipedCmd *PipedCmd
// PipedCmd struct manages a sequence of piped commands.
type PipedCmd struct {
cmds []*exec.Cmd
mu sync.Mutex
running bool
finished chan struct{}
}
// InitPipedCmd initializes a PipedCmd instance with a sequence of commands.
func InitPipedCmd(cmds []string) *PipedCmd {
pipedCmds := make([]*exec.Cmd, len(cmds))
// Initialize each command in the sequence
for i, cmd := range cmds {
if len(cmd) == 0 {
continue
}
cmdArgs := strings.Split(cmd, " ")
pipedCmds[i] = exec.Command(config.RootFS+"/"+cmdArgs[0], cmdArgs[1:]...)
}
pipedCmd = &PipedCmd{
cmds: pipedCmds,
finished: make(chan struct{}),
}
return pipedCmd
}
// Start begins execution of all commands in the piped sequence.
func (p *PipedCmd) Start() error {
p.mu.Lock()
defer p.mu.Unlock()
if p.running {
log.Println("Process is already running.")
return nil
}
// Pipe each command's output to the next command's input
for i := 0; i < len(p.cmds)-1; i++ {
stdout, err := p.cmds[i].StdoutPipe()
if err != nil {
return err
}
p.cmds[i+1].Stdin = stdout
}
// Start each command in the sequence
for _, cmd := range p.cmds {
if err := cmd.Start(); err != nil {
p.terminateAll() // Clean up if any command fails to start
return err
}
}
p.running = true
// Monitor commands in a goroutine to handle failures
go p.monitorCommands()
return nil
}
// monitorCommands waits for each command to finish and checks for errors.
func (p *PipedCmd) monitorCommands() {
var wg sync.WaitGroup
wg.Add(len(p.cmds))
for _, cmd := range p.cmds {
go func(cmd *exec.Cmd) {
defer wg.Done()
err := cmd.Wait()
if err != nil {
log.Printf("Command failed: %v", err)
p.terminateAll() // Terminate all if any command fails
}
}(cmd)
}
// Wait for all commands to complete or terminate
wg.Wait()
p.mu.Lock()
p.running = false
close(p.finished)
p.mu.Unlock()
}
// terminateAll sends a termination signal to all running commands.
func (p *PipedCmd) terminateAll() {
p.mu.Lock()
defer p.mu.Unlock()
for _, cmd := range p.cmds {
if cmd.Process != nil {
_ = cmd.Process.Signal(syscall.SIGTERM) // Send SIGTERM to allow graceful termination
}
}
}
// Stop manually stops all commands in the sequence.
func (p *PipedCmd) Stop() {
p.mu.Lock()
defer p.mu.Unlock()
if !p.running {
log.Println("Process is not running.")
return
}
log.Println("Stopping process...")
p.terminateAll()
p.running = false
close(p.finished)
}
/*
import (
"os"
"os/exec"
"strings"
"sync"
log "github.com/sirupsen/logrus"
)
type PipedCmd struct {
cmds []string
cmdsExec []*exec.Cmd
mu sync.Mutex
running bool
stopChan chan struct{}
finished chan struct{}
}
// Init initializes the PipedCmd with a slice of command strings
func InitPipedCmd(cmds []string) *PipedCmd {
return &PipedCmd{
cmds: cmds,
}
}
// Start initializes and starts the piped commands
func (p *PipedCmd) Start() {
p.mu.Lock()
defer p.mu.Unlock()
if p.running {
log.Debug("process is already running.")
return
}
p.stopChan = make(chan struct{})
p.finished = make(chan struct{})
log.Debugf("Starting piped commands: <%s>", strings.Join(p.cmds, " | "))
// Create commands and set up pipes
for i, cmdStr := range p.cmds {
// Split command string into command and arguments
cmdParts := strings.Fields(cmdStr)
if len(cmdParts) == 0 {
log.Errorf("Empty command string at index %d", i)
continue
}
cmd := exec.Command(cmdParts[0], cmdParts[1:]...)
// Set up pipes for stdin/stdout
if i > 0 {
stdin, err := p.cmdsExec[i-1].StdoutPipe()
if err != nil {
log.Errorf("Couldn't set up stdout pipe for command %s: %v", p.cmdsExec[i-1].Path, err)
return
}
cmd.Stdin = stdin
}
cmd.Stderr = os.Stderr // Log stderr to standard error output
p.cmdsExec = append(p.cmdsExec, cmd)
}
// Start the first command
if err := p.cmdsExec[0].Start(); err != nil {
log.Errorf("Failed to start command: %v", err)
return
}
p.running = true
// Start remaining commands
for _, cmd := range p.cmdsExec[1:] {
if err := cmd.Start(); err != nil {
log.Errorf("Failed to start command: %v", err)
return
}
}
go func() {
// Wait for the last command to finish
err := p.cmdsExec[len(p.cmdsExec)-1].Wait()
p.running = false
log.Errorf("process exited with error: %v", err)
// Signal that the process has finished
close(p.finished)
close(p.stopChan)
}()
}
// Stop terminates the piped commands gracefully
func (p *PipedCmd) Stop() {
p.mu.Lock()
defer p.mu.Unlock()
if !p.running {
log.Debug("process is not running.")
return
}
log.Warn("Stopping process...")
for _, cmd := range p.cmdsExec {
if err := cmd.Process.Kill(); err != nil {
log.Errorf("Failed to kill process: %v", err)
}
}
select {
case <-p.finished:
log.Info("stopped as expected")
case <-p.stopChan:
log.Info("was killed")
}
p.running = false
}
// ChangeArgs updates the command arguments
func (p *PipedCmd) ChangeArgs(newCmds []string) {
p.mu.Lock()
defer p.mu.Unlock()
p.cmds = newCmds
log.Printf("Updated process commands: %v", p.cmds)
}
// Watch monitors the process and restarts if it stops unexpectedly
func (p *PipedCmd) Watch() {
for {
select {
case <-p.stopChan:
log.Errorf("process stopped unexpectedly. Restarting...")
p.Start()
}
}
}
*/

View File

@@ -7,9 +7,11 @@ import (
"net/http"
"rkkvm/config"
"rkkvm/http/hw/rtc"
"rkkvm/http/reqrsp"
"strconv"
"github.com/gin-gonic/gin"
"github.com/pion/webrtc/v4"
log "github.com/sirupsen/logrus"
)
@@ -136,23 +138,29 @@ func MjpegHandler(c *gin.Context) {
}
}
func WebRTCHandshake(c *gin.Context) {
str, err := c.GetRawData()
if err != nil {
c.String(http.StatusBadRequest, err.Error())
func WebRTCPeerConnect(c *gin.Context) {
var offer webrtc.SessionDescription
if err := c.ShouldBindJSON(&offer); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid offer"})
return
}
log.Debugf("Client session description: %s", string(str))
r := rtc.Get()
localSession, err := r.Handshake(string(str))
peer, err := rtc.NewPeer()
if err != nil {
c.String(http.StatusBadRequest, err.Error())
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create peer connection"})
return
}
c.String(http.StatusOK, localSession)
answer, err := rtc.Get().AddPeer(peer, offer)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to add peer"})
return
}
c.JSON(http.StatusOK, reqrsp.NanoKVMRsp{
Msg: reqrsp.MsgSuccess,
Data: answer,
})
}
func WebRTCSettings(c *gin.Context) {