diff --git a/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs b/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs index 3e67cee..7163518 100644 --- a/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs +++ b/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs @@ -228,14 +228,16 @@ namespace JianGongYun.TRTC.Components } } + //public event Action OnRenderVideoFrameHandler; /// - /// 渲染回调int width, int height, int rotation,WriteableBitmap image + /// 渲染回调byte[] data,int width, int height /// - public event Action OnRenderVideoFrameHandler; + public event Action OnRenderVideoFrameHandler; public event Action OnViewRemove; private void RenderFillMode(DrawingContext dc, byte[] data, int width, int height, int rotation) { + OnRenderVideoFrameHandler?.Invoke(data, width, height); int viewWidth = (int)this.ActualWidth, viewHeight = (int)this.ActualHeight; PixelFormat pixelFormat = PixelFormats.Pbgra32; int bytesPerPixel = (pixelFormat.BitsPerPixel + 7) / 8; @@ -250,17 +252,6 @@ namespace JianGongYun.TRTC.Components mWriteableBitmap.AddDirtyRect(mInt32Rect); mWriteableBitmap.Unlock(); - //var tt = ConvertWriteableBitmapToBitmapImage(mWriteableBitmap); - //OpenCvSharp.Mat result = OpenCvSharp.Extensions.BitmapConverter.ToMat(tt); - //tt.Dispose(); - //result.Create(mWriteableBitmap.PixelHeight, mWriteableBitmap.PixelWidth, OpenCvSharp.MatType.CV_8UC4, 4); - //mWriteableBitmap.CopyPixels(Int32Rect.Empty, result.Data, data.Length, 4); - //LiveClassroom.bbb.Enqueue(result); - //result.ImWrite($"d:\\{DateTime.Now.ToString("yyyyMMddHHmmssfff")}.png"); - //var a = ConvertWriteableBitmapToBitmapImage(mWriteableBitmap); - //a.Save($"d:\\{DateTime.Now.ToString("yyyyMMddHHmmssfff")}.png"); - OnRenderVideoFrameHandler?.Invoke(mWriteableBitmap); - ImageBrush brush = new ImageBrush(mWriteableBitmap); if (rotation > 0) { @@ -280,6 +271,7 @@ namespace JianGongYun.TRTC.Components private void RenderFitMode(DrawingContext dc, byte[] data, int width, int height, int rotation) { + OnRenderVideoFrameHandler?.Invoke(data, width, height); int viewWidth = (int)this.ActualWidth, viewHeight = (int)this.ActualHeight; PixelFormat pixelFormat = PixelFormats.Pbgra32; int bytesPerPixel = (pixelFormat.BitsPerPixel + 7) / 8; @@ -287,9 +279,6 @@ namespace JianGongYun.TRTC.Components if (mWriteableBitmap == null || mWriteableBitmap.PixelWidth != width || mWriteableBitmap.PixelHeight != height) { mWriteableBitmap = new WriteableBitmap(width, height, 96, 96, pixelFormat, null); - //Mat result = new Mat(); - //result.Create(mWriteableBitmap.PixelHeight, mWriteableBitmap.PixelWidth, MatType.CV_8U, 4); - //mWriteableBitmap.CopyPixels(Int32Rect.Empty, result.Data, (int)result.Step() * result.Rows, (int)result.Step()); mInt32Rect = new Int32Rect(0, 0, width, height); } mWriteableBitmap.Lock(); @@ -297,7 +286,7 @@ namespace JianGongYun.TRTC.Components mWriteableBitmap.AddDirtyRect(mInt32Rect); mWriteableBitmap.Unlock(); - OnRenderVideoFrameHandler?.Invoke(mWriteableBitmap); + //OnRenderVideoFrameHandler?.Invoke(mWriteableBitmap); ImageBrush brush = new ImageBrush(mWriteableBitmap); if (rotation > 0) diff --git a/JianGongYun/TRTC/LiveClassroom.cs b/JianGongYun/TRTC/LiveClassroom.cs index 0515ccc..0fbb45c 100644 --- a/JianGongYun/TRTC/LiveClassroom.cs +++ b/JianGongYun/TRTC/LiveClassroom.cs @@ -47,8 +47,18 @@ namespace JianGongYun.TRTC private static LiveWindowViewModel liveWinMode; public static ClassroomEntity CurrentClassroomEntity { get; private set; } public static TRTCCloudCallback TRTCCloudCallback = new TRTCCloudCallback(); - - //private static ConcurrentBag VedioRecords = new ConcurrentBag(); + /// + /// 摄像头帧 + /// + public static VideoFrameEntity MainFrame = new VideoFrameEntity(); + /// + /// 屏幕帧 + /// + public static VideoFrameEntity SubFrame = new VideoFrameEntity(); + /// + /// 背景帧 + /// + public static Mat BackgroundFrame = null; //public static int UserCount = 999; /// /// 进入直播教室 @@ -154,7 +164,21 @@ namespace JianGongYun.TRTC var view = AddCustomVideoView(parent, CurrentClassroomEntity.TeacherId, TRTCVideoStreamType.TRTCVideoStreamTypeBig, true); if (liveWinMode.IsLive) { - VedioRecordTask(view, TRTCVideoStreamType.TRTCVideoStreamTypeBig); + //Stopwatch sw = new Stopwatch(); + //VideoRecordTask(view, TRTCVideoStreamType.TRTCVideoStreamTypeBig); + view.OnRenderVideoFrameHandler += (data, w, h) => + { + //sw.Restart(); + lock (MainFrame) + { + MainFrame.Width = w; + MainFrame.Height = h; + MainFrame.Data = new byte[data.Length]; + Buffer.BlockCopy(data, 0, MainFrame.Data, 0, data.Length); + } + //sw.Stop(); + //Debug.Print("main" + sw.ElapsedMilliseconds.ToString()); + }; } return view; } @@ -191,7 +215,21 @@ namespace JianGongYun.TRTC var view = AddCustomVideoView(parent, CurrentClassroomEntity.TeacherId, TRTCVideoStreamType.TRTCVideoStreamTypeSub, true); if (liveWinMode.IsLive) { - VedioRecordTask(view, TRTCVideoStreamType.TRTCVideoStreamTypeSub); + //Stopwatch sw = new Stopwatch(); + //VideoRecordTask(view, TRTCVideoStreamType.TRTCVideoStreamTypeSub); + view.OnRenderVideoFrameHandler += (data, w, h) => + { + //sw.Restart(); + lock (SubFrame) + { + SubFrame.Width = w; + SubFrame.Height = h; + SubFrame.Data = new byte[data.Length]; + Buffer.BlockCopy(data, 0, SubFrame.Data, 0, data.Length); + } + //sw.Stop(); + //Debug.Print("sub" + sw.ElapsedMilliseconds.ToString()); + }; } return view; } @@ -270,77 +308,94 @@ namespace JianGongYun.TRTC } - public static void VedioRecordTask(TXLiteAVVideoView view, TRTCVideoStreamType streamType) + public static void VideoRecordTask(ref Action onEnd) { var end = false; - Stopwatch sw1 = new Stopwatch(); - Stopwatch sw2 = new Stopwatch(); - ConcurrentQueue bitmaps = new ConcurrentQueue(); - view.OnRenderVideoFrameHandler += (b) => - { - sw1.Restart(); - bitmaps.Enqueue(b.ToMat()); - sw1.Stop(); - //Debug.Print("a" + sw1.Elapsed.TotalMilliseconds.ToString()); - }; - view.OnViewRemove += () => + onEnd = () => { end = true; + BackgroundFrame?.Dispose(); + BackgroundFrame = null; }; - var task = Task.Factory.StartNew(() => - { - Console.WriteLine($"VedioRecordTask Start {streamType}"); - var _recoderDir = RecoderDir; - var imgTemp = Path.Combine(_recoderDir, $"{streamType}.png"); - var videoFile = Path.Combine(_recoderDir, $"{Util.TimeStr()}_{streamType}.avi"); - VideoWriter vw = new VideoWriter(); - bool videoWriterInit = false; - var frameCount = 0; - Timer timer = new Timer((a) => - { - //Console.WriteLine($"{streamType} fps {frameCount}"); - Debug.Print($"{streamType} fps {frameCount}"); - Interlocked.Exchange(ref frameCount, 0); - }, null, 0, 1000); - while (!end || bitmaps.Count > 0) - { - if (bitmaps.Count == 0) - { - Thread.Sleep(100); - continue; - } - if (bitmaps.TryDequeue(out var mat)) - { - sw2.Restart(); - //mat.SaveImage(imgTemp); - //mat.Dispose(); - //mat = Cv2.ImRead(imgTemp, ImreadModes.AnyColor); - //Cv2.ImShow(streamType.ToString(), mat); - //Cv2.WaitKey(1); - if (!videoWriterInit) - { - vw.Open(videoFile, FourCC.H264, settingWindowViewModel.LiveFps, mat.Size()); - videoWriterInit = true; - } - vw.Write(mat); - mat.Dispose(); - Interlocked.Increment(ref frameCount); - //Debug.Print("b"+sw2.Elapsed.TotalMilliseconds.ToString()); - } - else - { - Thread.Sleep(100); - } - } - vw?.Release(); - vw?.Dispose(); - if (File.Exists(imgTemp)) - { - File.Delete(imgTemp); - } - timer.Dispose(); - Console.WriteLine($"VedioRecordTask End {streamType}"); - }, TaskCreationOptions.LongRunning); + var resolution = settingWindowViewModel.MainEncParams.videoResolution.ToString().Split('_');//屏幕分辨率 + var fps = settingWindowViewModel.LiveFps; + BackgroundFrame = new Mat(int.Parse(resolution[1]), int.Parse(resolution[2]), MatType.CV_8UC3, Scalar.FromRgb(0x20, 0x20, 0x20)); + + //Stopwatch sw1 = new Stopwatch(); + //Stopwatch sw2 = new Stopwatch(); + //ConcurrentQueue bitmaps = new ConcurrentQueue(); + //view.OnRenderVideoFrameHandler += (b) => + //{ + // sw1.Restart(); + // bitmaps.Enqueue(b.ToMat()); + // sw1.Stop(); + // //Debug.Print("a" + sw1.Elapsed.TotalMilliseconds.ToString()); + //}; + ////view.OnRenderVideoFrameHandler1 += (a, b, c) => + ////{ + //// var bb = System.Runtime.InteropServices.Marshal.AllocHGlobal(System.Runtime.InteropServices.Marshal.SizeOf(a[0]) * a.Length); + //// System.Runtime.InteropServices.Marshal.Copy(a, 0, bb, a.Length); + //// var mat = new Mat(c, b, MatType.CV_32SC4, bb); + //// mat.SaveImage("1.bmp"); + ////}; + ////view.OnViewRemove += () => + ////{ + //// end = true; + ////}; + //var task = Task.Factory.StartNew(() => + //{ + // Console.WriteLine($"VideoRecordTask Start {streamType}"); + // var _recoderDir = RecoderDir; + // var imgTemp = Path.Combine(_recoderDir, $"{streamType}.png"); + // var videoFile = Path.Combine(_recoderDir, $"{Util.TimeStr()}_{streamType}.avi"); + // VideoWriter vw = new VideoWriter(); + // bool videoWriterInit = false; + // var frameCount = 0; + // Timer timer = new Timer((a) => + // { + // //Console.WriteLine($"{streamType} fps {frameCount}"); + // Debug.Print($"{streamType} fps {frameCount}"); + // Interlocked.Exchange(ref frameCount, 0); + // }, null, 0, 1000); + // while (!end || bitmaps.Count > 0) + // { + // if (bitmaps.Count == 0) + // { + // Thread.Sleep(100); + // continue; + // } + // if (bitmaps.TryDequeue(out var mat)) + // { + // sw2.Restart(); + // //mat.SaveImage(imgTemp); + // //mat.Dispose(); + // //mat = Cv2.ImRead(imgTemp, ImreadModes.AnyColor); + // //Cv2.ImShow(streamType.ToString(), mat); + // //Cv2.WaitKey(1); + // if (!videoWriterInit) + // { + // vw.Open(videoFile, FourCC.H264, settingWindowViewModel.LiveFps, mat.Size()); + // videoWriterInit = true; + // } + // vw.Write(mat); + // mat.Dispose(); + // Interlocked.Increment(ref frameCount); + // //Debug.Print("b"+sw2.Elapsed.TotalMilliseconds.ToString()); + // } + // else + // { + // Thread.Sleep(100); + // } + // } + // vw?.Release(); + // vw?.Dispose(); + // if (File.Exists(imgTemp)) + // { + // File.Delete(imgTemp); + // } + // timer.Dispose(); + // Console.WriteLine($"VideoRecordTask End {streamType}"); + //}, TaskCreationOptions.LongRunning); } /// diff --git a/JianGongYun/TRTC/Models/VideoFrameEntity.cs b/JianGongYun/TRTC/Models/VideoFrameEntity.cs new file mode 100644 index 0000000..8d031e2 --- /dev/null +++ b/JianGongYun/TRTC/Models/VideoFrameEntity.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace JianGongYun.TRTC.Models +{ + public class VideoFrameEntity + { + public byte[] Data { get; set; } = null; + public int Width { get; set; } = 0; + public int Height { get; set; } = 0; + + + } +} diff --git a/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs b/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs index 83eb003..ff498a6 100644 --- a/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs +++ b/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs @@ -119,6 +119,7 @@ namespace JianGongYun.TRTC.Windows { var btn = sender as AduFlatButton; var start = Convert.ToBoolean(btn.Tag); + Action onEnd = default; if (start)//开始直播 { if (SettingWindowViewModel.DiskSize <= ViewModels.SettingWindowViewModel.DiskWarningSize) @@ -157,12 +158,14 @@ namespace JianGongYun.TRTC.Windows } LiveClassroom.StartMic(); LiveClassroom.SetMicMute(false); + LiveClassroom.VideoRecordTask(ref onEnd);//启动录制 }, SettingWindowViewModel.ScreenRecordingCountdown); } else { LiveWindowViewModel.IsLive = start; + onEnd?.Invoke(); //停止直播 LiveClassroom.StopMic(); LiveClassroom.StopVideoMain(AfterLiveViewWrap);