diff --git a/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs b/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs index 7163518..c2d7000 100644 --- a/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs +++ b/JianGongYun/TRTC/Components/TXLiteAVVideoView.cs @@ -105,7 +105,7 @@ namespace JianGongYun.TRTC.Components mLocalView = false; mFirstFrame = false; mRenderMode = TRTCVideoFillMode.TRTCVideoFillMode_Fit; - this.OnViewRemove?.Invoke(); + //this.OnViewRemove?.Invoke(); this.InvalidateVisual(); } @@ -233,7 +233,7 @@ namespace JianGongYun.TRTC.Components /// 渲染回调byte[] data,int width, int height /// public event Action OnRenderVideoFrameHandler; - public event Action OnViewRemove; + //public event Action OnViewRemove; private void RenderFillMode(DrawingContext dc, byte[] data, int width, int height, int rotation) { diff --git a/JianGongYun/TRTC/LiveClassroom.cs b/JianGongYun/TRTC/LiveClassroom.cs index a37b75b..8416555 100644 --- a/JianGongYun/TRTC/LiveClassroom.cs +++ b/JianGongYun/TRTC/LiveClassroom.cs @@ -172,7 +172,10 @@ namespace JianGongYun.TRTC //sw.Restart(); lock (MainFrame) { - MainFrame.Create(h, w, MatType.CV_8UC4); + if (MainFrame.Cols != w || MainFrame.Rows != h) + { + MainFrame.Create(h, w, MatType.CV_8UC4); + } Marshal.Copy(data, 0, MainFrame.Data, data.Length); } //sw.Stop(); @@ -214,18 +217,21 @@ namespace JianGongYun.TRTC var view = AddCustomVideoView(parent, CurrentClassroomEntity.TeacherId, TRTCVideoStreamType.TRTCVideoStreamTypeSub, true); if (liveWinMode.IsLive) { - Stopwatch sw = new Stopwatch(); + //Stopwatch sw = new Stopwatch(); //VideoRecordTask(view, TRTCVideoStreamType.TRTCVideoStreamTypeSub); view.OnRenderVideoFrameHandler += (data, w, h) => { - sw.Restart(); + //sw.Restart(); lock (SubFrame) { - SubFrame.Create(h, w, MatType.CV_8UC4); + if (SubFrame.Cols != w || SubFrame.Rows != h) + { + SubFrame.Create(h, w, MatType.CV_8UC4); + } Marshal.Copy(data, 0, SubFrame.Data, data.Length); } - sw.Stop(); - Debug.Print("sub" + sw.ElapsedMilliseconds.ToString()); + //sw.Stop(); + //Debug.Print("sub" + sw.ElapsedMilliseconds.ToString()); }; } return view; @@ -314,9 +320,141 @@ namespace JianGongYun.TRTC BackgroundFrame?.Dispose(); BackgroundFrame = null; }; - var resolution = settingWindowViewModel.MainEncParams.videoResolution.ToString().Split('_');//屏幕分辨率 - var fps = settingWindowViewModel.LiveFps; - BackgroundFrame = new Mat(int.Parse(resolution[1]), int.Parse(resolution[2]), MatType.CV_8UC3, Scalar.FromRgb(0x20, 0x20, 0x20)); + var backColor = Scalar.FromRgb(0x20, 0x20, 0x20); + var resolution = settingWindowViewModel.SubEncParams.videoResolution.ToString().Split('_');//屏幕分辨率，本地储存分辨率以屏幕分享分辨率为基准 + var fps = settingWindowViewModel.LiveFps;//视频采集的fps + BackgroundFrame = new Mat(int.Parse(resolution[2]), int.Parse(resolution[1]), MatType.CV_8UC3, backColor);//合成双路视频的背景 + var delay = 1000 / (int)fps;//每帧时间 + var delayEqualize = 0;//每帧时间补偿，在性能和其他因素影响下delay的时间不一定充足 + + + + + Task.Factory.StartNew(() => + { + Stopwatch stopwatch = new Stopwatch();//计时器 + + bool onlyCameraInit = false;//只有摄像头的话背景填充为backColor，变量标记只需设置一次 + bool noImgInit = false;//没有画面背景也填充backColor，变量标记只需设置一次 + + var _recoderDir = RecoderDir; + var videoFile = Path.Combine(_recoderDir, $"{Util.TimeStr()}.avi"); + VideoWriter vw = new VideoWriter(videoFile, FourCC.H264, fps, BackgroundFrame.Size()); + + + var backHeight = BackgroundFrame.Rows;//画面高度 + var backWidth = BackgroundFrame.Cols;//画面宽度 + + //屏幕分享画面 + var screenRoi = BackgroundFrame[new OpenCvSharp.Rect(0, 0, backWidth, backHeight)]; + var screenMask = new Mat(backHeight, backWidth, MatType.CV_8UC1, new Scalar(1)); + + //摄像头小画面定位 + var cameraMargin = 20;//右下角偏移量 + var smallCameraSize = 150; + var smallX = backWidth - smallCameraSize - cameraMargin; + var smallY = backHeight - smallCameraSize - cameraMargin; + var smallRoi = BackgroundFrame[new OpenCvSharp.Rect(smallX, smallY, smallCameraSize, smallCameraSize)]; + var smallMask = new Mat(smallCameraSize, smallCameraSize, MatType.CV_8UC1, new Scalar(1)); + + //摄像头大画面位置 + var bigRoi = BackgroundFrame[new OpenCvSharp.Rect((backWidth - backHeight) / 2, 0, backHeight, backHeight)]; + var bigMask = new Mat(backHeight, backHeight, MatType.CV_8UC1, new Scalar(1)); + + + while (!end) + { + stopwatch.Restart(); + + if (!liveWinMode.ScreenRunning && !liveWinMode.CameraRunning)//有画面的中途关闭画面，清空上一帧 + { + if (!noImgInit) + { + noImgInit = true; + BackgroundFrame.SetTo(backColor); + } + goto Skip2; + } + else + { + if (noImgInit) + { + noImgInit = false; + } + } + + //BackgroundFrame.SetTo(backColor); + if (liveWinMode.ScreenRunning)//屏幕分享中 + { + if (onlyCameraInit) + { + onlyCameraInit = false; + } + lock (SubFrame) + { + if (SubFrame.Cols != BackgroundFrame.Cols || SubFrame.Rows != BackgroundFrame.Rows) + { + goto Skip1;//图像数据不正常直接跳过 + } + SubFrame.CopyTo(screenRoi, screenMask);//屏幕分享和背景一样大，直接覆盖上去 + } + } + else + { + if (!onlyCameraInit)//双画面的中途关闭屏幕画面，清空上一帧 + { + onlyCameraInit = true; + BackgroundFrame.SetTo(backColor); + } + } + + Skip1: + + if (liveWinMode.CameraRunning)//摄像头分享中 + { + lock (MainFrame) + { + if (MainFrame.Cols <= 0 || MainFrame.Rows <= 0) + { + goto Skip2;//图像数据不正常直接跳过 + } + var mainHeight = MainFrame.Rows;//摄像头画面高度 + var mainWidth = MainFrame.Cols;//摄像头画面宽度 + var dst = MainFrame[new OpenCvSharp.Rect((mainWidth - mainHeight) / 2, 0, mainHeight, mainHeight)];//采集大小为原图高的居中正方形，程序分辨率只实现了横屏，因此宽大于高 + if (liveWinMode.ScreenRunning)//屏幕分享中摄像头150正方形大小并悬浮再右下角 + { + dst = dst.Resize(new OpenCvSharp.Size(smallCameraSize, smallCameraSize), interpolation: InterpolationFlags.Area); + dst.CopyTo(smallRoi, smallMask); + } + else//只有摄像头，摄像头画面全屏正方形 + { + dst = dst.Resize(new OpenCvSharp.Size(backHeight, backHeight), interpolation: InterpolationFlags.Linear); + var roi = new Mat(BackgroundFrame, new OpenCvSharp.Rect(0, 0, dst.Width, dst.Height)); + dst.CopyTo(roi); + Cv2.ImShow("123", BackgroundFrame); + Cv2.WaitKey(2); + } + dst.Dispose(); + } + } + + Skip2: + + + stopwatch.Stop(); + Debug.Print($"video frame run {stopwatch.ElapsedMilliseconds}"); + var sleep = delay - (int)stopwatch.ElapsedMilliseconds;//每帧时间减去每帧处理时间为sleep时间 + if (sleep < 0)//如果处理时间超过了每帧时间，记录下来 + { + delayEqualize += sleep; + } + sleep += delayEqualize;//理论休眠时间再去掉补偿时间 + if (sleep > 0) + { + Thread.Sleep(sleep); + } + } + }, TaskCreationOptions.LongRunning);//新开线程 //Stopwatch sw1 = new Stopwatch(); //Stopwatch sw2 = new Stopwatch(); diff --git a/JianGongYun/TRTC/ViewModels/LiveWindowViewModel.cs b/JianGongYun/TRTC/ViewModels/LiveWindowViewModel.cs index d00f300..0340684 100644 --- a/JianGongYun/TRTC/ViewModels/LiveWindowViewModel.cs +++ b/JianGongYun/TRTC/ViewModels/LiveWindowViewModel.cs @@ -235,6 +235,7 @@ namespace JianGongYun.TRTC.ViewModels } } + string[] excluded = new string[] { "Notifiaction" }; private SIZE thumbSize = new SIZE { cx = 300, cy = 200 }; private SIZE iconSize = new SIZE { cx = 50, cy = 50 }; /// @@ -253,6 +254,10 @@ namespace JianGongYun.TRTC.ViewModels for (uint i = 0; i < count; i++) { var info = temp.getSourceInfo(i); + if (string.IsNullOrWhiteSpace(info.sourceName) || excluded.Contains(info.sourceName)) + { + continue; + } //var icon = Util.ToWriteableBitmap(ref info.iconBGRA); //Console.WriteLine($"get LiveScreens {i} {info.sourceId}"); var thumb = info.thumbBGRA.ToWriteableBitmap(); @@ -329,7 +334,7 @@ namespace JianGongYun.TRTC.ViewModels get { var style = new Style(typeof(Border)); - if (CameraRunning &&ScreenRunning) + if (CameraRunning && ScreenRunning) { style.Setters.Add(new Setter { Property = FrameworkElement.HorizontalAlignmentProperty, Value = HorizontalAlignment.Right }); style.Setters.Add(new Setter { Property = FrameworkElement.VerticalAlignmentProperty, Value = VerticalAlignment.Bottom }); diff --git a/JianGongYun/TRTC/Windows/LiveWindow.xaml b/JianGongYun/TRTC/Windows/LiveWindow.xaml index a2f59df..3c9560b 100644 --- a/JianGongYun/TRTC/Windows/LiveWindow.xaml +++ b/JianGongYun/TRTC/Windows/LiveWindow.xaml @@ -82,16 +82,16 @@ - - + + - - + + - - + + @@ -167,6 +167,12 @@ + + 仅开启麦克风 + + + 所有设备被关闭 + diff --git a/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs b/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs index ff498a6..9ab49d3 100644 --- a/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs +++ b/JianGongYun/TRTC/Windows/LiveWindow.xaml.cs @@ -32,6 +32,7 @@ namespace JianGongYun.TRTC.Windows public LiveWindow() { InitializeComponent(); + NoticeManager.Initialize(); //BeforeLiveSubViewWrap.SizeChanged += BeforeLiveSubViewWrap_SizeChanged; //AfterLiveSubViewWrap.SizeChanged += AfterLiveSubViewWrap_SizeChanged; AttachConsole(-1);//把进程挂在控制台，通过命令行启动程序可以看到控制台输出 @@ -271,11 +272,25 @@ namespace JianGongYun.TRTC.Windows } else { - LiveClassroom.StartVideoSub(BeforeLiveSubViewWrap); + if (LiveWindowViewModel.IsLive)//预览和直播是不同的容器 + { + LiveClassroom.StartVideoSub(AfterLiveSubViewWrap); + } + else + { + LiveClassroom.StartVideoSub(BeforeLiveSubViewWrap); + } } if (LiveWindowViewModel.LiveType == Models.LiveTypeEnum.CameraAndScreen) { - LiveClassroom.StartVideoMain(BeforeLiveViewWrap); + if (LiveWindowViewModel.IsLive)//预览和直播是不同的容器 + { + LiveClassroom.StartVideoMain(AfterLiveViewWrap); + } + else + { + LiveClassroom.StartVideoMain(BeforeLiveViewWrap); + } } } @@ -284,5 +299,53 @@ namespace JianGongYun.TRTC.Windows LiveWindowViewModel.ShowShareScreenList = true; this.Dispatcher.Invoke(new Action(() => LiveWindowViewModel.LoadAllScreen())); } + + private void SetMute_Click(object sender, RoutedEventArgs e) + { + LiveClassroom.SetMicMute(null); + NoticeManager.NotifiactionShow.AddNotifiaction(new NotifiactionModel() + { + Title = "提醒", + Content = $"{(LiveWindowViewModel.MicMute ? "设置" : "取消")}静音成功" + }); + } + + private void SetCamera_Click(object sender, RoutedEventArgs e) + { + if (LiveWindowViewModel.CameraRunning) + { + LiveClassroom.StopVideoMain(AfterLiveViewWrap); + } + else + { + LiveClassroom.StartVideoMain(AfterLiveViewWrap); + } + NoticeManager.NotifiactionShow.AddNotifiaction(new NotifiactionModel() + { + Title = "提醒", + Content = $"{(LiveWindowViewModel.CameraRunning ? "开启" : "关闭")}摄像头成功" + }); + } + private void SetScreen_Click(object sender, RoutedEventArgs e) + { + if (LiveWindowViewModel.ScreenRunning) + { + LiveClassroom.StopVideoSub(AfterLiveSubViewWrap); + } + else + { + if (LiveWindowViewModel.CurrentShareScreen == null)//没选择过窗口需要选一次 + { + LiveWindowViewModel.ShowShareScreenList = true; + this.Dispatcher.Invoke(new Action(() => LiveWindowViewModel.LoadAllScreen())); + } + LiveClassroom.StartVideoSub(AfterLiveSubViewWrap); + } + NoticeManager.NotifiactionShow.AddNotifiaction(new NotifiactionModel() + { + Title = "提醒", + Content = $"{(LiveWindowViewModel.ScreenRunning ? "开启" : "关闭")}屏幕分享成功" + }); + } } } diff --git a/JianGongYun/TRTC/Windows/SettingWindow.xaml b/JianGongYun/TRTC/Windows/SettingWindow.xaml index cca645e..bf4a294 100644 --- a/JianGongYun/TRTC/Windows/SettingWindow.xaml +++ b/JianGongYun/TRTC/Windows/SettingWindow.xaml @@ -30,7 +30,7 @@ 摄像头 - + 麦克风