using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Security.Authentication; using System.Text; using System.Threading.Tasks; using System.Windows; using System.Windows.Input; using Tango.Core; using Tango.Core.DI; using Tango.Core.Threading; using Tango.Integration.ExternalBridge; using Tango.Logging; using Tango.PPC.Common.Application; using Tango.PPC.Common.Build; using Tango.PPC.Common.ExternalBridge; using Tango.PPC.Common.OS; using Tango.PPC.Common.Printing; using Tango.RemoteDesktop; using Tango.RemoteDesktop.CaptureMethods; using Tango.RemoteDesktop.Encoders; using Tango.RemoteDesktop.Engines; using Tango.RemoteDesktop.Frames; using Tango.RemoteDesktop.Input; using Tango.RemoteDesktop.Network; using Tango.Settings; using Tango.Transport; using Tango.WebRTC; using static Tango.RemoteDesktop.Input.MouseController; namespace Tango.PPC.Common.RemoteDesktop { [TangoCreateWhenRegistered] public class DefaultRemoteDesktopService : ExtendedObject, IRemoteDesktopService, IExternalBridgeRequestHandler { private RemoteDesktopPacket _initialPacket; private RasterScreenCaptureEngine _engine; private PPCSettings _settings; private List _clients; private JsonSerializerSettings _jsonSettings; private IOperationSystemManager _osManager; private IPPCApplicationManager _appManager; private IPrintingManager _printingManager; private bool _drawCursor; private bool _isMouseDown; private bool _ensureMouseDown; private ActionTimer _resetTimer; private IBuildProvider _buildProvider; /// /// Gets or sets a value indicating whether this is enabled. /// public bool Enabled { get; set; } = true; private bool _isStarted; /// /// Gets a value indicating whether the remote desktop service has started. /// public bool IsStarted { get { return _isStarted; } private set { _isStarted = value; RaisePropertyChangedAuto(); } } /// /// Gets a value indicating whether there is any active remote desktop session with a remote peer. /// public bool InSession { get { return _clients.Count > 0; } } /// /// Initializes a new instance of the class. /// /// The application manager. /// The external bridge. /// The os manager. public DefaultRemoteDesktopService(IPPCApplicationManager applicationManager, IPPCExternalBridgeService externalBridge, IOperationSystemManager osManager, IPrintingManager printingManager, IBuildProvider buildProvider) { _osManager = osManager; _appManager = applicationManager; _printingManager = printingManager; _buildProvider = buildProvider; _jsonSettings = new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.All }; _settings = SettingsManager.Default.GetOrCreate(); Enabled = _settings.EnableRemoteDesktop; applicationManager.ApplicationReady += ApplicationManager_ApplicationReady; externalBridge.RegisterRequestHandler(this); _clients = new List(); _engine = new RasterScreenCaptureEngine(); _engine.CaptureCursor = false; _engine.FrameRate = Math.Min(Math.Max(_settings.RemoteDesktopFrameRate, 1), 20); _engine.FrameReceived += _engine_FrameReceived; _resetTimer = new ActionTimer(TimeSpan.FromSeconds(5)); } private void ApplicationManager_ApplicationReady(object sender, EventArgs e) { var mainWindow = System.Windows.Application.Current.MainWindow; if (_buildProvider.MachineType.IsXMachine()) { _engine.CaptureRegion = new CaptureRegion() { Left = (int)mainWindow.Left, Top = 0, Width = 1920, Height = 1080 }; } if (mainWindow.WindowStyle != System.Windows.WindowStyle.None) { mainWindow.LocationChanged += (_, __) => { _engine.CaptureRegion = new CaptureRegion() { Left = (int)mainWindow.Left + 10, Top = (int)mainWindow.Top + 5, Width = (int)mainWindow.Width - 20, Height = (int)mainWindow.Height - 15 }; }; _engine.CaptureRegion = new CaptureRegion() { Left = (int)mainWindow.Left + 10, Top = (int)mainWindow.Top + 5, Width = (int)mainWindow.Width - 20, Height = (int)mainWindow.Height - 15 }; } else { //DirectX capturing is not working on PPC !! Maybe when we replace ? //try //{ // _engine.CaptureMethod = new DirectXScreenCapture(); //} //catch (Exception ex) //{ // LogManager.Log(ex, "Could not initialize DirectX screen capture method on this device. Falling back to GDI."); //} } mainWindow.PreviewMouseDown += (_, __) => { _isMouseDown = true; _ensureMouseDown = true; }; mainWindow.PreviewMouseUp += (_, __) => { _isMouseDown = false; }; _engine.Comparer.MaxDifferencesThrow = _engine.CaptureRegion.Width * _engine.CaptureRegion.Height / 2; } [ExternalBridgeRequestHandlerMethod(typeof(StartRemoteDesktopSessionRequest), RequestHandlerLoggingMode.LogRequestName)] public async Task OnStartRemoteDesktopSessionRequestReceived(StartRemoteDesktopSessionRequest request, String token, ExternalBridgeReceiver receiver) { this.ThrowIfDisabled(); var client = _clients.SingleOrDefault(x => x.Receiver == receiver); if (client != null) { _clients.Remove(client); } RemoteDesktopClient newClient = new RemoteDesktopClient(); newClient.Receiver = receiver; newClient.Token = token; newClient.WebRtcClient = new WebRtcClient(); newClient.WebRtcClient.Tag = receiver; newClient.WebRtcClient.TextMessageReceived += WebRtcClient_TextMessageReceived; _clients.Add(newClient); await receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse() { FrameRate = _engine.FrameRate }, token); if (!_engine.IsStarted) { _engine.Start(); } RaisePropertyChanged(nameof(InSession)); } [ExternalBridgeRequestHandlerMethod(typeof(WebRtcIceCandidateRequest), RequestHandlerLoggingMode.LogRequestName)] public async Task OnWebRtcIceCandidateRequestReceived(WebRtcIceCandidateRequest request, String token, ExternalBridgeReceiver receiver) { var client = _clients.SingleOrDefault(x => x.Receiver == receiver); if (client != null) { try { await receiver.SendGenericResponse(new WebRtcIceCandidateResponse() { }, token); client.WebRtcClient.AddIceCandidate(request.IceCandidate); } catch (Exception ex) { LogManager.Log(ex, "Error adding WebRTC ice candidate received from the remote connection."); } } } [ExternalBridgeRequestHandlerMethod(typeof(WebRtcOfferRequest), RequestHandlerLoggingMode.LogRequestName)] public async Task OnWebRtcOfferRequestReceived(WebRtcOfferRequest request, String token, ExternalBridgeReceiver receiver) { var client = _clients.SingleOrDefault(x => x.Receiver == receiver); if (client != null) { try { try { client.WebRtcClient.NewIceCandidate += async (x, e) => { try { await receiver.SendGenericRequest(new WebRtcIceCandidateRequest() { IceCandidate = e.IceCandidate }); } catch (Exception ex) { LogManager.Log(ex, "Error sending ice candidate to remote peer."); } }; client.WebRtcClient.Ready += (x, e) => { client.IsWebRtcReady = true; }; client.WebRtcClient.Disconnected += (x, e) => { client.IsWebRtcReady = false; }; client.WebRtcClient.FrameWidth = 800; client.WebRtcClient.FrameHeight = 1280; client.WebRtcClient.FrameRate = _engine.FrameRate; await client.WebRtcClient.Init(); var answer = await client.WebRtcClient.CreateAnswer(request.Offer); await receiver.SendGenericResponse(new WebRtcOfferResponse() { Answer = answer }, token); } catch (Exception ex) { throw LogManager.Log(ex, "Error initializing the WebRTC client."); } } catch (Exception ex) { throw LogManager.Log(ex, "Error responding to WebRTC offer request."); } } } [ExternalBridgeRequestHandlerMethod(typeof(StopRemoteDesktopSessionRequest), RequestHandlerLoggingMode.LogRequestName)] public async Task OnStopRemoteDesktopSessionRequestReceived(StopRemoteDesktopSessionRequest request, String token, ExternalBridgeReceiver receiver) { var client = _clients.SingleOrDefault(x => x.Receiver == receiver); if (client != null) { _clients.Remove(client); if (client.WebRtcClient != null) { client.WebRtcClient.Dispose(); } } if (_clients.Count == 0) { _engine.Stop(); } await receiver.SendGenericResponse(new StopRemoteDesktopSessionResponse(), token); if (client != null) { await receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse(), client.Token, new TransportResponseConfig() { Completed = true }); } RaisePropertyChanged(nameof(InSession)); } [ExternalBridgeRequestHandlerMethod(typeof(MouseStateRequest))] public async Task OnMouseStateRequestReceived(MouseStateRequest request, String token, ExternalBridgeReceiver receiver) { MouseController.SetCursorPosition((int)request.Location.X, (int)request.Location.Y); if (request.EventType == MouseEventType.Up || request.EventType == MouseEventType.Down) { MouseEventFlags flag = MouseEventFlags.LeftUp; switch (request.EventType) { case MouseEventType.Down: flag = request.Button == MouseButton.Right ? MouseEventFlags.RightDown : MouseEventFlags.LeftDown; break; case MouseEventType.Up: flag = request.Button == MouseButton.Right ? MouseEventFlags.RightUp : MouseEventFlags.LeftUp; break; } if (!receiver.AllowSafetyLevelOperations) { Debug.WriteLine("PREVENT PRINT"); _printingManager.PreventPrintingByRemoteDesktop = true; } MouseController.MouseEvent(flag); _resetTimer.ResetReplace(() => { _printingManager.PreventPrintingByRemoteDesktop = false; Debug.WriteLine("ALLOW PRINT"); }); } else if (request.EventType == MouseEventType.DoubleClick) { if (!receiver.AllowSafetyLevelOperations) { _printingManager.PreventPrintingByRemoteDesktop = true; } MouseController.DoubleClick(); _resetTimer.ResetReplace(() => { _printingManager.PreventPrintingByRemoteDesktop = false; }); } else if (request.EventType == MouseEventType.Scroll) { MouseController.Scroll(request.ScrollDelta); } if (receiver != null) { await receiver.SendGenericResponse(new MouseStateResponse(), token); } } [ExternalBridgeRequestHandlerMethod(typeof(TouchStateRequest))] public async Task OnTouchStateRequestReceived(TouchStateRequest request, String token, ExternalBridgeReceiver receiver) { switch (request.EventType) { case TouchEventType.TouchDown: TouchController.TouchDown((int)request.Location.X, (int)request.Location.Y); break; case TouchEventType.TouchMove: TouchController.TouchMove(request.MoveDeltaX, request.MoveDeltaY); break; case TouchEventType.TouchUp: TouchController.TouchUp(); break; } if (receiver != null) { await receiver.SendGenericResponse(new TouchStateResponse(), token); } } [ExternalBridgeRequestHandlerMethod(typeof(KeyboardStateRequest))] public async Task OnKeyboardStateRequestReceived(KeyboardStateRequest request, String token, ExternalBridgeReceiver receiver) { if (request.EventType == KeyboardEventType.Down) { KeyboardController.KeyDown(request.Key, request.IsCtrlDown, request.IsShiftDown, request.IsAltDown); } else { KeyboardController.KeyUp(request.Key, request.IsCtrlDown, request.IsShiftDown, request.IsAltDown); } if (receiver != null) { await receiver.SendGenericResponse(new KeyboardStateResponse(), token); } } [ExternalBridgeRequestHandlerMethod(typeof(RemoteDesktopCommandRequest), RequestHandlerLoggingMode.LogRequestName)] public async Task OnRemoteDesktopCommandRequest(RemoteDesktopCommandRequest request, String token, ExternalBridgeReceiver receiver) { switch (request.Command) { case RemoteDesktopCommand.HideAndOpenShell: _osManager.OpenShell(); _appManager.SetWindowState(System.Windows.WindowState.Minimized); break; } await receiver.SendGenericResponse(new RemoteDesktopCommandResponse(), token); } [ExternalBridgeRequestHandlerMethod(typeof(SetCursorVisibilityRequest), RequestHandlerLoggingMode.LogRequestName)] public async Task OnSetCursorVisibilityRequest(SetCursorVisibilityRequest request, String token, ExternalBridgeReceiver receiver) { _drawCursor = request.Visible; await receiver.SendGenericResponse(new SetCursorVisibilityResponse(), token); } [ExternalBridgeRequestHandlerMethod(typeof(GetScreenshotRequest), RequestHandlerLoggingMode.LogRequestName)] public async Task OnGetScreenshotRequest(GetScreenshotRequest request, String token, ExternalBridgeReceiver receiver) { GdiScreenCapture capture = new GdiScreenCapture(); var bitmap = capture.GetDesktopBitmap(CaptureRegion.PrimaryScreenBounds()); RasterFrame frame = new RasterFrame(bitmap); var data = frame.ToEncoder().ToArray(80); frame.Dispose(); await receiver.SendGenericResponse(new GetScreenshotResponse() { Bitmap = data }, token); } private async void _engine_FrameReceived(object sender, ScreenCaptureFrameReceivedEventArgs e) { try { if (_drawCursor) { e.Frame.DrawImage((_isMouseDown || _ensureMouseDown) ? Properties.Resources.tap : Properties.Resources.finger3, new System.Drawing.Point(System.Windows.Forms.Cursor.Position.X - 5, System.Windows.Forms.Cursor.Position.Y - 4)); _ensureMouseDown = false; } _initialPacket = new RemoteDesktopPacket() { Bitmap = e.Frame.ToEncoder().ToArray(), }; if (_clients.Count > 0) { bool useWebRTC = _clients.ToList().All(x => x.IsWebRtcReady); if (useWebRTC) { _engine.EnableComparer = false; foreach (var client in _clients.ToList()) { try { client.WebRtcClient.PushFrame(e.Frame.ToBitmap()); } catch (Exception ex) { LogManager.Log(ex, LogCategory.Debug, "Error pushing remote desktop frame via WebRTC channel."); } } e.Frame.Dispose(); } else { _engine.EnableComparer = !_buildProvider.MachineType.IsXMachine(); foreach (var client in _clients.ToList().Where(x => !x.InitialPacketSent)) { try { await client.Receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse() { Packet = _initialPacket, }, client.Token, new TransportResponseConfig() { Immediate = false, }); client.InitialPacketSent = true; } catch (Exception ex) { Debug.WriteLine(ex); } } if (e.Frame.DifferenceCount > 0 || _buildProvider.MachineType.IsXMachine()) { RemoteDesktopPacket packet = null; Point mousePosition = new Point(System.Windows.Forms.Cursor.Position.X, System.Windows.Forms.Cursor.Position.Y); if (!e.Frame.DifferenceAvailable || _buildProvider.MachineType.IsXMachine()) { packet = new RemoteDesktopPacket() { Bitmap = e.Frame.ToEncoder().ToArray(30), MousePosition = mousePosition, CursorVisible = _drawCursor }; } else { var diffFrame = e.Frame.ToDifference(); diffFrame = diffFrame.OptimizeBounds(); packet = new RemoteDesktopPacket() { Bitmap = diffFrame.ToEncoder().ToArray(), IsPartial = true, PartialRegion = new CaptureRegion(diffFrame.Left, diffFrame.Top, diffFrame.Width, diffFrame.Height), MousePosition = mousePosition, CursorVisible = _drawCursor }; diffFrame.Dispose(); } Debug.WriteLine($"Remote Desktop Bitmap Size: {packet.Bitmap.Length / 1000} kb"); foreach (var client in _clients.ToList().Where(x => x.InitialPacketSent)) { try { await client.Receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse() { Packet = packet }, client.Token, new TransportResponseConfig() { Immediate = false, }); } catch (Exception ex) { Debug.WriteLine(ex); } } } } } } catch (Exception ex) { LogManager.Log(ex, "Error occurred on remote desktop engine frame received event."); } finally { e.Frame.Dispose(); } } private async void WebRtcClient_TextMessageReceived(object sender, DataMessageReceivedEventArgs e) { try { var request = JsonConvert.DeserializeObject(e.Data, _jsonSettings); if (request.GetType() == typeof(MouseStateRequest)) { await OnMouseStateRequestReceived(request as MouseStateRequest, null, (sender as WebRtcClient).Tag as ExternalBridgeReceiver); } else if (request.GetType() == typeof(KeyboardStateRequest)) { await OnKeyboardStateRequestReceived(request as KeyboardStateRequest, null, null); } else if (request.GetType() == typeof(TouchStateRequest)) { await OnTouchStateRequestReceived(request as TouchStateRequest, null, null); } } catch (Exception ex) { LogManager.Log(ex, "Error deserializing incoming message on the WebRTC data Channel."); } } public void OnReceiverDisconnected(ExternalBridgeReceiver receiver) { var client = _clients.SingleOrDefault(x => x.Receiver == receiver); if (client != null) { LogManager.Log("Remote desktop client disconnected. Disposing WebRTC channel..."); _clients.Remove(client); try { if (client.WebRtcClient != null) { client.WebRtcClient.Dispose(); } } catch (Exception ex) { LogManager.Log(ex, "Error disposing the WebRTC channel."); } } if (_clients.Count == 0) { _engine.Stop(); } RaisePropertyChanged(nameof(InSession)); } } }