using Newtonsoft.Json; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; using System.Windows; using System.Windows.Input; using System.Windows.Media.Imaging; using Tango.BL.Enumerations; using Tango.Core; using Tango.Core.DI; using Tango.Core.ExtensionMethods; using Tango.Core.Threading; using Tango.FSE.Common.Authentication; using Tango.FSE.Common.Connection; using Tango.FSE.Common.Notifications; using Tango.FSE.Common.RemoteDesktop; using Tango.PMR; using Tango.PMR.Integration; using Tango.RemoteDesktop.Frames; using Tango.RemoteDesktop.Network; using Tango.Transport; using Tango.WebRTC; namespace Tango.FSE.UI.RemoteDesktop { /// /// Represents the default implementation. /// /// /// public class DefaultRemoteDesktopProvider : ExtendedObject, IRemoteDesktopProvider { private class MouseMovement { public Point Location { get; set; } public Size ViewSize { get; set; } } private class TouchMovement { public Size ViewSize { get; set; } public int DeltaX { get; set; } public int DeltaY { get; set; } } private IMachineProvider _machineProvider; private RasterFrame _currentFrame; private MemoryStream _currentStream; private Size? _frameSize; private IntervalMessageDispatcher _frameDispatcher; private Thread _mouseMoveThread; private Thread _touchMoveThread; private ProducerConsumerQueue _mouseMovements; private ProducerConsumerQueue _touchMovements; private WebRtcClient _webRtcClient; private List _iceCandidates; private bool _answerReceived; private JsonSerializerSettings _jsonSettings; private bool _cursorVisible; [TangoInject] private INotificationProvider NotificationProvider { get; set; } [TangoInject] private IAuthenticationProvider AuthenticationProvider { get; set; } #region Events /// /// Occurs when a remote desktop session has started. /// public event EventHandler SessionStarted; /// /// Occurs when a remote desktop session has stopped. /// public event EventHandler SessionStopped; /// /// Occurs when a new remote desktop screen frame is available. /// public event EventHandler FrameReceived; #endregion #region Properties /// /// Gets the current remote desktop session frame rate. /// public double FrameRate { get; private set; } /// /// Gets the current remote desktop session frame width. /// public int FrameWidth { get; private set; } /// /// Gets the current remote desktop session frame height. /// public int FrameHeight { get; private set; } private bool _inSession; /// /// Gets a value indicating whether a remote desktop session is active. /// public bool InSession { get { return _inSession; } set { bool changed = _inSession != value; _inSession = value; RaisePropertyChangedAuto(); RaisePropertyChanged(nameof(CanStartSession)); if (changed) { if (_inSession) { SessionStarted?.Invoke(this, new EventArgs()); } else { SessionStopped.Invoke(this, new EventArgs()); } } } } /// /// Gets or sets a value indicating whether enable the WebRTC channel. /// public bool EnableWebRtc { get; set; } private bool _isWebRtcActive; /// /// Gets a value indicating whether the WebRTC channel is available. /// public bool IsWebRtcActive { get { return _isWebRtcActive; } set { if (_isWebRtcActive != value) { _isWebRtcActive = value; RaisePropertyChangedAuto(); if (_isWebRtcActive) { LogManager.Log("WebRTC is now active."); } } } } /// /// Gets or sets the mouse move send interval. /// public TimeSpan MouseMoveInterval { get; set; } /// /// Gets a value indicating whether a remote desktop session can be started. /// public bool CanStartSession { get { return _machineProvider.IsConnected && !InSession; } } #endregion #region Constructors /// /// Initializes a new instance of the class. /// public DefaultRemoteDesktopProvider() { _jsonSettings = new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.All, }; _iceCandidates = new List(); MouseMoveInterval = TimeSpan.FromMilliseconds(100); EnableWebRtc = true; } /// /// Initializes a new instance of the class. /// /// The machine provider. public DefaultRemoteDesktopProvider(IMachineProvider machineProvider) : this() { _machineProvider = machineProvider; _machineProvider.MachineConnected += (_, __) => { InSession = false; }; _machineProvider.MachineDisconnected += (_, __) => { InSession = false; OnFrameReceived(null); }; _machineProvider.MachineOperator.RegisterRequestHandler(OnIceCandidateRequestReceived); } #endregion #region Request Handlers /// /// Called when a new ice candidate received from the remote peer. /// /// The transporter. /// The request. /// The token. private async void OnIceCandidateRequestReceived(ITransporter transporter, WebRtcIceCandidateRequest request, string token) { try { LogManager.Log("Ice candidate request received from the remote peer. Adding ice candidate and sending confirmation response..."); _webRtcClient.AddIceCandidate(request.IceCandidate); } catch (Exception ex) { LogManager.Log(ex, $"Error adding ice candidate:\n{request.IceCandidate.ToJsonString()}"); } try { await _machineProvider.MachineOperator.SendGenericResponse(new WebRtcIceCandidateResponse() { }, token); } catch (Exception ex) { LogManager.Log(ex, "Error sending Ice candidate confirmation response."); } } #endregion #region Start/Stop Session /// /// Starts a remote desktop session. /// /// /// Unable to start a remote desktop session at the moment. public async Task StartSession() { AuthenticationProvider.ThrowIfNoPermission(Permissions.FSE_RemoteDesktopView); TaskCompletionSource completionSource = new TaskCompletionSource(); if (!CanStartSession) { throw new InvalidOperationException("Unable to start a remote desktop session at the moment."); } if (!InSession) { LogManager.Log("Starting remote desktop session..."); _currentFrame = null; _currentStream = null; _iceCandidates.Clear(); _answerReceived = false; InSession = true; var taskItem = NotificationProvider.PushTaskItem("Starting remote desktop session..."); await Task.Delay(2000); if (_frameDispatcher != null) { _frameDispatcher.Dispose(); } _frameDispatcher = new IntervalMessageDispatcher(OnRemoteDesktopResponse); _frameDispatcher.DriftCompensationInterval = (int)TimeSpan.FromSeconds(10).TotalMilliseconds; _frameDispatcher.Start(); bool taskCompleted = false; LogManager.Log("Sending continuous remote desktop session request..."); _machineProvider.MachineOperator.SendGenericContinuousRequest(new StartRemoteDesktopSessionRequest() { }, new TransportContinuousRequestConfig() { Timeout = TimeSpan.FromSeconds(10) }).Subscribe((response) => { if (!taskCompleted) { LogManager.Log("Remote desktop session started."); taskCompleted = true; taskItem.Dispose(); completionSource.SetResult(true); if (EnableWebRtc) { LogManager.Log("WebRTC for remote desktop is enabled. Starting WebRTC channel..."); StartWebRTC(); } } _frameDispatcher.Push(response); }, (ex) => { if (!taskCompleted) { taskCompleted = true; taskItem.Dispose(); completionSource.SetException(ex); } else if (InSession) { OnFailed(ex); } InSession = false; }, () => { InSession = false; OnFrameReceived(null); }); await completionSource.Task; } else { await Task.FromResult(true); } } /// /// Ends the current remote desktop session. /// /// public async Task EndSession() { if (InSession) { using (NotificationProvider.PushTaskItem("Stopping remote desktop session...")) { LogManager.Log("Stopping remote desktop session..."); await Task.Delay(2000); try { LogManager.Log("Sending stop remote desktop session request..."); await _machineProvider.MachineOperator.SendGenericRequest(new StopRemoteDesktopSessionRequest(), new TransportRequestConfig() { ShouldLog = true }); LogManager.Log("Remote desktop session stopped."); } catch (Exception ex) { LogManager.Log(ex, "Error stopping the remote desktop session."); } DisposeWebRtc(); InSession = false; } } } #endregion #region Virtual Methods /// /// Raises the event. /// /// The source. /// The origin. protected virtual void OnFrameReceived(BitmapSource source, DesktopFrameReceivedEventArgs.FrameOrigin origin = DesktopFrameReceivedEventArgs.FrameOrigin.WebSockets) { IsWebRtcActive = origin == DesktopFrameReceivedEventArgs.FrameOrigin.WebRTC; if (InSession) { FrameWidth = source.PixelWidth; FrameHeight = source.PixelHeight; FrameReceived?.Invoke(this, new DesktopFrameReceivedEventArgs() { Source = source, Origin = origin, CursorVisible = _cursorVisible }); } } #endregion #region SignalR Response Received /// /// Called when by the interval message dispatcher for processing a frame received by the standard SignalR channel. /// /// The response. private void OnRemoteDesktopResponse(StartRemoteDesktopSessionResponse response) { if (!InSession) return; //To stop frame delivery immediately. if (response.Packet == null) { FrameRate = response.FrameRate; LogManager.Log($"Remote desktop session frame rate is now: {response.FrameRate} per second."); _frameDispatcher.Interval = 1000 / response.FrameRate; return; //This is the first message with empty bitmap and only the frame rate. } try { if (!response.Packet.IsPartial) { _currentFrame?.Dispose(); _currentStream?.Dispose(); _currentStream = new MemoryStream(response.Packet.Bitmap); _currentFrame = new RasterFrame(new System.Drawing.Bitmap(_currentStream)); _frameSize = new Size(_currentFrame.Width, _currentFrame.Height); } else { using (MemoryStream ms = new MemoryStream(response.Packet.Bitmap)) { var diffFrame = new RasterFrame(new System.Drawing.Bitmap(ms), response.Packet.PartialRegion.Left, response.Packet.PartialRegion.Top); diffFrame.Apply(_currentFrame); diffFrame.Dispose(); } } _cursorVisible = response.Packet.CursorVisible; OnFrameReceived(_currentFrame.ToBitmapSource()); } catch (Exception ex) { LogManager.Log(ex, "Error occurred while trying to process a remote desktop response."); } } #endregion #region Mouse / Keyboard / Touch /// /// Sends a mouse down command to the remote PPC. /// /// The button. /// The location. /// Size of the view. public async void MouseDown(MouseButton button, Point location, Size viewSize) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; try { var request = new MouseStateRequest() { Button = button, EventType = MouseEventType.Down, Location = TranslateLocation(location, viewSize) }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop mouse down command."); } } /// /// Sends a mouse up command to the remote PPC. /// /// The button. /// The location. /// Size of the view. public async void MouseUp(MouseButton button, Point location, Size viewSize) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; try { var request = new MouseStateRequest() { Button = button, EventType = MouseEventType.Up, Location = TranslateLocation(location, viewSize) }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop mouse up command."); } } /// /// Sends a mouse move command to the remote PPC. /// /// The location. /// Size of the view. public void MouseMove(Point location, Size viewSize) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; if (_mouseMoveThread == null) { _mouseMovements = new ProducerConsumerQueue(); _mouseMoveThread = new Thread(async () => { while (InSession) { MouseMovement movement = _mouseMovements.BlockDequeue(); while (_mouseMovements.Count > 0) { movement = _mouseMovements.BlockDequeue(); } try { var request = new MouseStateRequest() { Button = MouseButton.Left, EventType = MouseEventType.Move, Location = TranslateLocation(movement.Location, movement.ViewSize) }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop mouse move command."); } Thread.Sleep(MouseMoveInterval); } _mouseMoveThread = null; }); _mouseMoveThread.IsBackground = true; _mouseMoveThread.Start(); } _mouseMovements.BlockEnqueue(new MouseMovement() { Location = location, ViewSize = viewSize, }); } /// /// Sends a mouse double click command to the remote PPC. /// /// The button. /// The location. /// Size of the view. public async void MouseDoubleClick(MouseButton button, Point location, Size viewSize) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; try { var request = new MouseStateRequest() { Button = button, EventType = MouseEventType.DoubleClick, Location = TranslateLocation(location, viewSize) }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop mouse double click command."); } } /// /// Send a mouse scroll command to the remote PPC. /// /// The delta. /// The location. /// Size of the view. public async void MouseScroll(int delta, Point location, Size viewSize) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; try { var request = new MouseStateRequest() { Button = MouseButton.Middle, EventType = MouseEventType.Scroll, Location = TranslateLocation(location, viewSize), ScrollDelta = delta }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop mouse scroll command."); } } /// /// Sets the remote cursor visibility. /// /// if set to true [visible]. public async void SetRemoteCursorVisibility(bool visible) { try { var request = new SetCursorVisibilityRequest() { Visible = visible }; await _machineProvider.MachineOperator.SendGenericRequest(request); _cursorVisible = visible; } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop mouse visibility command."); } } /// /// Translates the location. /// /// The location. /// Size of the view. /// private Point TranslateLocation(Point location, Size viewSize) { return new Point( (location.X / viewSize.Width) * _frameSize.Value.Width, (location.Y / viewSize.Height) * _frameSize.Value.Height); } /// /// Sends a key down command to the remote PPC. /// /// The key. /// if set to true [control down]. /// if set to true [shit down]. /// if set to true [alt down]. public async void KeyboardDown(Key key, bool ctrlDown, bool shitDown, bool altDown) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; if (key == Key.LeftCtrl || key == Key.LeftShift || key == Key.LeftAlt) return; try { var request = new KeyboardStateRequest() { EventType = KeyboardEventType.Down, Key = key, IsCtrlDown = ctrlDown, IsShiftDown = shitDown, IsAltDown = altDown, }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop key down command."); } } /// /// Sends a key up command to the remote PPC. /// /// The key. /// if set to true [control down]. /// if set to true [shit down]. /// if set to true [alt down]. public async void KeyboardUp(Key key, bool ctrlDown, bool shitDown, bool altDown) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; if (key == Key.LeftCtrl || key == Key.LeftShift || key == Key.LeftAlt) return; try { var request = new KeyboardStateRequest() { EventType = KeyboardEventType.Up, Key = key, IsCtrlDown = ctrlDown, IsShiftDown = shitDown, IsAltDown = altDown, }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop key up command."); } } public async void TouchDown(Point location, Size viewSize) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; try { var request = new TouchStateRequest() { EventType = TouchEventType.TouchDown, Location = TranslateLocation(location, viewSize) }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop touch down command."); } } public void TouchMove(int deltaX, int deltaY, Size viewSize) { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; if (_touchMoveThread == null) { _touchMovements = new ProducerConsumerQueue(); _touchMoveThread = new Thread(async () => { while (InSession) { TouchMovement movement = _touchMovements.BlockDequeue(); while (_touchMovements.Count > 0) { movement = _touchMovements.BlockDequeue(); } try { var translatedDelta = TranslateLocation(new Point(deltaX, deltaY), viewSize); var request = new TouchStateRequest() { EventType = TouchEventType.TouchMove, MoveDeltaX = (int)translatedDelta.X, MoveDeltaY = (int)translatedDelta.Y, }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop touch move command."); } Thread.Sleep(MouseMoveInterval); } _touchMoveThread = null; }); _touchMoveThread.IsBackground = true; _touchMoveThread.Start(); } _touchMovements.BlockEnqueue(new TouchMovement() { DeltaX = deltaX, DeltaY = deltaY, ViewSize = viewSize, }); } public async void TouchUp() { if (!InSession || _frameSize == null) return; if (!AuthenticationProvider.CurrentUser.HasPermission(Permissions.FSE_RemoteDesktopControl)) return; try { var request = new TouchStateRequest() { EventType = TouchEventType.TouchUp }; if (IsWebRtcActive) { SendWebRtcObject(request); } else { await _machineProvider.MachineOperator.SendGenericRequest(request); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop touch up command."); } } #endregion #region Commands /// /// Sends the specified predefined remote desktop command. /// /// The command. public async void SendCommand(RemoteDesktopCommand command) { AuthenticationProvider.ThrowIfNoPermission(Permissions.FSE_RemoteDesktopControl); try { await _machineProvider.MachineOperator.SendGenericRequest(new RemoteDesktopCommandRequest()); } catch (Exception ex) { LogManager.Log(ex, $"Error sending remote desktop command '{command}'."); } } #endregion #region WebRTC /// /// Starts the WebRTC channel. /// private async void StartWebRTC() { try { LogManager.Log("Starting Remote desktop WebRTC channel..."); _webRtcClient = new WebRtcClient(); _webRtcClient.NewIceCandidate += _webRtcClient_NewIceCandidate; _webRtcClient.FrameReceived += _webRtcClient_FrameReceived; _webRtcClient.Disconnected += _webRtcClient_Disconnected; LogManager.Log("Initializing WebRTC client..."); await _webRtcClient.Init(); LogManager.Log("Creating WebRTC offer..."); var offer = await _webRtcClient.CreateOffer(); LogManager.Log("Sending WebRTC offer..."); var response = await _machineProvider.MachineOperator.SendGenericRequest(new WebRtcOfferRequest() { Offer = offer }, new TransportRequestConfig() { Timeout = TimeSpan.FromSeconds(30) }); LogManager.Log("Remote desktop WebRTC offer sent and responded with an answer. Setting WebRTC answer..."); _webRtcClient.SetAnswer(response.Answer); _answerReceived = true; foreach (var ice in _iceCandidates.ToList()) { LogManager.Log("Sending any existing ice candidate..."); try { await _machineProvider.MachineOperator.SendGenericRequest(new WebRtcIceCandidateRequest() { IceCandidate = ice }, new TransportRequestConfig() { Timeout = TimeSpan.FromSeconds(30), }); } catch (Exception ex) { LogManager.Log(ex, "Error sending existing ice candidate. Ignoring..."); } } } catch (Exception ex) { LogManager.Log(ex, "Error starting WebRTC."); } } /// /// Disposes the WebRTC channel. /// private void DisposeWebRtc() { if (_webRtcClient != null) { LogManager.Log("Disposing remote desktop WebRTC channel..."); _webRtcClient.NewIceCandidate -= _webRtcClient_NewIceCandidate; _webRtcClient.FrameReceived -= _webRtcClient_FrameReceived; _webRtcClient.Disconnected -= _webRtcClient_Disconnected; if (_webRtcClient != null) { try { _webRtcClient.Dispose(); _webRtcClient = null; LogManager.Log("Remote desktop WebRTC channel disposed properly."); } catch (Exception ex) { LogManager.Log(ex, "Error occurred while disposing remote desktop WebRTC channel."); } } } } /// /// Handles the event. /// /// The source of the event. /// The instance containing the event data. private void _webRtcClient_Disconnected(object sender, EventArgs e) { LogManager.Log("The remote desktop WebRTC channel has disconnected."); IsWebRtcActive = false; } /// /// Handles the event. /// /// The source of the event. /// The instance containing the event data. private async void _webRtcClient_NewIceCandidate(object sender, NewIceCandidateEventArgs e) { try { if (_answerReceived) { LogManager.Log("New remote desktop WebRTC candidate available. Sending ice to remote peer..."); await _machineProvider.MachineOperator.SendGenericRequest(new WebRtcIceCandidateRequest() { IceCandidate = e.IceCandidate }, new TransportRequestConfig() { Timeout = TimeSpan.FromSeconds(30), ShouldLog = true }); } else { LogManager.Log("New remote desktop WebRTC candidate available. Will be sent after an answer is received..."); _iceCandidates.Add(e.IceCandidate); } } catch (Exception ex) { LogManager.Log(ex, "Error sending remote desktop ice candidate to remote peer."); } } /// /// Handles the event. /// /// The source of the event. /// The instance containing the event data. private void _webRtcClient_FrameReceived(object sender, VideoFrameReceivedEventArgs e) { try { RasterFrame frame = new RasterFrame(e.Bitmap); OnFrameReceived(frame.ToBitmapSource(), DesktopFrameReceivedEventArgs.FrameOrigin.WebRTC); frame.Dispose(); } catch (Exception ex) { LogManager.Log(ex, "Error while processing a remote desktop frame received by WebRTC channel."); } } /// /// Sends a serialized JSON object via the WebRTC data channel. /// /// The object. private void SendWebRtcObject(object obj) { _webRtcClient.SendText(JsonConvert.SerializeObject(obj, _jsonSettings)); } #endregion #region OnFailed /// /// Called when the current remote desktop channel has completely failed. /// /// The exception. private void OnFailed(Exception exception) { LogManager.Log(exception, "The current remote desktop session has failed."); InSession = false; OnFrameReceived(null); DisposeWebRtc(); if (!(exception is TransporterDisconnectedException)) { NotificationProvider.PushSnackbarItem(MessageType.Error, "Remote desktop session terminated unexpectedly.", true, null, TimeSpan.FromSeconds(5)); } } #endregion #region Video Recording public VideoRecordingHandler CreateRecordingHandler() { return new VideoRecordingHandler(this); } #endregion #region Screenshot /// /// Gets a single remote desktop screen-shot. /// /// public async Task GetDesktopScreenShot() { var response = await _machineProvider.MachineOperator.SendGenericRequest(new GetScreenshotRequest(), new TransportRequestConfig() { Timeout = TimeSpan.FromSeconds(30) }); MemoryStream ms = new MemoryStream(response.Bitmap); ms.Position = 0; System.Drawing.Bitmap bitmap = new System.Drawing.Bitmap(ms); RasterFrame frame = new RasterFrame(bitmap); return frame; } #endregion } }