path: root/Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Input;
using System.Windows.Media.Imaging;
using Tango.Core;
using Tango.Core.DI;
using Tango.Core.Threading;
using Tango.FSE.Common.Connection;
using Tango.FSE.Common.Notifications;
using Tango.FSE.Common.RemoteDesktop;
using Tango.PMR;
using Tango.PMR.Integration;
using Tango.RemoteDesktop.Frames;
using Tango.RemoteDesktop.Network;
using Tango.Transport;
using Tango.WebRTC;

namespace Tango.FSE.UI.RemoteDesktop
{
    public class DefaultRemoteDesktopProvider : ExtendedObject, IRemoteDesktopProvider
    {
        private class MouseMovement
        {
            public Point Location { get; set; }
            public Size ViewSize { get; set; }
        }

        private IMachineProvider _machineProvider;
        private RasterFrame _currentFrame;
        private MemoryStream _currentStream;
        private Size? _frameSize;
        private IntervalMessageDispatcher<StartRemoteDesktopSessionResponse> _frameDispatcher;
        private Thread _mouseMoveThread;
        private ProducerConsumerQueue<MouseMovement> _mouseMovements;
        private WebRtcClient _webRtcClient;
        private List<IceCandidate> _iceCandidates;
        private bool _answerReceived;

        #region Events

        public event EventHandler<DesktopFrameReceivedEventArgs> FrameReceived;

        #endregion

        #region Properties

        [TangoInject]
        private INotificationProvider NotificationProvider { get; set; }

        private bool _inSession;
        public bool InSession
        {
            get { return _inSession; }
            set { _inSession = value; RaisePropertyChangedAuto(); RaisePropertyChanged(nameof(CanStartSession)); }
        }

        public bool EnableWebRtc { get; set; }

        private bool _isWebRtcActive;
        public bool IsWebRtcActive
        {
            get { return _isWebRtcActive; }
            set
            {
                if (_isWebRtcActive != value)
                {
                    _isWebRtcActive = value;
                    RaisePropertyChangedAuto();

                    if (_isWebRtcActive)
                    {
                        LogManager.Log("WebRTC is now active.");
                    }
                }
            }
        }

        public TimeSpan MouseMoveInterval { get; set; }

        public bool CanStartSession
        {
            get { return _machineProvider.IsConnected && !InSession; }
        }

        #endregion

        #region Constructors

        public DefaultRemoteDesktopProvider()
        {
            _iceCandidates = new List<IceCandidate>();
            MouseMoveInterval = TimeSpan.FromMilliseconds(100);
            EnableWebRtc = true;
        }

        public DefaultRemoteDesktopProvider(IMachineProvider machineProvider) : this()
        {
            _machineProvider = machineProvider;
            _machineProvider.MachineConnected += (_, __) =>
            {
                InSession = false;
            };
            _machineProvider.MachineDisconnected += (_, __) =>
            {
                InSession = false;
                OnFrameReceived(null);
            };
            //_machineProvider.MachineOperator.RequestReceived += MachineOperator_RequestReceived;
            _machineProvider.MachineOperator.RegisterRequestHandler<WebRtcIceCandidateRequest>(OnIceCandidateRequestReceived);
        }

        private async void OnIceCandidateRequestReceived(WebRtcIceCandidateRequest request, string token)
        {
            LogManager.Log("Ice candidate request received from the remote peer.");
            await _machineProvider.MachineOperator.SendGenericResponse(new WebRtcIceCandidateResponse() { }, token);

            LogManager.Log("Adding ice candidate...");
            _webRtcClient.AddIceCandidate(request.IceCandidate);
            LogManager.Log("Ice candidate added.");
        }

        #endregion

        //private async void MachineOperator_RequestReceived(object sender, PMR.Common.MessageContainer container)
        //{
        //    if (container.Type == PMR.Common.MessageType.GenericRequest)
        //    {
        //        var message = MessageFactory.ExtractMessageFromContainer(container);
        //        var type = Type.GetType((message as GenericRequest).Type);

        //        if (type == typeof(WebRtcIceCandidateRequest))
        //        {

        //        }
        //    }
        //}

        #region Start/Stop Session

        public async Task StartSession()
        {
            TaskCompletionSource<bool> completionSource = new TaskCompletionSource<bool>();

            if (!CanStartSession)
            {
                throw new InvalidOperationException("Unable to start a remote desktop session at the moment.");
            }

            if (!InSession)
            {
                LogManager.Log("Starting remote desktop session...");

                _iceCandidates.Clear();
                _answerReceived = false;

                InSession = true;

                var taskItem = NotificationProvider.PushTaskItem("Starting remote desktop session...");

                await Task.Delay(2000);

                if (_frameDispatcher != null)
                {
                    _frameDispatcher.Dispose();
                }

                _frameDispatcher = new IntervalMessageDispatcher<StartRemoteDesktopSessionResponse>(OnRemoteDesktopResponse);
                _frameDispatcher.Start();

                bool taskCompleted = false;

                _machineProvider.MachineOperator.SendGenericContinuousRequest<StartRemoteDesktopSessionRequest, StartRemoteDesktopSessionResponse>(new StartRemoteDesktopSessionRequest() { }, new TransportContinuousRequestConfig()
                {
                    Timeout = TimeSpan.FromSeconds(10),
                    ShouldLog = true
                }).Subscribe((response) =>
                {
                    if (!taskCompleted)
                    {
                        taskCompleted = true;
                        taskItem.Dispose();
                        completionSource.SetResult(true);

                        if (EnableWebRtc)
                        {
                            LogManager.Log("WebRTC enabled. Starting WebRTC...");
                            StartWebRTC();
                        }
                    }

                    _frameDispatcher.Push(response);

                }, (ex) =>
                {
                    if (!taskCompleted)
                    {
                        taskCompleted = true;
                        taskItem.Dispose();
                        completionSource.SetException(ex);
                    }
                    else if (InSession)
                    {
                        OnFailed(ex);
                    }

                    InSession = false;
                }, () =>
                {
                    InSession = false;
                    OnFrameReceived(null);
                });

                await completionSource.Task;
            }
            else
            {
                await Task.FromResult(true);
            }
        }

        public async Task EndSession()
        {
            if (InSession)
            {
                using (NotificationProvider.PushTaskItem("Stopping remote desktop session..."))
                {
                    LogManager.Log("Stopping remote desktop session...");
                    await Task.Delay(2000);

                    try
                    {
                        await _machineProvider.MachineOperator.SendGenericRequest<StopRemoteDesktopSessionRequest, StopRemoteDesktopSessionResponse>(new StopRemoteDesktopSessionRequest(), new TransportRequestConfig()
                        {
                            ShouldLog = true
                        });

                        LogManager.Log("Remote desktop session stopped.");
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log(ex, "Error stopping the remote desktop session.");
                    }

                    DisposeWebRtc();

                    InSession = false;
                }
            }
        }

        #endregion

        #region Virtual Methods

        protected virtual void OnFrameReceived(BitmapSource source, DesktopFrameReceivedEventArgs.FrameOrigin origin = DesktopFrameReceivedEventArgs.FrameOrigin.WebSockets)
        {
            IsWebRtcActive = origin == DesktopFrameReceivedEventArgs.FrameOrigin.WebRTC;

            if (InSession)
            {
                FrameReceived?.Invoke(this, new DesktopFrameReceivedEventArgs()
                {
                    Source = source,
                    Origin = origin
                });
            }
        }

        #endregion

        #region SignalR Response Received

        private void OnRemoteDesktopResponse(StartRemoteDesktopSessionResponse response)
        {
            if (!InSession) return; //To stop frame delivery immediately.

            if (response.Packet == null)
            {
                _frameDispatcher.Interval = 1000 / response.FrameRate;
                return; //This is the first message with empty bitmap and only the frame rate.
            }

            try
            {
                if (!response.Packet.IsPartial)
                {
                    _currentFrame?.Dispose();
                    _currentStream?.Dispose();

                    _currentStream = new MemoryStream(response.Packet.Bitmap);
                    _currentFrame = new RasterFrame(new System.Drawing.Bitmap(_currentStream));
                    _frameSize = new Size(_currentFrame.Width, _currentFrame.Height);
                }
                else
                {
                    using (MemoryStream ms = new MemoryStream(response.Packet.Bitmap))
                    {
                        var diffFrame = new RasterFrame(new System.Drawing.Bitmap(ms), response.Packet.PartialRegion.Left, response.Packet.PartialRegion.Top);
                        diffFrame.Apply(_currentFrame);
                        diffFrame.Dispose();
                    }
                }

                OnFrameReceived(_currentFrame.ToBitmapSource());
            }
            catch (Exception ex)
            {
                LogManager.Log(ex, "Error on remote desktop packet received.");
            }
        }

        #endregion

        #region Mouse/Keyboard

        public async void MouseDown(MouseButton button, Point location, Size viewSize)
        {
            if (!InSession || _frameSize == null) return;

            await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(new MouseStateRequest()
            {
                Button = button,
                EventType = MouseEventType.Down,
                Location = TranslateLocation(location, viewSize)
            });
        }

        public async void MouseUp(MouseButton button, Point location, Size viewSize)
        {
            if (!InSession || _frameSize == null) return;

            await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(new MouseStateRequest()
            {
                Button = button,
                EventType = MouseEventType.Up,
                Location = TranslateLocation(location, viewSize)
            });
        }

        public void MouseMove(Point location, Size viewSize)
        {
            if (!InSession || _frameSize == null) return;

            if (_mouseMoveThread == null)
            {
                _mouseMovements = new ProducerConsumerQueue<MouseMovement>();
                _mouseMoveThread = new Thread(async () =>
                {
                    while (InSession)
                    {
                        MouseMovement movement = _mouseMovements.BlockDequeue();

                        while (_mouseMovements.Count > 0)
                        {
                            movement = _mouseMovements.BlockDequeue();
                        }

                        try
                        {
                            await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(new MouseStateRequest()
                            {
                                Button = MouseButton.Left,
                                EventType = MouseEventType.Move,
                                Location = TranslateLocation(movement.Location, movement.ViewSize)
                            });
                        }
                        catch (Exception ex)
                        {
                            Debug.WriteLine($"Error on mouse move\n{ex.ToString()}");
                        }

                        Thread.Sleep(MouseMoveInterval);
                    }

                    _mouseMoveThread = null;
                });

                _mouseMoveThread.IsBackground = true;
                _mouseMoveThread.Start();
            }

            _mouseMovements.BlockEnqueue(new MouseMovement()
            {
                Location = location,
                ViewSize = viewSize,
            });
        }

        public async void MouseDoubleClick(MouseButton button, Point location, Size viewSize)
        {
            if (!InSession || _frameSize == null) return;

            await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(new MouseStateRequest()
            {
                Button = button,
                EventType = MouseEventType.DoubleClick,
                Location = TranslateLocation(location, viewSize)
            });
        }

        private Point TranslateLocation(Point location, Size viewSize)
        {
            return new Point(
                (location.X / viewSize.Width) * _frameSize.Value.Width,
                (location.Y / viewSize.Height) * _frameSize.Value.Height);
        }

        public async void KeyboardDown(Key key, bool ctrlDown, bool shitDown, bool altDown)
        {
            if (!InSession || _frameSize == null) return;

            await _machineProvider.MachineOperator.SendGenericRequest<KeyboardStateRequest, KeyboardStateResponse>(new KeyboardStateRequest()
            {
                EventType = KeyboardEventType.Down,
                Key = key,
                IsCtrlDown = ctrlDown,
                IsShiftDown = shitDown,
                IsAltDown = altDown,
            });
        }

        public async void KeyboardUp(Key key, bool ctrlDown, bool shitDown, bool altDown)
        {
            if (!InSession || _frameSize == null) return;

            await _machineProvider.MachineOperator.SendGenericRequest<KeyboardStateRequest, KeyboardStateResponse>(new KeyboardStateRequest()
            {
                EventType = KeyboardEventType.Up,
                Key = key,
                IsCtrlDown = ctrlDown,
                IsShiftDown = shitDown,
                IsAltDown = altDown,
            });
        }

        #endregion

        #region WebRTC

        private async void StartWebRTC()
        {
            _webRtcClient = new WebRtcClient();

            try
            {
                _webRtcClient.NewIceCandidate += _webRtcClient_NewIceCandidate;
                _webRtcClient.FrameReceived += _webRtcClient_FrameReceived;
                _webRtcClient.Disconnected += _webRtcClient_Disconnected;

                LogManager.Log("Initializing WebRTC client...");
                await _webRtcClient.Init();

                LogManager.Log("Creating WebRTC offer...");
                var offer = await _webRtcClient.CreateOffer();

                LogManager.Log("Sending WebRTC offer...");
                var response = await _machineProvider.MachineOperator.SendGenericRequest<WebRtcOfferRequest, WebRtcOfferResponse>(new WebRtcOfferRequest() { Offer = offer }, new TransportRequestConfig()
                {
                    Timeout = TimeSpan.FromSeconds(30),
                    ShouldLog = true
                });

                LogManager.Log("WebRTC offer sent and responded with an answer. Setting WebRTC answer...");
                _webRtcClient.SetAnswer(response.Answer);
                _answerReceived = true;

                foreach (var ice in _iceCandidates.ToList())
                {
                    LogManager.Log("Sending existing ice candidate...");

                    await _machineProvider.MachineOperator.SendGenericRequest<WebRtcIceCandidateRequest, WebRtcIceCandidateResponse>(new WebRtcIceCandidateRequest() { IceCandidate = ice }, new TransportRequestConfig()
                    {
                        Timeout = TimeSpan.FromSeconds(30),
                        ShouldLog = true
                    });
                }
            }
            catch (Exception ex)
            {
                LogManager.Log(ex, "Error starting WebRTC.");
            }
        }

        private void DisposeWebRtc()
        {
            if (_webRtcClient != null)
            {
                LogManager.Log("Disposing WebRTC client...");

                _webRtcClient.NewIceCandidate -= _webRtcClient_NewIceCandidate;
                _webRtcClient.FrameReceived -= _webRtcClient_FrameReceived;
                _webRtcClient.Disconnected -= _webRtcClient_Disconnected;

                if (_webRtcClient != null)
                {
                    try
                    {
                        _webRtcClient.Dispose();
                        _webRtcClient = null;
                        LogManager.Log("WebRTC client disposed.");
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log(ex, "Error disposing WebRTC client.");
                    }
                }
            }
        }

        private void _webRtcClient_Disconnected(object sender, EventArgs e)
        {
            LogManager.Log("WebRTC client is now disconnected.");
            IsWebRtcActive = false;
        }

        private async void _webRtcClient_NewIceCandidate(object sender, NewIceCandidateEventArgs e)
        {
            try
            {
                if (_answerReceived)
                {
                    LogManager.Log("New WebRTC candidate available. Sending ice to remote peer...");

                    await _machineProvider.MachineOperator.SendGenericRequest<WebRtcIceCandidateRequest, WebRtcIceCandidateResponse>(new WebRtcIceCandidateRequest() { IceCandidate = e.IceCandidate }, new TransportRequestConfig()
                    {
                        Timeout = TimeSpan.FromSeconds(30),
                        ShouldLog = true
                    });
                }
                else
                {
                    LogManager.Log("New WebRTC candidate available. Will be sent after an answer is received...");

                    _iceCandidates.Add(e.IceCandidate);
                }
            }
            catch (Exception ex)
            {
                LogManager.Log(ex, "Error sending ice candidate to remote peer.");
            }
        }

        private void _webRtcClient_FrameReceived(object sender, VideoFrameReceivedEventArgs e)
        {
            try
            {
                RasterFrame frame = new RasterFrame(e.Bitmap);
                OnFrameReceived(frame.ToBitmapSource(), DesktopFrameReceivedEventArgs.FrameOrigin.WebRTC);
                frame.Dispose();
            }
            catch (Exception ex)
            {
                LogManager.Log(ex, "Error on WebRTC client frame received.");
            }
        }

        #endregion

        #region OnFailed

        private void OnFailed(Exception exception)
        {
            LogManager.Log(exception, "Remote desktop session failed.");

            InSession = false;
            OnFrameReceived(null);
            DisposeWebRtc();

            if (!(exception is TransporterDisconnectedException))
            {
                NotificationProvider.PushSnackbarItem(MessageType.Error, "Remote desktop session terminated unexpectedly.", true, null, TimeSpan.FromSeconds(5));
            }
        }

        #endregion
    }
}