Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/DefaultRemoteDesktopService.cs


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
<// Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt)
// This code is distributed under the GNU LGPL (for details please see \doc\license.txt)

using System;
using System.Windows;
using System.Windows.Media.TextFormatting;

namespace Tango.Scripting.Editors.Rendering
{
	sealed class VisualLineTextParagraphProperties : TextParagraphProperties
	{
		internal TextRunProperties defaultTextRunProperties;
		internal TextWrapping textWrapping;
		internal double tabSize;
		internal double indent;
		internal bool firstLineInParagraph;
		
		public override double DefaultIncrementalTab {
			get { return tabSize; }
		}
		
		public override FlowDirection FlowDirection { get { return FlowDirection.LeftToRight; } }
		public override TextAlignment TextAlignment { get { return TextAlignment.Left; } }
		public override double LineHeight { get { return double.NaN; } }
		public override bool FirstLineInParagraph { get { return firstLineInParagraph; } }
		public override TextRunProperties DefaultTextRunProperties { get { return defaultTextRunProperties; } }
		public override TextWrapping TextWrapping { get { return textWrapping; } }
		public override TextMarkerProperties TextMarkerProperties { get { return null; } }
		public override double Indent { get { return indent; } }
	}
}
using Newtonsoft.Json;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Security.Authentication;
using System.Text;
using System.Threading.Tasks;
using System.Windows;
using System.Windows.Input;
using Tango.Core;
using Tango.Core.DI;
using Tango.Integration.ExternalBridge;
using Tango.Logging;
using Tango.PPC.Common.Application;
using Tango.PPC.Common.ExternalBridge;
using Tango.PPC.Common.OS;
using Tango.RemoteDesktop;
using Tango.RemoteDesktop.CaptureMethods;
using Tango.RemoteDesktop.Encoders;
using Tango.RemoteDesktop.Engines;
using Tango.RemoteDesktop.Frames;
using Tango.RemoteDesktop.Input;
using Tango.RemoteDesktop.Network;
using Tango.Settings;
using Tango.Transport;
using Tango.WebRTC;
using static Tango.RemoteDesktop.Input.MouseController;

namespace Tango.PPC.Common.RemoteDesktop
{
    [TangoCreateWhenRegistered]
    public class DefaultRemoteDesktopService : ExtendedObject, IRemoteDesktopService, IExternalBridgeRequestHandler
    {
        private RemoteDesktopPacket _initialPacket;
        private RasterScreenCaptureEngine _engine;
        private PPCSettings _settings;
        private List<RemoteDesktopClient> _clients;
        private JsonSerializerSettings _jsonSettings;
        private IOperationSystemManager _osManager;
        private IPPCApplicationManager _appManager;
        private bool _drawCursor;
        private bool _isMouseDown;
        private bool _ensureMouseDown;

        /// <summary>
        /// Gets or sets a value indicating whether this <see cref="IPPCService" /> is enabled.
        /// </summary>
        public bool Enabled { get; set; } = true;

        private bool _isStarted;
        /// <summary>
        /// Gets a value indicating whether the remote desktop service has started.
        /// </summary>
        public bool IsStarted
        {
            get { return _isStarted; }
            private set { _isStarted = value; RaisePropertyChangedAuto(); }
        }

        /// <summary>
        /// Gets a value indicating whether there is any active remote desktop session with a remote peer.
        /// </summary>
        public bool InSession
        {
            get
            {
                return _clients.Count > 0;
            }
        }

        /// <summary>
        /// Initializes a new instance of the <see cref="DefaultRemoteDesktopService"/> class.
        /// </summary>
        /// <param name="applicationManager">The application manager.</param>
        /// <param name="externalBridge">The external bridge.</param>
        /// <param name="osManager">The os manager.</param>
        public DefaultRemoteDesktopService(IPPCApplicationManager applicationManager, IPPCExternalBridgeService externalBridge, IOperationSystemManager osManager)
        {
            _osManager = osManager;
            _appManager = applicationManager;

            _jsonSettings = new JsonSerializerSettings()
            {
                TypeNameHandling = TypeNameHandling.All
            };

            _settings = SettingsManager.Default.GetOrCreate<PPCSettings>();
            Enabled = _settings.EnableRemoteDesktop;

            applicationManager.ApplicationReady += ApplicationManager_ApplicationReady;

            externalBridge.RegisterRequestHandler(this);

            _clients = new List<RemoteDesktopClient>();
            _engine = new RasterScreenCaptureEngine();

            _engine.CaptureCursor = false;

            _engine.FrameRate = Math.Min(Math.Max(_settings.RemoteDesktopFrameRate, 1), 20);
            _engine.FrameReceived += _engine_FrameReceived;
        }

        private void ApplicationManager_ApplicationReady(object sender, EventArgs e)
        {


            var mainWindow = System.Windows.Application.Current.MainWindow;

            if (mainWindow.WindowStyle != System.Windows.WindowStyle.None)
            {
                mainWindow.LocationChanged += (_, __) =>
                {
                    _engine.CaptureRegion = new CaptureRegion()
                    {
                        Left = (int)mainWindow.Left + 10,
                        Top = (int)mainWindow.Top + 5,
                        Width = (int)mainWindow.Width - 20,
                        Height = (int)mainWindow.Height - 15
                    };
                };

                _engine.CaptureRegion = new CaptureRegion()
                {
                    Left = (int)mainWindow.Left + 10,
                    Top = (int)mainWindow.Top + 5,
                    Width = (int)mainWindow.Width - 20,
                    Height = (int)mainWindow.Height - 15
                };
            }
            else
            {
                //DirectX capturing is not working on PPC !! Maybe when we replace ?
                //try
                //{
                //    _engine.CaptureMethod = new DirectXScreenCapture();
                //}
                //catch (Exception ex)
                //{
                //    LogManager.Log(ex, "Could not initialize DirectX screen capture method on this device. Falling back to GDI.");
                //}
            }

            mainWindow.PreviewMouseDown += (_, __) =>
            {
                _isMouseDown = true;
                _ensureMouseDown = true;
            };

            mainWindow.PreviewMouseUp += (_, __) =>
            {
                _isMouseDown = false;
            };

            _engine.Comparer.MaxDifferencesThrow = _engine.CaptureRegion.Width * _engine.CaptureRegion.Height / 2;
        }

        [ExternalBridgeRequestHandlerMethod(typeof(StartRemoteDesktopSessionRequest), RequestHandlerLoggingMode.LogRequestName)]
        public async Task OnStartRemoteDesktopSessionRequestReceived(StartRemoteDesktopSessionRequest request, String token, ExternalBridgeReceiver receiver)
        {
            this.ThrowIfDisabled();

            var client = _clients.SingleOrDefault(x => x.Receiver == receiver);

            if (client != null)
            {
                _clients.Remove(client);
            }

            RemoteDesktopClient newClient = new RemoteDesktopClient();
            newClient.Receiver = receiver;
            newClient.Token = token;
            newClient.WebRtcClient = new WebRtcClient();
            newClient.WebRtcClient.TextMessageReceived += WebRtcClient_TextMessageReceived;
            _clients.Add(newClient);

            await receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse()
            {
                FrameRate = _engine.FrameRate
            }, token);


            if (!_engine.IsStarted)
            {
                _engine.Start();
            }

            RaisePropertyChanged(nameof(InSession));
        }

        [ExternalBridgeRequestHandlerMethod(typeof(WebRtcIceCandidateRequest), RequestHandlerLoggingMode.LogRequestName)]
        public async Task OnWebRtcIceCandidateRequestReceived(WebRtcIceCandidateRequest request, String token, ExternalBridgeReceiver receiver)
        {
            var client = _clients.SingleOrDefault(x => x.Receiver == receiver);

            if (client != null)
            {
                try
                {
                    await receiver.SendGenericResponse(new WebRtcIceCandidateResponse() { }, token);
                    client.WebRtcClient.AddIceCandidate(request.IceCandidate);
                }
                catch (Exception ex)
                {
                    LogManager.Log(ex, "Error adding WebRTC ice candidate received from the remote connection.");
                }
            }
        }

        [ExternalBridgeRequestHandlerMethod(typeof(WebRtcOfferRequest), RequestHandlerLoggingMode.LogRequestName)]
        public async Task OnWebRtcOfferRequestReceived(WebRtcOfferRequest request, String token, ExternalBridgeReceiver receiver)
        {
            var client = _clients.SingleOrDefault(x => x.Receiver == receiver);

            if (client != null)
            {
                try
                {

                    try
                    {
                        client.WebRtcClient.NewIceCandidate += async (x, e) =>
                        {
                            try
                            {
                                await receiver.SendGenericRequest<WebRtcIceCandidateRequest, WebRtcIceCandidateResponse>(new WebRtcIceCandidateRequest() { IceCandidate = e.IceCandidate });
                            }
                            catch (Exception ex)
                            {
                                LogManager.Log(ex, "Error sending ice candidate to remote peer.");
                            }
                        };
                        client.WebRtcClient.Ready += (x, e) =>
                        {
                            client.IsWebRtcReady = true;
                        };
                        client.WebRtcClient.Disconnected += (x, e) =>
                        {
                            client.IsWebRtcReady = false;
                        };

                        client.WebRtcClient.FrameWidth = 800;
                        client.WebRtcClient.FrameHeight = 1280;
                        client.WebRtcClient.FrameRate = _engine.FrameRate;

                        await client.WebRtcClient.Init();

                        var answer = await client.WebRtcClient.CreateAnswer(request.Offer);
                        await receiver.SendGenericResponse(new WebRtcOfferResponse() { Answer = answer }, token);
                    }
                    catch (Exception ex)
                    {
                        throw LogManager.Log(ex, "Error initializing the WebRTC client.");
                    }
                }
                catch (Exception ex)
                {
                    throw LogManager.Log(ex, "Error responding to WebRTC offer request.");
                }
            }
        }

        [ExternalBridgeRequestHandlerMethod(typeof(StopRemoteDesktopSessionRequest), RequestHandlerLoggingMode.LogRequestName)]
        public async Task OnStopRemoteDesktopSessionRequestReceived(StopRemoteDesktopSessionRequest request, String token, ExternalBridgeReceiver receiver)
        {
            var client = _clients.SingleOrDefault(x => x.Receiver == receiver);

            if (client != null)
            {
                _clients.Remove(client);

                if (client.WebRtcClient != null)
                {
                    client.WebRtcClient.Dispose();
                }
            }

            if (_clients.Count == 0)
            {
                _engine.Stop();
            }

            await receiver.SendGenericResponse(new StopRemoteDesktopSessionResponse(), token);

            if (client != null)
            {
                await receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse(), client.Token, new TransportResponseConfig() { Completed = true });
            }

            RaisePropertyChanged(nameof(InSession));
        }

        [ExternalBridgeRequestHandlerMethod(typeof(MouseStateRequest))]
        public async Task OnMouseStateRequestReceived(MouseStateRequest request, String token, ExternalBridgeReceiver receiver)
        {
            MouseController.SetCursorPosition((int)request.Location.X, (int)request.Location.Y);

            if (request.EventType == MouseEventType.Up || request.EventType == MouseEventType.Down)
            {
                MouseEventFlags flag = MouseEventFlags.LeftUp;

                switch (request.EventType)
                {
                    case MouseEventType.Down:
                        flag = request.Button == MouseButton.Right ? MouseEventFlags.RightDown : MouseEventFlags.LeftDown;
                        break;
                    case MouseEventType.Up:
                        flag = request.Button == MouseButton.Right ? MouseEventFlags.RightUp : MouseEventFlags.LeftUp;
                        break;
                }

                MouseController.MouseEvent(flag);
            }
            else if (request.EventType == MouseEventType.DoubleClick)
            {
                MouseController.DoubleClick();
            }
            else if (request.EventType == MouseEventType.Scroll)
            {
                MouseController.Scroll(request.ScrollDelta);
            }

            if (receiver != null)
            {
                await receiver.SendGenericResponse(new MouseStateResponse(), token);
            }
        }

        [ExternalBridgeRequestHandlerMethod(typeof(TouchStateRequest))]
        public async Task OnTouchStateRequestReceived(TouchStateRequest request, String token, ExternalBridgeReceiver receiver)
        {
            switch (request.EventType)
            {
                case TouchEventType.TouchDown:
                    TouchController.TouchDown((int)request.Location.X, (int)request.Location.Y);
                    break;
                case TouchEventType.TouchMove:
                    TouchController.TouchMove(request.MoveDeltaX, request.MoveDeltaY);
                    break;
                case TouchEventType.TouchUp:
                    TouchController.TouchUp();
                    break;
            }

            if (receiver != null)
            {
                await receiver.SendGenericResponse(new TouchStateResponse(), token);
            }
        }

        [ExternalBridgeRequestHandlerMethod(typeof(KeyboardStateRequest))]
        public async Task OnKeyboardStateRequestReceived(KeyboardStateRequest request, String token, ExternalBridgeReceiver receiver)
        {
            if (request.EventType == KeyboardEventType.Down)
            {
                KeyboardController.KeyDown(request.Key, request.IsCtrlDown, request.IsShiftDown, request.IsAltDown);
            }
            else
            {
                KeyboardController.KeyUp(request.Key, request.IsCtrlDown, request.IsShiftDown, request.IsAltDown);
            }

            if (receiver != null)
            {
                await receiver.SendGenericResponse(new KeyboardStateResponse(), token);
            }
        }

        [ExternalBridgeRequestHandlerMethod(typeof(RemoteDesktopCommandRequest), RequestHandlerLoggingMode.LogRequestName)]
        public async Task OnRemoteDesktopCommandRequest(RemoteDesktopCommandRequest request, String token, ExternalBridgeReceiver receiver)
        {
            switch (request.Command)
            {
                case RemoteDesktopCommand.HideAndOpenShell:
                    _osManager.OpenShell();
                    _appManager.SetWindowState(System.Windows.WindowState.Minimized);
                    break;
            }

            await receiver.SendGenericResponse(new RemoteDesktopCommandResponse(), token);
        }

        [ExternalBridgeRequestHandlerMethod(typeof(SetCursorVisibilityRequest), RequestHandlerLoggingMode.LogRequestName)]
        public async Task OnSetCursorVisibilityRequest(SetCursorVisibilityRequest request, String token, ExternalBridgeReceiver receiver)
        {
            _drawCursor = request.Visible;
            await receiver.SendGenericResponse(new SetCursorVisibilityResponse(), token);
        }

        private async void _engine_FrameReceived(object sender, ScreenCaptureFrameReceivedEventArgs<RasterFrame> e)
        {
            if (_drawCursor)
            {
                e.Frame.DrawImage((_isMouseDown || _ensureMouseDown) ? Properties.Resources.tap : Properties.Resources.finger3, new System.Drawing.Point(System.Windows.Forms.Cursor.Position.X - 5, System.Windows.Forms.Cursor.Position.Y - 4));
                _ensureMouseDown = false;
            }

            _initialPacket = new RemoteDesktopPacket()
            {
                Bitmap = e.Frame.ToEncoder<PngEncoder>().ToArray(),
            };

            if (_clients.Count > 0)
            {
                bool useWebRTC = _clients.ToList().All(x => x.IsWebRtcReady);

                if (useWebRTC)
                {
                    _engine.EnableComparer = false;

                    foreach (var client in _clients.ToList())
                    {
                        try
                        {
                            client.WebRtcClient.PushFrame(e.Frame.ToBitmap());
                        }
                        catch (Exception ex)
                        {
                            LogManager.Log(ex, LogCategory.Debug, "Error pushing remote desktop frame via WebRTC channel.");
                        }
                    }

                    e.Frame.Dispose();
                }
                else
                {
                    _engine.EnableComparer = true;

                    foreach (var client in _clients.ToList().Where(x => !x.InitialPacketSent))
                    {
                        try
                        {
                            await client.Receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse()
                            {
                                Packet = _initialPacket,
                            }, client.Token, new TransportResponseConfig()
                            {
                                Immediate = false,
                            });

                            client.InitialPacketSent = true;
                        }
                        catch (Exception ex)
                        {
                            Debug.WriteLine(ex);
                        }
                    }

                    if (e.Frame.DifferenceCount > 0)
                    {
                        RemoteDesktopPacket packet = null;

                        Point mousePosition = new Point(System.Windows.Forms.Cursor.Position.X, System.Windows.Forms.Cursor.Position.Y);

                        if (!e.Frame.DifferenceAvailable)
                        {
                            packet = new RemoteDesktopPacket()
                            {
                                Bitmap = e.Frame.ToEncoder<TurboJpegEncoder>().ToArray(30),
                                MousePosition = mousePosition,
                                CursorVisible = _drawCursor
                            };
                        }
                        else
                        {
                            var diffFrame = e.Frame.ToDifference();
                            diffFrame = diffFrame.OptimizeBounds();

                            packet = new RemoteDesktopPacket()
                            {
                                Bitmap = diffFrame.ToEncoder<PngEncoder>().ToArray(),
                                IsPartial = true,
                                PartialRegion = new CaptureRegion(diffFrame.Left, diffFrame.Top, diffFrame.Width, diffFrame.Height),
                                MousePosition = mousePosition,
                                CursorVisible = _drawCursor
                            };

                            diffFrame.Dispose();
                        }

                        Debug.WriteLine($"Remote Desktop Bitmap Size: {packet.Bitmap.Length / 1000} kb");

                        foreach (var client in _clients.ToList().Where(x => x.InitialPacketSent))
                        {
                            try
                            {
                                await client.Receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse()
                                {
                                    Packet = packet
                                }, client.Token, new TransportResponseConfig()
                                {
                                    Immediate = false,
                                });
                            }
                            catch (Exception ex)
                            {
                                Debug.WriteLine(ex);
                            }
                        }
                    }
                }
            }

            e.Frame.Dispose();
        }

        private async void WebRtcClient_TextMessageReceived(object sender, DataMessageReceivedEventArgs<string> e)
        {
            try
            {
                var request = JsonConvert.DeserializeObject(e.Data, _jsonSettings);

                if (request.GetType() == typeof(MouseStateRequest))
                {
                    await OnMouseStateRequestReceived(request as MouseStateRequest, null, null);
                }
                else if (request.GetType() == typeof(KeyboardStateRequest))
                {
                    await OnKeyboardStateRequestReceived(request as KeyboardStateRequest, null, null);
                }
                else if (request.GetType() == typeof(TouchStateRequest))
                {
                    await OnTouchStateRequestReceived(request as TouchStateRequest, null, null);
                }
            }
            catch (Exception ex)
            {
                LogManager.Log(ex, "Error deserializing incoming message on the WebRTC data Channel.");
            }
        }

        public void OnReceiverDisconnected(ExternalBridgeReceiver receiver)
        {
            var client = _clients.SingleOrDefault(x => x.Receiver == receiver);

            if (client != null)
            {
                LogManager.Log("Remote desktop client disconnected. Disposing WebRTC channel...");

                _clients.Remove(client);

                try
                {
                    if (client.WebRtcClient != null)
                    {
                        client.WebRtcClient.Dispose();
                    }
                }
                catch (Exception ex)
                {
                    LogManager.Log(ex, "Error disposing the WebRTC channel.");
                }
            }

            if (_clients.Count == 0)
            {
                _engine.Stop();
            }

            RaisePropertyChanged(nameof(InSession));
        }
    }
}