From f17d39f37cac50861467e07a7bee40534d20100a Mon Sep 17 00:00:00 2001 From: Roy Ben Shabat Date: Wed, 4 Mar 2020 21:32:42 +0200 Subject: Improved "Notify Continuous Requests About Disconnection". Integrated FSE/PPC Remote Desktop. Implemented RemoteDesktopService / RemoteDesktopProvider. Implemented Mouse/Keyboard gestures. --- .../ViewModels/RemoteDesktopViewVM.cs | 114 +++---- .../Views/RemoteDesktopView.xaml | 6 +- .../Views/RemoteDesktopView.xaml.cs | 42 +++ .../FSE/Tango.FSE.Common/FSEViewModel.cs | 7 + .../Tango.FSE.Common/Notifications/SnackbarItem.cs | 4 +- .../RemoteDesktop/DesktopFrameReceivedEventArgs.cs | 14 + .../RemoteDesktop/IRemoteDesktopProvider.cs | 27 ++ .../FSE/Tango.FSE.Common/Tango.FSE.Common.csproj | 8 +- .../Notifications/DefaultNotificationProvider.cs | 28 +- .../RemoteDesktop/DefaultRemoteDesktopProvider.cs | 341 +++++++++++++++++++++ .../FSE/Tango.FSE.UI/Tango.FSE.UI.csproj | 27 +- .../FSE/Tango.FSE.UI/ViewModelLocator.cs | 5 +- .../FSE/Tango.FSE.UI/Views/MainView.xaml | 4 +- .../Visual_Studio/FSE/Tango.FSE.UI/packages.config | 6 + .../RemoteDesktop/DefaultRemoteDesktopService.cs | 124 ++++++-- .../RemoteDesktop/IRemoteDesktopService.cs | 2 - .../Visual_Studio/Tango.Core/Tango.Core.csproj | 4 +- .../Threading/IntervalMessageDispatcher.cs | 119 +++++++ .../Tango.Core/Threading/SequencerThread.cs | 71 ----- .../ExternalBridge/ExternalBridgeTcpClient.cs | 6 +- .../Input/KeyboardController.cs | 79 +++++ .../Tango.RemoteDesktop/Input/MouseController.cs | 80 +++++ .../Network/KeyboardEventType.cs | 14 + .../Network/KeyboardStateRequest.cs | 18 ++ .../Network/KeyboardStateResponse.cs | 12 + .../Tango.RemoteDesktop/Network/MouseEventType.cs | 16 + .../Network/MouseStateRequest.cs | 17 + .../Network/MouseStateResponse.cs | 12 + .../Network/StopRemoteDesktopSessionRequest.cs | 12 + .../Network/StopRemoteDesktopSessionResponse.cs | 12 + .../Tango.RemoteDesktop/ScreenCaptureEngine.cs | 6 +- .../Tango.RemoteDesktop/Tango.RemoteDesktop.csproj | 13 + .../Tango.RemoteDesktop/packages.config | 1 + .../Tango.Transport/TransporterBase.cs | 17 +- 34 files changed, 1085 insertions(+), 183 deletions(-) create mode 100644 Software/Visual_Studio/FSE/Tango.FSE.Common/RemoteDesktop/DesktopFrameReceivedEventArgs.cs create mode 100644 Software/Visual_Studio/FSE/Tango.FSE.Common/RemoteDesktop/IRemoteDesktopProvider.cs create mode 100644 Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs create mode 100644 Software/Visual_Studio/Tango.Core/Threading/IntervalMessageDispatcher.cs delete mode 100644 Software/Visual_Studio/Tango.Core/Threading/SequencerThread.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Input/KeyboardController.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Input/MouseController.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardEventType.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardStateRequest.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardStateResponse.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseEventType.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseStateRequest.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseStateResponse.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Network/StopRemoteDesktopSessionRequest.cs create mode 100644 Software/Visual_Studio/Tango.RemoteDesktop/Network/StopRemoteDesktopSessionResponse.cs (limited to 'Software/Visual_Studio') diff --git a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/ViewModels/RemoteDesktopViewVM.cs b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/ViewModels/RemoteDesktopViewVM.cs index eb5d57447..c8365af3b 100644 --- a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/ViewModels/RemoteDesktopViewVM.cs +++ b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/ViewModels/RemoteDesktopViewVM.cs @@ -6,10 +6,13 @@ using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; +using System.Windows; +using System.Windows.Input; using System.Windows.Media.Imaging; using Tango.Core.Commands; using Tango.Core.Threading; using Tango.FSE.Common; +using Tango.FSE.Common.RemoteDesktop; using Tango.RemoteDesktop.Frames; using Tango.RemoteDesktop.Network; @@ -17,8 +20,6 @@ namespace Tango.FSE.PPCConsole.ViewModels { public class RemoteDesktopViewVM : FSEViewModel { - private RasterFrame _currentFrame; - private BitmapSource _source; public BitmapSource Source { @@ -31,7 +32,7 @@ namespace Tango.FSE.PPCConsole.ViewModels public RemoteDesktopViewVM() { - StartCommand = new RelayCommand(StartRemoteDesktop, () => MachineProvider.IsConnected); + StartCommand = new RelayCommand(StartRemoteDesktop); StopCommand = new RelayCommand(StopRemoteDesktop); } @@ -39,71 +40,70 @@ namespace Tango.FSE.PPCConsole.ViewModels { base.OnApplicationStarted(); - MachineProvider.MachineConnected += (_, __) => InvalidateRelayCommands(); - MachineProvider.MachineDisconnected += (_, __) => InvalidateRelayCommands(); + RemoteDesktopProvider.FrameReceived += RemoteDesktopProvider_FrameReceived; } - private void StartRemoteDesktop() + private async void StartRemoteDesktop() { - SequencerThread sequencer = null; - sequencer = new SequencerThread((response) => - { - if (response.Packet == null) - { - sequencer.FrameRate = 1000 / response.FrameRate; - return; //Returned just to notice that there was no timeout.. - } - - try - { - if (!response.Packet.IsPartial) - { - using (MemoryStream ms = new MemoryStream(response.Packet.Bitmap)) - { - if (_currentFrame != null) - { - _currentFrame.Dispose(); - } - - _currentFrame = new RasterFrame(new Bitmap(ms)); - } - } - else - { - using (MemoryStream ms = new MemoryStream(response.Packet.Bitmap)) - { - var diffFrame = new RasterFrame(new Bitmap(ms), response.Packet.PartialRegion.Left, response.Packet.PartialRegion.Top); - diffFrame.Apply(_currentFrame); - diffFrame.Dispose(); - } - } - - Source = _currentFrame.ToBitmapSource(); - } - catch (Exception ex) - { - Debug.WriteLine($"Error on remote desktop packet received: {ex.Message}"); - } - }); - - sequencer.Start(); - - MachineProvider.MachineOperator.SendGenericContinuousRequest(new StartRemoteDesktopSessionRequest() { }, new Transport.TransportContinuousRequestConfig() + try { - ContinuousTimeout = TimeSpan.FromSeconds(30), - }).Subscribe((response) => + await RemoteDesktopProvider.StartSession(); + } + catch (Exception ex) { - sequencer.Push(response); + await NotificationProvider.ShowError($"Error starting remote desktop session.\n{ex.FlattenMessage()}"); + } + } - }, (ex) => + private async void StopRemoteDesktop() + { + try + { + await RemoteDesktopProvider.EndSession(); + } + catch (Exception ex) { - Debug.WriteLine(ex); - }, () => { }); + await NotificationProvider.ShowError($"Error stopping remote desktop session.\n{ex.FlattenMessage()}"); + } + } + + private void RemoteDesktopProvider_FrameReceived(object sender, DesktopFrameReceivedEventArgs e) + { + Source = e.Source; + } + + #region Mouse & Keyboard Handlers From View + + public void OnMouseDown(MouseButton changedButton, System.Windows.Point point, System.Windows.Size size) + { + RemoteDesktopProvider.MouseDown(changedButton, point, size); } - private void StopRemoteDesktop() + public void OnMouseUp(MouseButton changedButton, System.Windows.Point point, System.Windows.Size size) { + RemoteDesktopProvider.MouseUp(changedButton, point, size); + } + + public void OnMouseMove(System.Windows.Point point, System.Windows.Size size) + { + //RemoteDesktopProvider.MouseMove(point, size); + } + + public void OnMouseDoubleClick(MouseButton changedButton, System.Windows.Point point, System.Windows.Size size) + { + RemoteDesktopProvider.MouseDoubleClick(changedButton, point, size); + } + + public void OnKeyboardDown(Key key, bool ctrlDown, bool shitDown, bool altDown) + { + throw new NotImplementedException(); + } + public void OnKeyboardUp(Key key, bool ctrlDown, bool shitDown, bool altDown) + { + throw new NotImplementedException(); } + + #endregion } } diff --git a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml index b6a4eec62..9d3308741 100644 --- a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml +++ b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml @@ -19,8 +19,8 @@ - - + + @@ -42,7 +42,7 @@ - + diff --git a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml.cs b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml.cs index 27be82c5d..be7c280c8 100644 --- a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml.cs +++ b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml.cs @@ -12,6 +12,7 @@ using System.Windows.Media; using System.Windows.Media.Imaging; using System.Windows.Navigation; using System.Windows.Shapes; +using Tango.FSE.PPCConsole.ViewModels; namespace Tango.FSE.PPCConsole.Views { @@ -20,9 +21,50 @@ namespace Tango.FSE.PPCConsole.Views /// public partial class RemoteDesktopView : UserControl { + private RemoteDesktopViewVM _vm; + public RemoteDesktopView() { InitializeComponent(); + Loaded += (_, __) => _vm = DataContext as RemoteDesktopViewVM; + + img.PreviewMouseDown += Img_PreviewMouseDown; + img.PreviewMouseUp += Img_PreviewMouseUp; + img.PreviewMouseMove += Img_PreviewMouseMove; + img.PreviewKeyDown += Img_PreviewKeyDown; + img.PreviewKeyUp += Img_PreviewKeyUp; + } + + private void Img_PreviewKeyDown(object sender, KeyEventArgs e) + { + _vm.OnKeyboardDown(e.Key, Keyboard.IsKeyDown(Key.LeftCtrl), Keyboard.IsKeyDown(Key.LeftShift), Keyboard.IsKeyDown(Key.LeftAlt)); + } + + private void Img_PreviewKeyUp(object sender, KeyEventArgs e) + { + _vm.OnKeyboardUp(e.Key, Keyboard.IsKeyDown(Key.LeftCtrl), Keyboard.IsKeyDown(Key.LeftShift), Keyboard.IsKeyDown(Key.LeftAlt)); + } + + private void Img_PreviewMouseDown(object sender, MouseButtonEventArgs e) + { + if (e.ClickCount == 2) + { + _vm.OnMouseDoubleClick(e.ChangedButton, e.GetPosition(img), new Size(img.ActualWidth, img.ActualHeight)); + } + else + { + _vm.OnMouseDown(e.ChangedButton, e.GetPosition(img), new Size(img.ActualWidth, img.ActualHeight)); + } + } + + private void Img_PreviewMouseUp(object sender, MouseButtonEventArgs e) + { + _vm.OnMouseUp(e.ChangedButton, e.GetPosition(img), new Size(img.ActualWidth, img.ActualHeight)); + } + + private void Img_PreviewMouseMove(object sender, MouseEventArgs e) + { + _vm.OnMouseMove(e.GetPosition(img), new Size(img.ActualWidth, img.ActualHeight)); } } } diff --git a/Software/Visual_Studio/FSE/Tango.FSE.Common/FSEViewModel.cs b/Software/Visual_Studio/FSE/Tango.FSE.Common/FSEViewModel.cs index c0abe6dab..bde374597 100644 --- a/Software/Visual_Studio/FSE/Tango.FSE.Common/FSEViewModel.cs +++ b/Software/Visual_Studio/FSE/Tango.FSE.Common/FSEViewModel.cs @@ -21,6 +21,7 @@ using Tango.FSE.Common.FSEApplication; using Tango.FSE.Common.Gateway; using Tango.FSE.Common.Navigation; using Tango.FSE.Common.Notifications; +using Tango.FSE.Common.RemoteDesktop; using Tango.Settings; using Tango.SharedUI; using static Tango.SharedUI.Controls.NavigationControl; @@ -83,6 +84,12 @@ namespace Tango.FSE.Common [TangoInject] public IDiagnosticsProvider DiagnosticsProvider { get; set; } + /// + /// Gets or sets the remote desktop provider. + /// + [TangoInject] + public IRemoteDesktopProvider RemoteDesktopProvider { get; set; } + /// /// Gets or sets the FSE service. /// diff --git a/Software/Visual_Studio/FSE/Tango.FSE.Common/Notifications/SnackbarItem.cs b/Software/Visual_Studio/FSE/Tango.FSE.Common/Notifications/SnackbarItem.cs index 832b69d92..6246d0ce1 100644 --- a/Software/Visual_Studio/FSE/Tango.FSE.Common/Notifications/SnackbarItem.cs +++ b/Software/Visual_Studio/FSE/Tango.FSE.Common/Notifications/SnackbarItem.cs @@ -79,9 +79,9 @@ namespace Tango.FSE.Common.Notifications Task.Delay(TimeSpan.FromSeconds(2)).ContinueWith((y) => { Close(); - }, TaskScheduler.FromCurrentSynchronizationContext()); + }); } - }, TaskScheduler.FromCurrentSynchronizationContext()); + }); } } diff --git a/Software/Visual_Studio/FSE/Tango.FSE.Common/RemoteDesktop/DesktopFrameReceivedEventArgs.cs b/Software/Visual_Studio/FSE/Tango.FSE.Common/RemoteDesktop/DesktopFrameReceivedEventArgs.cs new file mode 100644 index 000000000..1589c2b99 --- /dev/null +++ b/Software/Visual_Studio/FSE/Tango.FSE.Common/RemoteDesktop/DesktopFrameReceivedEventArgs.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Media.Imaging; + +namespace Tango.FSE.Common.RemoteDesktop +{ + public class DesktopFrameReceivedEventArgs : EventArgs + { + public BitmapSource Source { get; set; } + } +} diff --git a/Software/Visual_Studio/FSE/Tango.FSE.Common/RemoteDesktop/IRemoteDesktopProvider.cs b/Software/Visual_Studio/FSE/Tango.FSE.Common/RemoteDesktop/IRemoteDesktopProvider.cs new file mode 100644 index 000000000..716739d0a --- /dev/null +++ b/Software/Visual_Studio/FSE/Tango.FSE.Common/RemoteDesktop/IRemoteDesktopProvider.cs @@ -0,0 +1,27 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Input; +using System.Windows.Media.Imaging; + +namespace Tango.FSE.Common.RemoteDesktop +{ + public interface IRemoteDesktopProvider + { + event EventHandler FrameReceived; + bool InSession { get; } + bool CanStartSession { get; } + TimeSpan MouseMoveInterval { get; set; } + Task StartSession(); + Task EndSession(); + void MouseDown(MouseButton button, Point location, Size viewSize); + void MouseUp(MouseButton button, Point location, Size viewSize); + void MouseMove(Point location, Size viewSize); + void MouseDoubleClick(MouseButton button, Point location, Size viewSize); + void KeyboardDown(Key key, bool ctrlDown, bool shitDown, bool altDown); + void KeyboardUp(Key key, bool ctrlDown, bool shitDown, bool altDown); + } +} diff --git a/Software/Visual_Studio/FSE/Tango.FSE.Common/Tango.FSE.Common.csproj b/Software/Visual_Studio/FSE/Tango.FSE.Common/Tango.FSE.Common.csproj index 664643f95..6e219c67a 100644 --- a/Software/Visual_Studio/FSE/Tango.FSE.Common/Tango.FSE.Common.csproj +++ b/Software/Visual_Studio/FSE/Tango.FSE.Common/Tango.FSE.Common.csproj @@ -62,6 +62,8 @@ + + ..\..\packages\ControlzEx.3.0.2.4\lib\net45\System.Windows.Interactivity.dll @@ -116,6 +118,8 @@ + + @@ -183,7 +187,9 @@ ResXFileCodeGenerator Resources.Designer.cs - + + Designer + diff --git a/Software/Visual_Studio/FSE/Tango.FSE.UI/Notifications/DefaultNotificationProvider.cs b/Software/Visual_Studio/FSE/Tango.FSE.UI/Notifications/DefaultNotificationProvider.cs index f76e8751d..40d2e58e7 100644 --- a/Software/Visual_Studio/FSE/Tango.FSE.UI/Notifications/DefaultNotificationProvider.cs +++ b/Software/Visual_Studio/FSE/Tango.FSE.UI/Notifications/DefaultNotificationProvider.cs @@ -542,8 +542,11 @@ namespace Tango.FSE.UI.Notifications /// public TaskItem PushTaskItem(TaskItem taskBarItem) { - TaskItems.Add(taskBarItem); - RaisePropertyChanged(nameof(HasTaskItems)); + InvokeUI(() => + { + TaskItems.Add(taskBarItem); + RaisePropertyChanged(nameof(HasTaskItems)); + }); return taskBarItem; } @@ -573,8 +576,11 @@ namespace Tango.FSE.UI.Notifications /// public void PopTaskItem(TaskItem taskBarItem) { - TaskItems.Remove(taskBarItem); - RaisePropertyChanged(nameof(HasTaskItems)); + InvokeUI(() => + { + TaskItems.Remove(taskBarItem); + RaisePropertyChanged(nameof(HasTaskItems)); + }); } /// @@ -605,8 +611,11 @@ namespace Tango.FSE.UI.Notifications /// public SnackbarItem PushSnackbarItem(SnackbarItem snackbarItem) { - SnackbarItems.Insert(0, snackbarItem); - RaisePropertyChanged(nameof(HasSnackbarItems)); + InvokeUI(() => + { + SnackbarItems.Insert(0, snackbarItem); + RaisePropertyChanged(nameof(HasSnackbarItems)); + }); return snackbarItem; } @@ -632,8 +641,11 @@ namespace Tango.FSE.UI.Notifications /// The snack bar item. public void PopSnackbarItem(SnackbarItem snackbarItem) { - SnackbarItems.Remove(snackbarItem); - RaisePropertyChanged(nameof(HasSnackbarItems)); + InvokeUI(() => + { + SnackbarItems.Remove(snackbarItem); + RaisePropertyChanged(nameof(HasSnackbarItems)); + }); } } } diff --git a/Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs b/Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs new file mode 100644 index 000000000..3fc553127 --- /dev/null +++ b/Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs @@ -0,0 +1,341 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Input; +using System.Windows.Media.Imaging; +using Tango.Core; +using Tango.Core.DI; +using Tango.Core.Threading; +using Tango.FSE.Common.Connection; +using Tango.FSE.Common.Notifications; +using Tango.FSE.Common.RemoteDesktop; +using Tango.RemoteDesktop.Frames; +using Tango.RemoteDesktop.Network; +using Tango.Transport; + +namespace Tango.FSE.UI.RemoteDesktop +{ + public class DefaultRemoteDesktopProvider : ExtendedObject, IRemoteDesktopProvider + { + private class MouseMovement + { + public Point Location { get; set; } + public Size ViewSize { get; set; } + } + + private IMachineProvider _machineProvider; + private RasterFrame _currentFrame; + private Size? _frameSize; + private IntervalMessageDispatcher _frameDispatcher; + private Thread _mouseMoveThread; + private ProducerConsumerQueue _mouseMovements; + + public event EventHandler FrameReceived; + + [TangoInject] + private INotificationProvider NotificationProvider { get; set; } + + private bool _inSession; + public bool InSession + { + get { return _inSession; } + set { _inSession = value; RaisePropertyChangedAuto(); RaisePropertyChanged(nameof(CanStartSession)); } + } + + public TimeSpan MouseMoveInterval { get; set; } + + public bool CanStartSession + { + get { return _machineProvider.IsConnected && !InSession; } + } + + public DefaultRemoteDesktopProvider() + { + MouseMoveInterval = TimeSpan.FromMilliseconds(100); + } + + public DefaultRemoteDesktopProvider(IMachineProvider machineProvider) : this() + { + _machineProvider = machineProvider; + _machineProvider.MachineConnected += (_, __) => + { + InSession = false; + }; + _machineProvider.MachineDisconnected += (_, __) => + { + InSession = false; + OnFrameReceived(null); + }; + } + + public Task StartSession() + { + TaskCompletionSource completionSource = new TaskCompletionSource(); + + if (!CanStartSession) + { + throw new InvalidOperationException("Unable to start a remote desktop session at the moment."); + } + + if (!InSession) + { + InSession = true; + + var taskItem = NotificationProvider.PushTaskItem("Starting remote desktop session..."); + + if (_frameDispatcher != null) + { + _frameDispatcher.Dispose(); + } + + _frameDispatcher = new IntervalMessageDispatcher(OnRemoteDesktopResponse); + _frameDispatcher.Start(); + + bool taskCompleted = false; + + _machineProvider.MachineOperator.SendGenericContinuousRequest(new StartRemoteDesktopSessionRequest() { }, new TransportContinuousRequestConfig() + { + Timeout = TimeSpan.FromSeconds(10), + }).Subscribe((response) => + { + if (!taskCompleted) + { + taskCompleted = true; + taskItem.Dispose(); + completionSource.SetResult(true); + } + + _frameDispatcher.Push(response); + + }, (ex) => + { + if (!taskCompleted) + { + taskCompleted = true; + taskItem.Dispose(); + completionSource.SetException(ex); + } + else if (InSession) + { + OnFailed(ex); + } + + InSession = false; + + //TODO: handle exception. + }, () => + { + //This cannot happen! + InSession = false; + OnFrameReceived(null); + }); + + return completionSource.Task; + } + else + { + return Task.FromResult(true); + } + } + + public async Task EndSession() + { + if (InSession) + { + using (NotificationProvider.PushTaskItem("Stopping remote desktop session...")) + { + await _machineProvider.MachineOperator.SendGenericRequest(new StopRemoteDesktopSessionRequest()); + InSession = false; + } + } + } + + private void OnFailed(Exception exception) + { + InSession = false; + OnFrameReceived(null); + + if (!(exception is TransporterDisconnectedException)) + { + NotificationProvider.PushSnackbarItem(MessageType.Error, "Remote desktop session terminated unexpectedly.", true, null, TimeSpan.FromSeconds(5)); + } + } + + private void OnRemoteDesktopResponse(StartRemoteDesktopSessionResponse response) + { + if (!InSession) return; //To stop frame delivery immediately. + + if (response.Packet == null) + { + _frameDispatcher.Interval = 1000 / response.FrameRate; + return; //This is the first message with empty bitmap and only the frame rate. + } + + try + { + if (!response.Packet.IsPartial) + { + using (MemoryStream ms = new MemoryStream(response.Packet.Bitmap)) + { + if (_currentFrame != null) + { + _currentFrame.Dispose(); + } + + _currentFrame = new RasterFrame(new System.Drawing.Bitmap(ms)); + _frameSize = new Size(_currentFrame.Width, _currentFrame.Height); + } + } + else + { + using (MemoryStream ms = new MemoryStream(response.Packet.Bitmap)) + { + var diffFrame = new RasterFrame(new System.Drawing.Bitmap(ms), response.Packet.PartialRegion.Left, response.Packet.PartialRegion.Top); + diffFrame.Apply(_currentFrame); + diffFrame.Dispose(); + } + } + + OnFrameReceived(_currentFrame.ToBitmapSource()); + } + catch (Exception ex) + { + Debug.WriteLine($"Error on remote desktop packet received: {ex.Message}"); + } + } + + protected virtual void OnFrameReceived(BitmapSource source) + { + FrameReceived?.Invoke(this, new DesktopFrameReceivedEventArgs() + { + Source = source, + }); + } + + public async void MouseDown(MouseButton button, Point location, Size viewSize) + { + if (!InSession || _frameSize == null) return; + + await _machineProvider.MachineOperator.SendGenericRequest(new MouseStateRequest() + { + Button = button, + EventType = MouseEventType.Down, + Location = TranslateLocation(location, viewSize) + }); + } + + public async void MouseUp(MouseButton button, Point location, Size viewSize) + { + if (!InSession || _frameSize == null) return; + + await _machineProvider.MachineOperator.SendGenericRequest(new MouseStateRequest() + { + Button = button, + EventType = MouseEventType.Up, + Location = TranslateLocation(location, viewSize) + }); + } + + public void MouseMove(Point location, Size viewSize) + { + if (!InSession || _frameSize == null) return; + + if (_mouseMoveThread == null) + { + _mouseMovements = new ProducerConsumerQueue(); + _mouseMoveThread = new Thread(async () => + { + while (InSession) + { + MouseMovement movement = _mouseMovements.BlockDequeue(); + + while (_mouseMovements.Count > 0) + { + movement = _mouseMovements.BlockDequeue(); + } + + try + { + await _machineProvider.MachineOperator.SendGenericRequest(new MouseStateRequest() + { + Button = MouseButton.Left, + EventType = MouseEventType.Move, + Location = TranslateLocation(movement.Location, movement.ViewSize) + }); + } + catch (Exception ex) + { + Debug.WriteLine($"Error on mouse move\n{ex.ToString()}"); + } + + Thread.Sleep(MouseMoveInterval); + } + + _mouseMoveThread = null; + }); + + _mouseMoveThread.IsBackground = true; + _mouseMoveThread.Start(); + } + + _mouseMovements.BlockEnqueue(new MouseMovement() + { + Location = location, + ViewSize = viewSize, + }); + } + + public async void MouseDoubleClick(MouseButton button, Point location, Size viewSize) + { + if (!InSession || _frameSize == null) return; + + await _machineProvider.MachineOperator.SendGenericRequest(new MouseStateRequest() + { + Button = button, + EventType = MouseEventType.DoubleClick, + Location = TranslateLocation(location, viewSize) + }); + } + + private Point TranslateLocation(Point location, Size viewSize) + { + return new Point( + (location.X / viewSize.Width) * _frameSize.Value.Width, + (location.Y / viewSize.Height) * _frameSize.Value.Height); + } + + public async void KeyboardDown(Key key, bool ctrlDown, bool shitDown, bool altDown) + { + if (!InSession || _frameSize == null) return; + + await _machineProvider.MachineOperator.SendGenericRequest(new KeyboardStateRequest() + { + EventType = KeyboardEventType.Down, + Key = key, + IsCtrlDown = ctrlDown, + IsShiftDown = shitDown, + IsAltDown = altDown, + }); + } + + public async void KeyboardUp(Key key, bool ctrlDown, bool shitDown, bool altDown) + { + if (!InSession || _frameSize == null) return; + + await _machineProvider.MachineOperator.SendGenericRequest(new KeyboardStateRequest() + { + EventType = KeyboardEventType.Up, + Key = key, + IsCtrlDown = ctrlDown, + IsShiftDown = shitDown, + IsAltDown = altDown, + }); + } + } +} diff --git a/Software/Visual_Studio/FSE/Tango.FSE.UI/Tango.FSE.UI.csproj b/Software/Visual_Studio/FSE/Tango.FSE.UI/Tango.FSE.UI.csproj index 7b3e64ff3..04d0e12d6 100644 --- a/Software/Visual_Studio/FSE/Tango.FSE.UI/Tango.FSE.UI.csproj +++ b/Software/Visual_Studio/FSE/Tango.FSE.UI/Tango.FSE.UI.csproj @@ -65,6 +65,24 @@ + + + ..\..\packages\System.Reactive.Core.3.1.1\lib\net46\System.Reactive.Core.dll + + + ..\..\packages\System.Reactive.Interfaces.3.1.1\lib\net45\System.Reactive.Interfaces.dll + + + ..\..\packages\System.Reactive.Linq.3.1.1\lib\net46\System.Reactive.Linq.dll + + + ..\..\packages\System.Reactive.PlatformServices.3.1.1\lib\net46\System.Reactive.PlatformServices.dll + + + ..\..\packages\System.Reactive.Windows.Threading.3.1.1\lib\net45\System.Reactive.Windows.Threading.dll + + + ..\..\packages\ControlzEx.3.0.2.4\lib\net45\System.Windows.Interactivity.dll @@ -128,6 +146,7 @@ MachineConnectionPane.xaml + @@ -260,7 +279,9 @@ - + + Designer + @@ -295,6 +316,10 @@ {e4927038-348d-4295-aaf4-861c58cb3943} Tango.PMR + + {A78068D4-2061-4376-8EDE-583D8D880DEC} + Tango.RemoteDesktop + {d8f1ad85-526a-4f50-b6dc-d437af63d8d8} Tango.Settings diff --git a/Software/Visual_Studio/FSE/Tango.FSE.UI/ViewModelLocator.cs b/Software/Visual_Studio/FSE/Tango.FSE.UI/ViewModelLocator.cs index fac3d7018..536a42ae6 100644 --- a/Software/Visual_Studio/FSE/Tango.FSE.UI/ViewModelLocator.cs +++ b/Software/Visual_Studio/FSE/Tango.FSE.UI/ViewModelLocator.cs @@ -14,6 +14,7 @@ using Tango.FSE.Common.Gateway; using Tango.FSE.Common.Modules; using Tango.FSE.Common.Navigation; using Tango.FSE.Common.Notifications; +using Tango.FSE.Common.RemoteDesktop; using Tango.FSE.Common.Threading; using Tango.FSE.Common.Web; using Tango.FSE.UI.Authentication; @@ -24,6 +25,7 @@ using Tango.FSE.UI.Gateway; using Tango.FSE.UI.Modules; using Tango.FSE.UI.Navigation; using Tango.FSE.UI.Notifications; +using Tango.FSE.UI.RemoteDesktop; using Tango.FSE.UI.Threading; using Tango.FSE.UI.ViewModels; @@ -43,6 +45,7 @@ namespace Tango.FSE.UI TangoIOC.Default.Unregister(); TangoIOC.Default.Unregister(); TangoIOC.Default.Unregister(); + TangoIOC.Default.Unregister(); //TangoIOC.Default.Unregister(); //TangoIOC.Default.Unregister(); //TangoIOC.Default.Unregister(); @@ -60,8 +63,8 @@ namespace Tango.FSE.UI TangoIOC.Default.Register(); TangoIOC.Default.Register(); TangoIOC.Default.Register(); - //TangoIOC.Default.Register(); TangoIOC.Default.Register(); + TangoIOC.Default.Register(); TangoIOC.Default.Register(); TangoIOC.Default.Register(); diff --git a/Software/Visual_Studio/FSE/Tango.FSE.UI/Views/MainView.xaml b/Software/Visual_Studio/FSE/Tango.FSE.UI/Views/MainView.xaml index 36c1cba6f..dca8e2740 100644 --- a/Software/Visual_Studio/FSE/Tango.FSE.UI/Views/MainView.xaml +++ b/Software/Visual_Studio/FSE/Tango.FSE.UI/Views/MainView.xaml @@ -104,8 +104,8 @@ - - + + diff --git a/Software/Visual_Studio/FSE/Tango.FSE.UI/packages.config b/Software/Visual_Studio/FSE/Tango.FSE.UI/packages.config index dd8c723e4..d17a56d7d 100644 --- a/Software/Visual_Studio/FSE/Tango.FSE.UI/packages.config +++ b/Software/Visual_Studio/FSE/Tango.FSE.UI/packages.config @@ -7,4 +7,10 @@ + + + + + + \ No newline at end of file diff --git a/Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/DefaultRemoteDesktopService.cs b/Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/DefaultRemoteDesktopService.cs index 67d0b85de..0d5bd8559 100644 --- a/Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/DefaultRemoteDesktopService.cs +++ b/Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/DefaultRemoteDesktopService.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Linq; using System.Text; using System.Threading.Tasks; +using System.Windows.Input; using Tango.Core; using Tango.Core.DI; using Tango.Integration.ExternalBridge; @@ -14,9 +15,11 @@ using Tango.RemoteDesktop.CaptureMethods; using Tango.RemoteDesktop.Encoders; using Tango.RemoteDesktop.Engines; using Tango.RemoteDesktop.Frames; +using Tango.RemoteDesktop.Input; using Tango.RemoteDesktop.Network; using Tango.Settings; using Tango.Transport; +using static Tango.RemoteDesktop.Input.MouseController; namespace Tango.PPC.Common.RemoteDesktop { @@ -45,6 +48,8 @@ namespace Tango.PPC.Common.RemoteDesktop _clients = new List(); _engine = new RasterScreenCaptureEngine(); + _engine.CaptureCursor = false; + _engine.FrameRate = Math.Min(Math.Max(_settings.RemoteDesktopFrameRate, 1), 20); _engine.FrameReceived += _engine_FrameReceived; } @@ -56,6 +61,16 @@ namespace Tango.PPC.Common.RemoteDesktop _engine.CaptureMethod.Dispose(); var mainWindow = System.Windows.Application.Current.MainWindow; + mainWindow.LocationChanged += (_, __) => + { + _engine.CaptureRegion = new CaptureRegion() + { + Left = (int)mainWindow.Left, + Top = (int)mainWindow.Top, + Width = (int)mainWindow.Width, + Height = (int)mainWindow.Height + }; + }; _engine.CaptureRegion = new CaptureRegion() { @@ -68,11 +83,6 @@ namespace Tango.PPC.Common.RemoteDesktop #endif _engine.Comparer.MaxDifferencesThrow = _engine.CaptureRegion.Width * _engine.CaptureRegion.Height / 2; - - if (_settings.EnableRemoteDesktop) - { - Start(); - } } private bool _isStarted; @@ -82,24 +92,6 @@ namespace Tango.PPC.Common.RemoteDesktop private set { _isStarted = value; RaisePropertyChangedAuto(); } } - public void Start() - { - if (!IsStarted) - { - _engine.Start(); - IsStarted = true; - } - } - - public void Stop() - { - if (IsStarted) - { - _engine.Stop(); - IsStarted = false; - } - } - [ExternalBridgeRequestHandlerMethod(typeof(StartRemoteDesktopSessionRequest))] public async void OnStartRemoteDesktopSessionRequestReceived(StartRemoteDesktopSessionRequest request, String token, ExternalBridgeReceiver receiver) { @@ -119,10 +111,85 @@ namespace Tango.PPC.Common.RemoteDesktop await receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse() { FrameRate = _engine.FrameRate - }, token, new TransportResponseConfig() + }, token); + + if (_settings.EnableRemoteDesktop) { - Immediate = false, - }); + if (!_engine.IsStarted) + { + _engine.Start(); + } + } + } + + [ExternalBridgeRequestHandlerMethod(typeof(StopRemoteDesktopSessionRequest))] + public async void OnStopRemoteDesktopSessionRequestReceived(StopRemoteDesktopSessionRequest request, String token, ExternalBridgeReceiver receiver) + { + var client = _clients.SingleOrDefault(x => x.Receiver == receiver); + + if (client != null) + { + _clients.Remove(client); + } + + if (_clients.Count == 0) + { + _engine.Stop(); + } + + await receiver.SendGenericResponse(new StopRemoteDesktopSessionResponse(), token); + + if (client != null) + { + await receiver.SendGenericResponse(new StartRemoteDesktopSessionResponse(), client.Token, new TransportResponseConfig() { Completed = true }); + } + } + + [ExternalBridgeRequestHandlerMethod(typeof(MouseStateRequest))] + public async void OnMouseStateRequestReceived(MouseStateRequest request, String token, ExternalBridgeReceiver receiver) + { + MouseController.SetCursorPosition((int)request.Location.X, (int)request.Location.Y); + + if (request.EventType == MouseEventType.Up || request.EventType == MouseEventType.Down) + { + MouseEventFlags flag = MouseEventFlags.LeftUp; + + switch (request.EventType) + { + case MouseEventType.Down: + flag = request.Button == MouseButton.Right ? MouseEventFlags.RightDown : MouseEventFlags.LeftDown; + break; + case MouseEventType.Up: + flag = request.Button == MouseButton.Right ? MouseEventFlags.RightUp : MouseEventFlags.LeftUp; + break; + } + + MouseController.MouseEvent(flag); + } + else if (request.EventType == MouseEventType.DoubleClick) + { + MouseController.MouseEvent(MouseEventFlags.LeftDown); + MouseController.MouseEvent(MouseEventFlags.LeftUp); + MouseController.MouseEvent(MouseEventFlags.LeftDown); + MouseController.MouseEvent(MouseEventFlags.LeftUp); + } + + await receiver.SendGenericResponse(new MouseStateResponse(), token); + } + + [ExternalBridgeRequestHandlerMethod(typeof(KeyboardStateRequest))] + public async void OnKeyboardStateRequestReceived(KeyboardStateRequest request, String token, ExternalBridgeReceiver receiver) + { + if (request.EventType == KeyboardEventType.Down) + { + KeyboardController.KeyDown(request.Key, request.IsCtrlDown, request.IsShiftDown, request.IsAltDown); + } + else + { + KeyboardController.KeyUp(request.Key, request.IsCtrlDown, request.IsShiftDown, request.IsAltDown); + } + + await receiver.SendGenericResponse(new KeyboardStateResponse(), token); } private async void _engine_FrameReceived(object sender, ScreenCaptureFrameReceivedEventArgs e) @@ -211,6 +278,11 @@ namespace Tango.PPC.Common.RemoteDesktop { _clients.Remove(client); } + + if (_clients.Count == 0) + { + _engine.Stop(); + } } } } diff --git a/Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/IRemoteDesktopService.cs b/Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/IRemoteDesktopService.cs index bce1bf51e..3c23c4832 100644 --- a/Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/IRemoteDesktopService.cs +++ b/Software/Visual_Studio/PPC/Tango.PPC.Common/RemoteDesktop/IRemoteDesktopService.cs @@ -9,7 +9,5 @@ namespace Tango.PPC.Common.RemoteDesktop public interface IRemoteDesktopService { bool IsStarted { get; } - void Start(); - void Stop(); } } diff --git a/Software/Visual_Studio/Tango.Core/Tango.Core.csproj b/Software/Visual_Studio/Tango.Core/Tango.Core.csproj index 8014ac3f2..605ba53dc 100644 --- a/Software/Visual_Studio/Tango.Core/Tango.Core.csproj +++ b/Software/Visual_Studio/Tango.Core/Tango.Core.csproj @@ -98,7 +98,7 @@ - + @@ -206,7 +206,7 @@ - + diff --git a/Software/Visual_Studio/Tango.Core/Threading/IntervalMessageDispatcher.cs b/Software/Visual_Studio/Tango.Core/Threading/IntervalMessageDispatcher.cs new file mode 100644 index 000000000..ed61bea62 --- /dev/null +++ b/Software/Visual_Studio/Tango.Core/Threading/IntervalMessageDispatcher.cs @@ -0,0 +1,119 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Tango.Core.Threading +{ + /// + /// Represents a message queue dispatcher for delivering messages at a constant interval. + /// + /// + /// + public class IntervalMessageDispatcher : IDisposable + { + private Thread _queueThread; + private ProducerConsumerQueue _queue; + private Action _onNext; + + /// + /// Gets or sets the interval. + /// + public int Interval { get; set; } + + /// + /// Gets a value indicating whether this instance has started. + /// + public bool IsStarted { get; private set; } + + /// + /// Gets the number of queued messages. + /// + public int Count + { + get { return _queue.Count; } + } + + /// + /// Initializes a new instance of the class. + /// + /// The delivery callback. + public IntervalMessageDispatcher(Action onNext) + { + _onNext = onNext; + _queue = new ProducerConsumerQueue(); + } + + /// + /// Starts the dispatching of messages. + /// + public void Start() + { + if (!IsStarted) + { + IsStarted = true; + _queueThread = new Thread(QueueThreadMethod); + _queueThread.Name = "Sequencer Thread"; + _queueThread.IsBackground = true; + _queueThread.Start(); + } + } + + /// + /// Pushes the specified message. + /// + /// The message. + public void Push(TMessage message) + { + _queue.BlockEnqueue(message); + } + + private void QueueThreadMethod() + { + Stopwatch watch = new Stopwatch(); + watch.Start(); + + while (IsStarted) + { + watch.Restart(); + var item = _queue.BlockDequeue(); + + if (!IsStarted) break; + + try + { + _onNext?.Invoke(item); + } + catch { } + + Thread.Sleep(Math.Max(1, (int)(Interval - watch.ElapsedMilliseconds))); + } + } + + /// + /// Releases unmanaged and - optionally - managed resources. + /// + public void Dispose() + { + IsStarted = false; + } + + /// + /// Creates a new instance of and starts it immediately. + /// + /// + /// The on next. + /// The interval. + /// + public static IntervalMessageDispatcher StartNew(Action onNext, int interval) + { + IntervalMessageDispatcher dispatcher = new IntervalMessageDispatcher(onNext); + dispatcher.Interval = interval; + dispatcher.Start(); + return dispatcher; + } + } +} diff --git a/Software/Visual_Studio/Tango.Core/Threading/SequencerThread.cs b/Software/Visual_Studio/Tango.Core/Threading/SequencerThread.cs deleted file mode 100644 index 603790466..000000000 --- a/Software/Visual_Studio/Tango.Core/Threading/SequencerThread.cs +++ /dev/null @@ -1,71 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace Tango.Core.Threading -{ - public class SequencerThread : IDisposable - { - private Thread _queueThread; - private ProducerConsumerQueue _queue; - private Action _onNext; - - public int FrameRate { get; set; } - - public bool IsStarted { get; private set; } - - public SequencerThread(Action onNext) - { - _onNext = onNext; - _queue = new ProducerConsumerQueue(); - } - - public void Start() - { - if (!IsStarted) - { - IsStarted = true; - _queueThread = new Thread(QueueThreadMethod); - _queueThread.Name = "Sequencer Thread"; - _queueThread.IsBackground = true; - _queueThread.Start(); - } - } - - public void Push(T item) - { - _queue.BlockEnqueue(item); - } - - private void QueueThreadMethod() - { - Stopwatch watch = new Stopwatch(); - watch.Start(); - - while (IsStarted) - { - watch.Restart(); - var item = _queue.BlockDequeue(); - - if (!IsStarted) break; - - try - { - _onNext?.Invoke(item); - } - catch { } - - Thread.Sleep(Math.Max(1, (int)(FrameRate - watch.ElapsedMilliseconds))); - } - } - - public void Dispose() - { - IsStarted = false; - } - } -} diff --git a/Software/Visual_Studio/Tango.Integration/ExternalBridge/ExternalBridgeTcpClient.cs b/Software/Visual_Studio/Tango.Integration/ExternalBridge/ExternalBridgeTcpClient.cs index 28ebc5341..5b59429d7 100644 --- a/Software/Visual_Studio/Tango.Integration/ExternalBridge/ExternalBridgeTcpClient.cs +++ b/Software/Visual_Studio/Tango.Integration/ExternalBridge/ExternalBridgeTcpClient.cs @@ -223,14 +223,16 @@ namespace Tango.Integration.ExternalBridge try { var response = await SendRequest(request, new TransportRequestConfig() { ShouldLog = true }); - - Status = MachineStatuses.Standby; } catch { } + + Status = MachineStatuses.Standby; } State = TransportComponentState.Disconnected; + NotifyContinuousRequestMessagesDisconnection(); + SessionLogger.EndSession(); if (Adapter != null) diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Input/KeyboardController.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Input/KeyboardController.cs new file mode 100644 index 000000000..832018dac --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Input/KeyboardController.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Input; +using WindowsInput; +using WindowsInput.Native; + +namespace Tango.RemoteDesktop.Input +{ + public static class KeyboardController + { + private static InputSimulator simulator; + + static KeyboardController() + { + simulator = new InputSimulator(); + } + + public static void KeyDown(Key key, bool ctrlDown, bool shitDown, bool altDown) + { + VirtualKeyCode virtualKey = (VirtualKeyCode)KeyInterop.VirtualKeyFromKey(key); + + if (ctrlDown || shitDown || altDown) + { + List modifierKeys = new List(); + + if (ctrlDown) + { + modifierKeys.Add(VirtualKeyCode.LCONTROL); + } + if (shitDown) + { + modifierKeys.Add(VirtualKeyCode.LSHIFT); + } + if (altDown) + { + modifierKeys.Add(VirtualKeyCode.MENU); + } + + simulator.Keyboard.ModifiedKeyStroke(modifierKeys, virtualKey); + } + else + { + simulator.Keyboard.KeyDown(virtualKey); + } + } + + public static void KeyUp(Key key, bool ctrlDown, bool shitDown, bool altDown) + { + VirtualKeyCode virtualKey = (VirtualKeyCode)KeyInterop.VirtualKeyFromKey(key); + + if (ctrlDown || shitDown || altDown) + { + List modifierKeys = new List(); + + if (ctrlDown) + { + modifierKeys.Add(VirtualKeyCode.LCONTROL); + } + if (shitDown) + { + modifierKeys.Add(VirtualKeyCode.LSHIFT); + } + if (altDown) + { + modifierKeys.Add(VirtualKeyCode.MENU); + } + + simulator.Keyboard.ModifiedKeyStroke(modifierKeys, virtualKey); + } + else + { + simulator.Keyboard.KeyUp(virtualKey); + } + } + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Input/MouseController.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Input/MouseController.cs new file mode 100644 index 000000000..ec556f88c --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Input/MouseController.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; + +namespace Tango.RemoteDesktop.Input +{ + public static class MouseController + { + [Flags] + public enum MouseEventFlags + { + LeftDown = 0x00000002, + LeftUp = 0x00000004, + MiddleDown = 0x00000020, + MiddleUp = 0x00000040, + Move = 0x00000001, + Absolute = 0x00008000, + RightDown = 0x00000008, + RightUp = 0x00000010 + } + + [DllImport("user32.dll", EntryPoint = "SetCursorPos")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool SetCursorPos(int x, int y); + + [DllImport("user32.dll")] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GetCursorPos(out MousePoint lpMousePoint); + + [DllImport("user32.dll")] + private static extern void mouse_event(int dwFlags, int dx, int dy, int dwData, int dwExtraInfo); + + public static void SetCursorPosition(int x, int y) + { + SetCursorPos(x, y); + } + + public static void SetCursorPosition(MousePoint point) + { + SetCursorPos(point.X, point.Y); + } + + public static MousePoint GetCursorPosition() + { + MousePoint currentMousePoint; + var gotPoint = GetCursorPos(out currentMousePoint); + if (!gotPoint) { currentMousePoint = new MousePoint(0, 0); } + return currentMousePoint; + } + + public static void MouseEvent(MouseEventFlags value) + { + MousePoint position = GetCursorPosition(); + + mouse_event + ((int)value, + position.X, + position.Y, + 0, + 0) + ; + } + + [StructLayout(LayoutKind.Sequential)] + public struct MousePoint + { + public int X; + public int Y; + + public MousePoint(int x, int y) + { + X = x; + Y = y; + } + } + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardEventType.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardEventType.cs new file mode 100644 index 000000000..b1b5b0353 --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardEventType.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Tango.RemoteDesktop.Network +{ + public enum KeyboardEventType + { + Up, + Down + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardStateRequest.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardStateRequest.cs new file mode 100644 index 000000000..27ae49f64 --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardStateRequest.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows.Input; + +namespace Tango.RemoteDesktop.Network +{ + public class KeyboardStateRequest + { + public KeyboardEventType EventType { get; set; } + public Key Key { get; set; } + public bool IsCtrlDown { get; set; } + public bool IsShiftDown { get; set; } + public bool IsAltDown { get; set; } + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardStateResponse.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardStateResponse.cs new file mode 100644 index 000000000..6544078f0 --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Network/KeyboardStateResponse.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Tango.RemoteDesktop.Network +{ + public class KeyboardStateResponse + { + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseEventType.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseEventType.cs new file mode 100644 index 000000000..2c9f84e03 --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseEventType.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Tango.RemoteDesktop.Network +{ + public enum MouseEventType + { + Down, + Up, + Move, + DoubleClick + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseStateRequest.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseStateRequest.cs new file mode 100644 index 000000000..36e98351f --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseStateRequest.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Windows; +using System.Windows.Input; + +namespace Tango.RemoteDesktop.Network +{ + public class MouseStateRequest + { + public MouseButton Button { get; set; } + public MouseEventType EventType { get; set; } + public Point Location { get; set; } + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseStateResponse.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseStateResponse.cs new file mode 100644 index 000000000..f67715d99 --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Network/MouseStateResponse.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Tango.RemoteDesktop.Network +{ + public class MouseStateResponse + { + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Network/StopRemoteDesktopSessionRequest.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Network/StopRemoteDesktopSessionRequest.cs new file mode 100644 index 000000000..058f5bbe1 --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Network/StopRemoteDesktopSessionRequest.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Tango.RemoteDesktop.Network +{ + public class StopRemoteDesktopSessionRequest + { + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Network/StopRemoteDesktopSessionResponse.cs b/Software/Visual_Studio/Tango.RemoteDesktop/Network/StopRemoteDesktopSessionResponse.cs new file mode 100644 index 000000000..92a239d11 --- /dev/null +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Network/StopRemoteDesktopSessionResponse.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Tango.RemoteDesktop.Network +{ + public class StopRemoteDesktopSessionResponse + { + } +} diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/ScreenCaptureEngine.cs b/Software/Visual_Studio/Tango.RemoteDesktop/ScreenCaptureEngine.cs index 81efb6a30..8920b983b 100644 --- a/Software/Visual_Studio/Tango.RemoteDesktop/ScreenCaptureEngine.cs +++ b/Software/Visual_Studio/Tango.RemoteDesktop/ScreenCaptureEngine.cs @@ -111,7 +111,6 @@ namespace Tango.RemoteDesktop if (IsStarted) { IsStarted = false; - _previousBitmap?.Dispose(); } } @@ -180,15 +179,20 @@ namespace Tango.RemoteDesktop catch (Exception ex) { Debug.WriteLine($"Error in screen capture engine: {ex.Message}"); + _previousBitmap = null; } int delay = Math.Max(5, (1000 / FrameRate) - (int)watch.ElapsedMilliseconds); Thread.Sleep(delay); } + + _previousBitmap = null; } private void OnFrameReceived(Bitmap currentBitmap, TFrame diffFrame, uint differenceCount) { + if (!IsStarted) return; + FrameReceived?.Invoke(this, new ScreenCaptureFrameReceivedEventArgs() { Frame = new ScreenCaptureFrame(currentBitmap, diffFrame) diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/Tango.RemoteDesktop.csproj b/Software/Visual_Studio/Tango.RemoteDesktop/Tango.RemoteDesktop.csproj index b0feb438d..dfffe90ec 100644 --- a/Software/Visual_Studio/Tango.RemoteDesktop/Tango.RemoteDesktop.csproj +++ b/Software/Visual_Studio/Tango.RemoteDesktop/Tango.RemoteDesktop.csproj @@ -66,6 +66,9 @@ + + ..\packages\InputSimulator.1.0.4.0\lib\net20\WindowsInput.dll + @@ -82,11 +85,21 @@ + + + + + + + + + + diff --git a/Software/Visual_Studio/Tango.RemoteDesktop/packages.config b/Software/Visual_Studio/Tango.RemoteDesktop/packages.config index 2cbb3038a..67634c01e 100644 --- a/Software/Visual_Studio/Tango.RemoteDesktop/packages.config +++ b/Software/Visual_Studio/Tango.RemoteDesktop/packages.config @@ -1,4 +1,5 @@  + \ No newline at end of file diff --git a/Software/Visual_Studio/Tango.Transport/TransporterBase.cs b/Software/Visual_Studio/Tango.Transport/TransporterBase.cs index f61e29795..45b89f9ff 100644 --- a/Software/Visual_Studio/Tango.Transport/TransporterBase.cs +++ b/Software/Visual_Studio/Tango.Transport/TransporterBase.cs @@ -21,6 +21,8 @@ using System.IO; using Tango.Core.ExtensionMethods; using Tango.PMR.Integration; using Newtonsoft.Json; +using System.Diagnostics; +using System.Reactive.Concurrency; namespace Tango.Transport { @@ -333,6 +335,7 @@ namespace Tango.Transport { try { + _pendingRequests.Remove(request); LogManager.Log($"Notifying continuous request '{(request.Message as ITangoMessage).Type}'..."); OnRequestFailed(request, new TransporterDisconnectedException("Transporter disconnected.")); request.SetException(new TransporterDisconnectedException("Transporter disconnected.")); @@ -908,8 +911,10 @@ namespace Tango.Transport var responseObject = JsonConvert.DeserializeObject(response.Message.Data.ToStringUtf8()); subject.OnNext(responseObject); } - catch + catch (Exception ex) { + Debugger.Break(); + System.Diagnostics.Debug.WriteLine($"Exception thrown by the generic request continuous handler.\n{ex.ToString()}"); //Ignore exception at the client side. } @@ -920,8 +925,10 @@ namespace Tango.Transport { subject.OnError(ex); } - catch + catch (Exception xx) { + Debugger.Break(); + System.Diagnostics.Debug.WriteLine($"Exception thrown by the generic request error handler.\n{xx.ToString()}"); //Ignore exception at the client side. } @@ -931,8 +938,10 @@ namespace Tango.Transport { subject.OnCompleted(); } - catch + catch (Exception ex) { + Debugger.Break(); + System.Diagnostics.Debug.WriteLine($"Exception thrown by the generic request completed handler.\n{ex.ToString()}"); //Ignore exception at the client side. } }); @@ -1052,7 +1061,7 @@ namespace Tango.Transport { config = config ?? new TransportResponseConfig(); - if (_pushThread == null || _pushThread.ThreadState == ThreadState.Aborted) + if (_pushThread == null || _pushThread.ThreadState == System.Threading.ThreadState.Aborted) { throw new InvalidOperationException("Transporter push thread is not in a running state."); } -- cgit v1.3.1