Added support for FSE mouse gestures via WebRTC.

Added SendBinaryData via WebRtcClient.

5 files changed, 75 insertions, 28 deletions

diff --git a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/ViewModels/RemoteDesktopViewVM.cs b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/ViewModels/RemoteDesktopViewVM.cs
index f2ed84d8a..7fab5efd2 100644
--- a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/ViewModels/RemoteDesktopViewVM.cs
+++ b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/ViewModels/RemoteDesktopViewVM.cs
@@ -106,7 +106,7 @@ namespace Tango.FSE.PPCConsole.ViewModels
 
         public void OnMouseMove(System.Windows.Point point, System.Windows.Size size)
         {
-            //RemoteDesktopProvider.MouseMove(point, size);
+            RemoteDesktopProvider.MouseMove(point, size);
         }
 
         public void OnMouseDoubleClick(MouseButton changedButton, System.Windows.Point point, System.Windows.Size size)
diff --git a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml.cs b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml.cs
index be7c280c8..a63404aec 100644
--- a/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml.cs
+++ b/Software/Visual_Studio/FSE/Modules/Tango.FSE.PPCConsole/Views/RemoteDesktopView.xaml.cs
@@ -47,6 +47,8 @@ namespace Tango.FSE.PPCConsole.Views
 
         private void Img_PreviewMouseDown(object sender, MouseButtonEventArgs e)
         {
+            Mouse.Capture(img);
+
             if (e.ClickCount == 2)
             {
                 _vm.OnMouseDoubleClick(e.ChangedButton, e.GetPosition(img), new Size(img.ActualWidth, img.ActualHeight));
@@ -59,6 +61,7 @@ namespace Tango.FSE.PPCConsole.Views
 
         private void Img_PreviewMouseUp(object sender, MouseButtonEventArgs e)
         {
+            img.ReleaseMouseCapture();
             _vm.OnMouseUp(e.ChangedButton, e.GetPosition(img), new Size(img.ActualWidth, img.ActualHeight));
         }
 
diff --git a/Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs b/Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs
index 4866e87a5..13e906ce0 100644
--- a/Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs
+++ b/Software/Visual_Studio/FSE/Tango.FSE.UI/RemoteDesktop/DefaultRemoteDesktopProvider.cs
@@ -1,4 +1,5 @@
-using System;
+using Newtonsoft.Json;
+using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -42,6 +43,7 @@ namespace Tango.FSE.UI.RemoteDesktop
         private WebRtcClient _webRtcClient;
         private List<IceCandidate> _iceCandidates;
         private bool _answerReceived;
+        private JsonSerializerSettings _jsonSettings;
 
         #region Events
 
@@ -95,6 +97,11 @@ namespace Tango.FSE.UI.RemoteDesktop
 
         public DefaultRemoteDesktopProvider()
         {
+            _jsonSettings = new JsonSerializerSettings()
+            {
+                TypeNameHandling = TypeNameHandling.All,
+            };
+
             _iceCandidates = new List<IceCandidate>();
             MouseMoveInterval = TimeSpan.FromMilliseconds(100);
             EnableWebRtc = true;
@@ -112,7 +119,7 @@ namespace Tango.FSE.UI.RemoteDesktop
                 InSession = false;
                 OnFrameReceived(null);
             };
-            //_machineProvider.MachineOperator.RequestReceived += MachineOperator_RequestReceived;
+
             _machineProvider.MachineOperator.RegisterRequestHandler<WebRtcIceCandidateRequest>(OnIceCandidateRequestReceived);
         }
 
@@ -128,20 +135,6 @@ namespace Tango.FSE.UI.RemoteDesktop
 
         #endregion
 
-        //private async void MachineOperator_RequestReceived(object sender, PMR.Common.MessageContainer container)
-        //{
-        //    if (container.Type == PMR.Common.MessageType.GenericRequest)
-        //    {
-        //        var message = MessageFactory.ExtractMessageFromContainer(container);
-        //        var type = Type.GetType((message as GenericRequest).Type);
-
-        //        if (type == typeof(WebRtcIceCandidateRequest))
-        //        {
-
-        //        }
-        //    }
-        //}
-
         #region Start/Stop Session
 
         public async Task StartSession()
@@ -324,24 +317,42 @@ namespace Tango.FSE.UI.RemoteDesktop
         {
             if (!InSession || _frameSize == null) return;
 
-            await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(new MouseStateRequest()
+            var request = new MouseStateRequest()
             {
                 Button = button,
                 EventType = MouseEventType.Down,
                 Location = TranslateLocation(location, viewSize)
-            });
+            };
+
+            if (IsWebRtcActive)
+            {
+                SendWebRtcObject(request);
+            }
+            else
+            {
+                await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(request);
+            }
         }
 
         public async void MouseUp(MouseButton button, Point location, Size viewSize)
         {
             if (!InSession || _frameSize == null) return;
 
-            await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(new MouseStateRequest()
+            var request = new MouseStateRequest()
             {
                 Button = button,
                 EventType = MouseEventType.Up,
                 Location = TranslateLocation(location, viewSize)
-            });
+            };
+
+            if (IsWebRtcActive)
+            {
+                SendWebRtcObject(request);
+            }
+            else
+            {
+                await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(request);
+            }
         }
 
         public void MouseMove(Point location, Size viewSize)
@@ -364,12 +375,21 @@ namespace Tango.FSE.UI.RemoteDesktop
 
                         try
                         {
-                            await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(new MouseStateRequest()
+                            var request = new MouseStateRequest()
                             {
                                 Button = MouseButton.Left,
                                 EventType = MouseEventType.Move,
                                 Location = TranslateLocation(movement.Location, movement.ViewSize)
-                            });
+                            };
+
+                            if (IsWebRtcActive)
+                            {
+                                SendWebRtcObject(request);
+                            }
+                            else
+                            {
+                                await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(request);
+                            }
                         }
                         catch (Exception ex)
                         {
@@ -397,12 +417,21 @@ namespace Tango.FSE.UI.RemoteDesktop
         {
             if (!InSession || _frameSize == null) return;
 
-            await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(new MouseStateRequest()
+            var request = new MouseStateRequest()
             {
                 Button = button,
                 EventType = MouseEventType.DoubleClick,
                 Location = TranslateLocation(location, viewSize)
-            });
+            };
+
+            if (IsWebRtcActive)
+            {
+                SendWebRtcObject(request);  
+            }
+            else
+            {
+                await _machineProvider.MachineOperator.SendGenericRequest<MouseStateRequest, MouseStateResponse>(request);
+            }
         }
 
         private Point TranslateLocation(Point location, Size viewSize)
@@ -561,6 +590,11 @@ namespace Tango.FSE.UI.RemoteDesktop
             }
         }
 
+        private void SendWebRtcObject(object obj)
+        {
+            _webRtcClient.SendText(JsonConvert.SerializeObject(obj, _jsonSettings));
+        }
+
         #endregion
 
         #region OnFailed
diff --git a/Software/Visual_Studio/Tango.SystemInfo/SystemObjectsCollection.cs b/Software/Visual_Studio/Tango.SystemInfo/SystemObjectsCollection.cs
index 92937f561..0596b5745 100644
--- a/Software/Visual_Studio/Tango.SystemInfo/SystemObjectsCollection.cs
+++ b/Software/Visual_Studio/Tango.SystemInfo/SystemObjectsCollection.cs
@@ -23,10 +23,13 @@ namespace Tango.SystemInfo
 
             str += Name + "\n\n";
 
-            foreach (var obj in Objects)
+            if (Objects != null)
             {
-                str += obj.ToString();
-                str += "\n";
+                foreach (var obj in Objects)
+                {
+                    str += obj.ToString();
+                    str += "\n";
+                }
             }
 
             str += "\n";
diff --git a/Software/Visual_Studio/Tango.WebRTC/WebRtcClient.cs b/Software/Visual_Studio/Tango.WebRTC/WebRtcClient.cs
index bbbabb4e4..e1d49e18c 100644
--- a/Software/Visual_Studio/Tango.WebRTC/WebRtcClient.cs
+++ b/Software/Visual_Studio/Tango.WebRTC/WebRtcClient.cs
@@ -412,6 +412,13 @@ namespace Tango.WebRTC
             _conductor.DataChannelSendText(msg);
         }
 
+        public void SendBinary(byte[] data)
+        {
+            EnsureInitialized();
+
+            _conductor.DataChannelSendData(data);
+        }
+
         public void AddIceCandidate(IceCandidate ice)
         {
             EnsureInitialized();