Telemetry Fixes.

7 files changed, 79 insertions, 57 deletions

diff --git a/Software/Visual_Studio/Tango.Telemetry/DateTimeUTCFixer.cs b/Software/Visual_Studio/Tango.Telemetry/DateTimeUTCFixer.cs
index 2a0c3cc85..490a49e53 100644
--- a/Software/Visual_Studio/Tango.Telemetry/DateTimeUTCFixer.cs
+++ b/Software/Visual_Studio/Tango.Telemetry/DateTimeUTCFixer.cs
@@ -47,7 +47,7 @@ namespace Tango.Telemetry
                     var value = dict[key];
                     if (TrySpecifyUtcOnBoxedDateTime(ref value))
                     {
-                        dict[key] = value!;
+                        dict[key] = value;
                     }
                     else
                     {
@@ -65,7 +65,7 @@ namespace Tango.Telemetry
                     var element = list[i];
                     if (TrySpecifyUtcOnBoxedDateTime(ref element))
                     {
-                        list[i] = element!;
+                        list[i] = element;
                     }
                     else
                     {
diff --git a/Software/Visual_Studio/Tango.Telemetry/PendingTelemetry.cs b/Software/Visual_Studio/Tango.Telemetry/PendingTelemetry.cs
index 5e1b5778d..39c36cb8d 100644
--- a/Software/Visual_Studio/Tango.Telemetry/PendingTelemetry.cs
+++ b/Software/Visual_Studio/Tango.Telemetry/PendingTelemetry.cs
@@ -12,7 +12,6 @@ namespace Tango.Telemetry
         [BsonId(true)]
         public int Id { get; set; }
         public DateTime Created { get; set; }
-        public DateTime Expires { get; set; }
         public ITelemetry TelemetryObject { get; set; }
         public String Source { get; set; }
         public TelemetrySourceTypes SourceType { get; set; }
diff --git a/Software/Visual_Studio/Tango.Telemetry/Telemetries/TelemetryLog.cs b/Software/Visual_Studio/Tango.Telemetry/Telemetries/TelemetryLog.cs
index e32243d54..ab141bbe0 100644
--- a/Software/Visual_Studio/Tango.Telemetry/Telemetries/TelemetryLog.cs
+++ b/Software/Visual_Studio/Tango.Telemetry/Telemetries/TelemetryLog.cs
@@ -20,10 +20,5 @@ namespace Tango.Telemetry.Telemetries
         public int Line { get; set; }
 
         public String Message { get; set; }
-
-        public TelemetryLog() : base()
-        {
-            ID = String.Empty;
-        }
     }
 }
diff --git a/Software/Visual_Studio/Tango.Telemetry/TelemetryLiteDBStorageManager.cs b/Software/Visual_Studio/Tango.Telemetry/TelemetryLiteDBStorageManager.cs
index f92d508e7..538a3a0d7 100644
--- a/Software/Visual_Studio/Tango.Telemetry/TelemetryLiteDBStorageManager.cs
+++ b/Software/Visual_Studio/Tango.Telemetry/TelemetryLiteDBStorageManager.cs
@@ -248,8 +248,8 @@ namespace Tango.Telemetry
         public void PerformPublishedTelemetriesCleanUp(DateTime olderThan)
         {
             LogManager.Log("Performing published telemetries cache cleanup...");
-            var collection = GetSourcesCheckpointCollection();
-            int deleted = collection.DeleteMany(x => x.Time < olderThan);
+            var collection = GetPublishedTelemetriesCollection();
+            int deleted = collection.DeleteMany(x => x.CreatedAt < olderThan);
             LogManager.Log($"Published telemetries cleanup completed. {deleted} cleaned.");
         }
 
diff --git a/Software/Visual_Studio/Tango.Telemetry/TelemetryPendingStorageSource.cs b/Software/Visual_Studio/Tango.Telemetry/TelemetryPendingStorageSource.cs
index a5e176ca7..96022341d 100644
--- a/Software/Visual_Studio/Tango.Telemetry/TelemetryPendingStorageSource.cs
+++ b/Software/Visual_Studio/Tango.Telemetry/TelemetryPendingStorageSource.cs
@@ -9,7 +9,7 @@ namespace Tango.Telemetry
     public class TelemetryPendingStorageSource : ITelemetrySource
     {
         public string Name { get; private set; } = "Pending Storage";
-        public bool RequiresTelemetryDuplicationTracking { get => false; }
+        public bool RequiresTelemetryDuplicationTracking { get => true; }
 
         public void Dispose()
         {
diff --git a/Software/Visual_Studio/Tango.Telemetry/TelemetryPublisher.cs b/Software/Visual_Studio/Tango.Telemetry/TelemetryPublisher.cs
index 4c91e6733..4f13eda9d 100644
--- a/Software/Visual_Studio/Tango.Telemetry/TelemetryPublisher.cs
+++ b/Software/Visual_Studio/Tango.Telemetry/TelemetryPublisher.cs
@@ -523,7 +523,6 @@ namespace Tango.Telemetry
         {
             PendingTelemetry pendingTelemetry = new PendingTelemetry();
             pendingTelemetry.Created = DateTime.UtcNow;
-            //pendingTelemetry.Expires = source.GetExpiration();
             pendingTelemetry.Source = source.Name;
             pendingTelemetry.SourceType = sourceType;
             pendingTelemetry.TelemetryObject = telemetry;
@@ -824,41 +823,46 @@ namespace Tango.Telemetry
             }
 
             var batch = StorageManager.GetPendingTelemetries(maxCount);
-            LogManager.Log($"Flushing {batch.Count} pending telemetry package(s).", LogCategory.Info);
 
             List<TelemetryPublishResult> results = new List<TelemetryPublishResult>();
 
-            foreach (var pendingTelemetry in batch)
+            if (batch.Count > 0)
             {
-                try
+                LogManager.Log($"Flushing {batch.Count} pending telemetry package(s).", LogCategory.Info);
+
+                foreach (var pendingTelemetry in batch)
                 {
-                    var package = new TelemetryPublishPackage()
+                    try
                     {
-                        Source = _pendingStorageSource,
-                        PendingTelemetry = pendingTelemetry,
-                        SourceType = TelemetrySourceTypes.PendingStorage
-                    };
+                        var package = new TelemetryPublishPackage()
+                        {
+                            Source = _pendingStorageSource,
+                            PendingTelemetry = pendingTelemetry,
+                            SourceType = TelemetrySourceTypes.PendingStorage
+                        };
 
-                    var result = await PushTelemetryPackageAwait(package);
-                    results.Add(result);
+                        var result = await PushTelemetryPackageAwait(package);
+                        results.Add(result);
 
-                    LogManager.Log(
-                        $"Flushed telemetry to destinations: {string.Join(", ", result.DestinationsResults.Select(r => $"{r.Destination.Name}={r.Status}"))}",
-                        LogCategory.Debug);
-                }
-                catch (Exception ex)
-                {
-                    LogManager.Log(ex, LogCategory.Error, "Exception occurred while flushing a pending telemetry package.");
-                }
+                        LogManager.Log(
+                            $"Flushed telemetry to destinations: {string.Join(", ", result.DestinationsResults.Select(r => $"{r.Destination.Name}={r.Status}"))}",
+                            LogCategory.Debug);
+                    }
+                    catch (Exception ex)
+                    {
+                        LogManager.Log(ex, LogCategory.Error, "Exception occurred while flushing a pending telemetry package.");
+                    }
 
-                if (!IsStarted || _isDisposed)
-                {
-                    LogManager.Log("Flush operation interrupted: publisher is no longer active.", LogCategory.Warning);
-                    return results;
+                    if (!IsStarted || _isDisposed)
+                    {
+                        LogManager.Log("Flush operation interrupted: publisher is no longer active.", LogCategory.Warning);
+                        return results;
+                    }
                 }
+
+                LogManager.Log("FlushPendingTelemetries completed successfully.", LogCategory.Info);
             }
 
-            LogManager.Log("FlushPendingTelemetries completed successfully.", LogCategory.Info);
             return results;
         }
 
diff --git a/Software/Visual_Studio/Utilities/Tango.Telemetry.Tester.IOT.CLI/Program.cs b/Software/Visual_Studio/Utilities/Tango.Telemetry.Tester.IOT.CLI/Program.cs
index a75b63771..3346b425e 100644
--- a/Software/Visual_Studio/Utilities/Tango.Telemetry.Tester.IOT.CLI/Program.cs
+++ b/Software/Visual_Studio/Utilities/Tango.Telemetry.Tester.IOT.CLI/Program.cs
@@ -41,30 +41,12 @@ namespace Tango.Telemetry.Tester.IOT.CLI
 
             //publisher.RegisterSource(new JobRunsTestSource());
             //publisher.RegisterSource(new DiagnosticsTestSource());
-            publisher.RegisterSource(new TelemetryMachineUpdatesStreamingSource());
+            publisher.RegisterSource(new EventsTestSource());
             publisher.RegisterDestination(new TelemetryAzureHubDestination("HostName=iot-twine-dev-weu.azure-devices.net;DeviceId=telemetry-dev-01;SharedAccessKey=cZhCMhiVL+TF7p13fpX+lFmyxoy8ZqCkbxUwumWw18Q="));
 
             publisher.PublishResultAvailable += Publisher_PublishResultAvailable;
             publisher.PublishingPackage += Publisher_PublishingPackage;
 
-            Task.Factory.StartNew(() => 
-            {
-                Thread.Sleep(2000);
-                using (ObservablesContext db = new ObservablesContext())
-                {
-                    TangoUpdate update = new TangoUpdate();
-                    update.ApplicationVersion = "1.0.0.0";
-                    update.FirmwareVersion = "1.2.3.4";
-                    update.UpdateStatus = TangoUpdateStatuses.DatabaseFailed;
-                    update.FailedReason = "Fake Reason";
-                    update.FailedLog = "Failed log something...";
-                    update.StartDate = DateTime.UtcNow;
-                    update.EndDate = DateTime.UtcNow.AddMinutes(1);
-                    db.TangoUpdates.Add(update);
-                    db.SaveChanges();
-                }
-            });
-
             publisher.Start().GetAwaiter().GetResult();
 
             Console.Clear();
@@ -119,7 +101,7 @@ namespace Tango.Telemetry.Tester.IOT.CLI
             Console.ForegroundColor = ConsoleColor.DarkGray;
             Console.WriteLine($"Package publish result available:\n{e.PublishResult.ToString()}");
 
-            if (e.PublishResult.DestinationsResults.Any(d => d.Status == TelemetryPublishResult.DestinationStatus.Failed))
+            if (e.PublishResult.DestinationsResults.Any(d => d.Status == TelemetryPublishResult.DestinationStatus.Failed || d.Status == TelemetryPublishResult.DestinationStatus.Unavailable))
             {
                 Console.ForegroundColor = ConsoleColor.Red;
                 Console.WriteLine("One or more destinations failed to receive the package.");
@@ -203,7 +185,7 @@ namespace Tango.Telemetry.Tester.IOT.CLI
         {
             IsStarted = true;
 
-            Task.Factory.StartNew(() => 
+            Task.Factory.StartNew(() =>
             {
                 InsightsFile file = InsightsFile.FromFile("sample.insights");
                 foreach (var rawFrame in file.Frames.Where(x => !x.IsEmpty))
@@ -239,7 +221,7 @@ namespace Tango.Telemetry.Tester.IOT.CLI
         {
             base.Start();
 
-            Task.Factory.StartNew(() => 
+            Task.Factory.StartNew(() =>
             {
                 for (int i = 0; i < 100; i++)
                 {
@@ -249,4 +231,46 @@ namespace Tango.Telemetry.Tester.IOT.CLI
             });
         }
     }
+
+    public class EventsTestSource : ITelemetryStreamingSource
+    {
+        public bool IsStarted { get; }
+        public string Name { get; } = "Events Test Streaming";
+        public bool RequiresTelemetryDuplicationTracking { get; }
+
+        public event EventHandler<TelemetryAvailableEventArgs> TelemetryAvailable;
+
+        public void Dispose()
+        {
+
+        }
+
+        public void Start()
+        {
+            Task.Factory.StartNew(() =>
+            {
+                Thread.Sleep(2000);
+
+                using (ObservablesContext db = ObservablesContext.CreateDefault())
+                {
+                    foreach (var evType in db.EventTypes.ToList())
+                    {
+                        TelemetryEvent ev = new TelemetryEvent();
+                        ev.Description = $"Description of {evType.Description}";
+                        ev.EventTypeGuid = evType.Guid;
+                        ev.HostName = "TEST MACHINE";
+                        ev.Time = DateTime.UtcNow;
+                        ev.Time = DateTime.SpecifyKind(ev.Time, DateTimeKind.Unspecified);
+                        TelemetryAvailable?.Invoke(this, new TelemetryAvailableEventArgs() { TelemetryObject = ev });
+                        Thread.Sleep(200);
+                    }
+                }
+            });
+        }
+
+        public void Stop()
+        {
+
+        }
+    }
 }