Improved Telemetry IoT Destination.

3 files changed, 686 insertions, 0 deletions

diff --git a/Software/Visual_Studio/Tango.Telemetry/Helpers/DateTimeUTCFixer.cs b/Software/Visual_Studio/Tango.Telemetry/Helpers/DateTimeUTCFixer.cs
new file mode 100644
index 000000000..3ce0c700f
--- /dev/null
+++ b/Software/Visual_Studio/Tango.Telemetry/Helpers/DateTimeUTCFixer.cs
@@ -0,0 +1,209 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+namespace Tango.Telemetry.Helpers
+{
+    internal static class DateTimeUtcFixer
+    {
+        /// <summary>
+        /// Recursively scans the object graph starting at <paramref name="obj"/> and
+        /// sets Kind=Utc on any DateTime/DateTime? properties or collection elements that are Kind=Unspecified.
+        /// Uses DateTime.SpecifyKind(value, DateTimeKind.Utc) (no clock adjustment).
+        /// Notes:
+        /// - Only properties are updated (not fields).
+        /// - Collections: updates elements for IList/arrays and dictionary values; traverses other IEnumerable to reach nested objects.
+        /// - Read-only properties (no setter) are skipped.
+        /// - Root objects that are a DateTime value cannot be changed in place (struct); wrap them in a container if needed.
+        /// </summary>
+        public static void EnsureDateTimeUTC(object obj)
+        {
+            var visited = new HashSet<object>(ReferenceEqualityComparer.Instance);
+            EnsureUtcInternal(obj, visited);
+        }
+
+        private static void EnsureUtcInternal(object obj, HashSet<object> visited)
+        {
+            if (obj == null) return;
+
+            var type = obj.GetType();
+
+            // Avoid revisiting reference objects (handle cycles, EF proxies, etc.)
+            if (!type.IsValueType) // only track reference types
+            {
+                if (!visited.Add(obj)) return;
+            }
+
+            // Handle dictionaries: update values, recurse into objects
+            if (obj is IDictionary dict)
+            {
+                var keys = new List<object>();
+                foreach (var k in dict.Keys) keys.Add(k);
+
+                foreach (var key in keys)
+                {
+                    var value = dict[key];
+                    if (TrySpecifyUtcOnBoxedDateTime(ref value))
+                    {
+                        dict[key] = value;
+                    }
+                    else
+                    {
+                        EnsureUtcInternal(value, visited);
+                    }
+                }
+                return;
+            }
+
+            // Handle lists/arrays: update elements, recurse into objects
+            if (obj is IList list)
+            {
+                for (int i = 0; i < list.Count; i++)
+                {
+                    var element = list[i];
+                    if (TrySpecifyUtcOnBoxedDateTime(ref element))
+                    {
+                        list[i] = element;
+                    }
+                    else
+                    {
+                        EnsureUtcInternal(element, visited);
+                    }
+                }
+                return;
+            }
+
+            // Traverse other enumerables just to reach nested objects (cannot replace DateTime elements)
+            if (obj is IEnumerable enumerable && !(obj is string))
+            {
+                foreach (var element in enumerable)
+                {
+                    EnsureUtcInternal(element, visited);
+                }
+            }
+
+            // Inspect properties
+            const BindingFlags flags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic;
+            foreach (var prop in type.GetProperties(flags))
+            {
+                // indexers not supported
+                if (prop.GetIndexParameters().Length != 0) continue;
+
+                // must be readable
+                if (!prop.CanRead) continue;
+
+                var pType = prop.PropertyType;
+
+                // DateTime
+                if (pType == typeof(DateTime))
+                {
+                    var val = (DateTime)prop.GetValue(obj);
+                    if (val.Kind == DateTimeKind.Unspecified)
+                    {
+                        var newVal = DateTime.SpecifyKind(val, DateTimeKind.Utc);
+                        TrySetProperty(obj, prop, newVal);
+                    }
+                    continue;
+                }
+
+                // Nullable<DateTime>
+                var underlying = Nullable.GetUnderlyingType(pType);
+                if (underlying == typeof(DateTime))
+                {
+                    var val = (DateTime?)prop.GetValue(obj);
+                    if (val.HasValue && val.Value.Kind == DateTimeKind.Unspecified)
+                    {
+                        var newVal = (DateTime?)DateTime.SpecifyKind(val.Value, DateTimeKind.Utc);
+                        TrySetProperty(obj, prop, newVal);
+                    }
+                    continue;
+                }
+
+                // Skip obvious simple types
+                if (IsSimple(pType)) continue;
+
+                // Recurse into complex objects
+                var child = prop.GetValue(obj);
+                EnsureUtcInternal(child, visited);
+            }
+        }
+
+        private static bool TrySpecifyUtcOnBoxedDateTime(ref object obj)
+        {
+            if (obj == null) return false;
+
+            var t = obj.GetType();
+
+            // Exact DateTime
+            if (t == typeof(DateTime))
+            {
+                var dt = (DateTime)obj;
+                if (dt.Kind == DateTimeKind.Unspecified)
+                {
+                    obj = DateTime.SpecifyKind(dt, DateTimeKind.Utc);
+                }
+                return true;
+            }
+
+            // Nullable<DateTime> boxed
+            var underlying = Nullable.GetUnderlyingType(t);
+            if (underlying == typeof(DateTime))
+            {
+                // boxed Nullable<DateTime> can be unboxed to DateTime? safely
+                var ndt = (DateTime?)obj;
+                if (ndt.HasValue && ndt.Value.Kind == DateTimeKind.Unspecified)
+                {
+                    obj = (DateTime?)DateTime.SpecifyKind(ndt.Value, DateTimeKind.Utc);
+                }
+                return true;
+            }
+
+            return false;
+        }
+
+        private static bool TrySetProperty(object target, PropertyInfo prop, object value)
+        {
+            try
+            {
+                // Prefer invoking the setter (even if non-public)
+                var setter = prop.GetSetMethod(nonPublic: true);
+                if (setter == null) return false;
+                setter.Invoke(target, new[] { value });
+                return true;
+            }
+            catch
+            {
+                // Fall back to PropertyInfo.SetValue (may still work in some runtimes)
+                try
+                {
+                    prop.SetValue(target, value);
+                    return true;
+                }
+                catch
+                {
+                    return false;
+                }
+            }
+        }
+
+        private static bool IsSimple(Type t)
+        {
+            if (t.IsPrimitive || t.IsEnum) return true;
+            if (t == typeof(string) || t == typeof(decimal) || t == typeof(Guid) ||
+                t == typeof(Uri) || t == typeof(TimeSpan) || t == typeof(DateTimeOffset))
+                return true;
+            // Value types other than DateTime/Nullable<DateTime> are considered simple
+            return t.IsValueType;
+        }
+
+        /// <summary>Reference equality comparer for the visited set.</summary>
+        private sealed class ReferenceEqualityComparer : IEqualityComparer<object>
+        {
+            public static readonly ReferenceEqualityComparer Instance = new ReferenceEqualityComparer();
+            public new bool Equals(object x, object y) => ReferenceEquals(x, y);
+            public int GetHashCode(object obj) => RuntimeHelpers.GetHashCode(obj);
+        }
+    }
+}
diff --git a/Software/Visual_Studio/Tango.Telemetry/Helpers/InternetConnectivity.cs b/Software/Visual_Studio/Tango.Telemetry/Helpers/InternetConnectivity.cs
new file mode 100644
index 000000000..f747bb263
--- /dev/null
+++ b/Software/Visual_Studio/Tango.Telemetry/Helpers/InternetConnectivity.cs
@@ -0,0 +1,388 @@
+using System;
+using System.Linq;
+using System.Net;
+using System.Net.Http;
+using System.Net.NetworkInformation;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Tango.Telemetry.Helpers
+{
+    /// <summary>
+    /// Active internet connectivity monitor with fast cached reads.
+    /// - Immediate wake on network changes (address/availability) -> near-instant pickup after Wi-Fi reconnect.
+    /// - Burst probing after reconnect to avoid brief false negatives.
+    /// - .NET Framework 4.6.1 / C# 7 compatible.
+    /// </summary>
+    public static class InternetConnectivity
+    {
+        // -------- Public API --------
+
+        /// <summary>
+        /// Returns the last known internet status instantly (safe to call every 200ms).
+        /// First call primes quickly so you don't get default false.
+        /// </summary>
+        public static bool IsInternetAvailable()
+        {
+            if (!_primed)
+            {
+                EnsureStarted();
+                PrimeOnce();
+            }
+            return _lastIsUp;
+        }
+
+        /// <summary>Raised when status flips.</summary>
+        public static event Action<bool> StatusChanged;
+
+        public static void EnsureStarted()
+        {
+            if (_started) return;
+            lock (_startLock)
+            {
+                if (_started) return;
+                _started = true;
+
+                if (ServicePointManager.DefaultConnectionLimit < 16)
+                    ServicePointManager.DefaultConnectionLimit = 16;
+
+                // Subscribe to network changes to wake the loop immediately.
+                try
+                {
+                    NetworkChange.NetworkAddressChanged += OnNetworkChanged;
+                    NetworkChange.NetworkAvailabilityChanged += OnNetworkAvailabilityChanged;
+                }
+                catch { /* non-fatal */ }
+
+                Task.Run(() => ProbeLoop());
+            }
+        }
+
+        public static void Dispose()
+        {
+            if (_disposed) return;
+            _disposed = true;
+            try
+            {
+                NetworkChange.NetworkAddressChanged -= OnNetworkChanged;
+                NetworkChange.NetworkAvailabilityChanged -= OnNetworkAvailabilityChanged;
+            }
+            catch { }
+            try { _cts.Cancel(); } catch { }
+            try { _http.Dispose(); } catch { }
+            try { _wake.Set(); } catch { }
+        }
+
+        // -------- Config --------
+
+        private static readonly TimeSpan MinRefreshIntervalWhenUp = TimeSpan.FromSeconds(3);
+        private static readonly TimeSpan MinRefreshIntervalWhenDown = TimeSpan.FromSeconds(2);
+        private static readonly TimeSpan MaxBackoffWhenDown = TimeSpan.FromSeconds(30);
+
+        private static readonly TimeSpan DnsTimeout = TimeSpan.FromMilliseconds(800);
+        private static readonly TimeSpan HttpTimeout = TimeSpan.FromMilliseconds(1200);
+
+        // First-call quick prime
+        private static readonly TimeSpan PrimeBudget = TimeSpan.FromMilliseconds(300);
+
+        // After a reconnect, probe aggressively a few times to avoid stale state
+        private const int ReconnectBurstAttempts = 3;
+        private static readonly TimeSpan ReconnectBurstDelay = TimeSpan.FromMilliseconds(300);
+
+        private const string DnsProbeHost = "dns.google";
+        private static readonly Uri NcsiUri = new Uri("http://www.msftconnecttest.com/connecttest.txt");
+
+        // -------- State --------
+
+        private static volatile bool _lastIsUp;
+        private static volatile bool _started;
+        private static volatile bool _disposed;
+        private static volatile bool _primed;
+
+        private static readonly object _startLock = new object();
+        private static readonly CancellationTokenSource _cts = new CancellationTokenSource();
+        private static readonly HttpClient _http = CreateHttpClient();
+
+        // AsyncAutoResetEvent (since we’re on .NET Fx) to wake sleep early.
+        private static readonly AsyncAutoResetEvent _wake = new AsyncAutoResetEvent();
+
+        private static DateTime _lastAddressChangeUtc = DateTime.MinValue;
+
+        // -------- Network change hooks --------
+
+        private static void OnNetworkChanged(object sender, EventArgs e)
+        {
+            _lastAddressChangeUtc = DateTime.UtcNow;
+            _wake.Set(); // wake probe loop immediately
+        }
+
+        private static void OnNetworkAvailabilityChanged(object sender, NetworkAvailabilityEventArgs e)
+        {
+            _lastAddressChangeUtc = DateTime.UtcNow;
+            _wake.Set(); // wake probe loop immediately
+        }
+
+        // -------- Prime-on-first-call --------
+
+        private static void PrimeOnce()
+        {
+            if (_primed) return;
+            lock (_startLock)
+            {
+                if (_primed) return;
+                _primed = true;
+
+                try
+                {
+                    bool isUp = false;
+                    if (HasViableLocalNetwork())
+                    {
+                        var primeCts = new CancellationTokenSource();
+                        primeCts.CancelAfter(PrimeBudget);
+                        isUp = TryDns(primeCts.Token).GetAwaiter().GetResult();
+                    }
+
+                    _lastIsUp = isUp;
+                    if (isUp) SafeRaise(true);
+                }
+                catch
+                {
+                    _lastIsUp = false;
+                }
+            }
+        }
+
+        // -------- Probe loop --------
+
+        private static async Task ProbeLoop()
+        {
+            var ct = _cts.Token;
+            var last = _lastIsUp;
+            var backoff = TimeSpan.Zero;
+            var lastHttpConfirmedUpUtc = DateTime.MinValue;
+
+            while (!ct.IsCancellationRequested)
+            {
+                bool isUp = false;
+                bool localUp = HasViableLocalNetwork();
+
+                if (localUp)
+                {
+                    // If we just had a network change, run a small aggressive burst.
+                    if ((DateTime.UtcNow - _lastAddressChangeUtc) < TimeSpan.FromSeconds(3))
+                    {
+                        for (int i = 0; i < ReconnectBurstAttempts; i++)
+                        {
+                            if (await TryDns(ct).ConfigureAwait(false))
+                            {
+                                // Confirm once via HTTP (short timeout) after DNS says yes
+                                if (await TryHttp(ct).ConfigureAwait(false))
+                                {
+                                    isUp = true;
+                                    break;
+                                }
+                            }
+                            await SleepNoThrow(ReconnectBurstDelay, ct).ConfigureAwait(false);
+                        }
+                    }
+                    else
+                    {
+                        // Regular cadence
+                        isUp = await TryDns(ct).ConfigureAwait(false);
+                        if (isUp)
+                        {
+                            bool needHttpConfirm = !last ||
+                                (DateTime.UtcNow - lastHttpConfirmedUpUtc) > TimeSpan.FromSeconds(30);
+
+                            if (needHttpConfirm)
+                            {
+                                isUp = await TryHttp(ct).ConfigureAwait(false);
+                                if (isUp) lastHttpConfirmedUpUtc = DateTime.UtcNow;
+                            }
+                        }
+                    }
+                }
+
+                if (isUp != last)
+                {
+                    _lastIsUp = isUp;
+                    last = isUp;
+                    SafeRaise(isUp);
+                }
+                else
+                {
+                    _lastIsUp = isUp;
+                }
+
+                // Compute next wait (with backoff when down)
+                TimeSpan wait;
+                if (isUp)
+                {
+                    backoff = TimeSpan.Zero;
+                    wait = MinRefreshIntervalWhenUp;
+                }
+                else
+                {
+                    backoff = backoff == TimeSpan.Zero
+                        ? MinRefreshIntervalWhenDown
+                        : TimeSpan.FromMilliseconds(Math.Min(
+                              MaxBackoffWhenDown.TotalMilliseconds,
+                              backoff.TotalMilliseconds * 2));
+                    wait = backoff;
+                }
+
+                // Wait for either the timer OR a wake signal (network change), whichever comes first
+                await WaitWithWake(wait, ct).ConfigureAwait(false);
+            }
+        }
+
+        private static async Task WaitWithWake(TimeSpan delay, CancellationToken ct)
+        {
+            // Race delay vs wake; return as soon as either completes.
+            var delayTask = Task.Delay(delay, ct);
+            var wakeTask = _wake.WaitAsync(ct);
+            var completed = await Task.WhenAny(delayTask, wakeTask).ConfigureAwait(false);
+            // No need to do anything with the result; loop will run again immediately.
+        }
+
+        // -------- Tiers --------
+
+        private static bool HasViableLocalNetwork()
+        {
+            try
+            {
+                var nics = NetworkInterface.GetAllNetworkInterfaces();
+                return nics.Any(nic =>
+                    nic.OperationalStatus == OperationalStatus.Up &&
+                    nic.NetworkInterfaceType != NetworkInterfaceType.Loopback &&
+                    nic.NetworkInterfaceType != NetworkInterfaceType.Tunnel &&
+                    nic.GetIPProperties().GatewayAddresses.Any());
+            }
+            catch
+            {
+                return false;
+            }
+        }
+
+        private static async Task<bool> TryDns(CancellationToken ct)
+        {
+            try
+            {
+                var dnsTask = Dns.GetHostEntryAsync(DnsProbeHost);
+                var delayTask = Task.Delay(DnsTimeout, ct);
+                var completed = await Task.WhenAny(dnsTask, delayTask).ConfigureAwait(false);
+                if (completed == dnsTask)
+                {
+                    var entry = await dnsTask.ConfigureAwait(false);
+                    return entry != null && entry.AddressList != null && entry.AddressList.Length > 0;
+                }
+                return false;
+            }
+            catch
+            {
+                return false;
+            }
+        }
+
+        private static async Task<bool> TryHttp(CancellationToken ct)
+        {
+            try
+            {
+                using (var cts = CancellationTokenSource.CreateLinkedTokenSource(ct))
+                {
+                    cts.CancelAfter(HttpTimeout);
+
+                    using (var headReq = new HttpRequestMessage(HttpMethod.Head, NcsiUri))
+                    using (var headResp = await _http.SendAsync(
+                        headReq, HttpCompletionOption.ResponseHeadersRead, cts.Token).ConfigureAwait(false))
+                    {
+                        if (headResp.IsSuccessStatusCode) return true;
+                    }
+
+                    using (var getReq = new HttpRequestMessage(HttpMethod.Get, NcsiUri))
+                    using (var getResp = await _http.SendAsync(
+                        getReq, HttpCompletionOption.ResponseHeadersRead, cts.Token).ConfigureAwait(false))
+                    {
+                        return getResp.IsSuccessStatusCode;
+                    }
+                }
+            }
+            catch
+            {
+                return false;
+            }
+        }
+
+        // -------- Helpers --------
+
+        private static HttpClient CreateHttpClient()
+        {
+            var handler = new HttpClientHandler
+            {
+                AllowAutoRedirect = false,
+                AutomaticDecompression = DecompressionMethods.None,
+                UseProxy = true
+            };
+            var client = new HttpClient(handler, true) { Timeout = TimeSpan.FromSeconds(2) };
+            return client;
+        }
+
+        private static void SafeRaise(bool value)
+        {
+            try { var h = StatusChanged; if (h != null) h(value); } catch { }
+        }
+
+        private static async Task SleepNoThrow(TimeSpan delay, CancellationToken ct)
+        {
+            try { await Task.Delay(delay, ct).ConfigureAwait(false); } catch { }
+        }
+
+        /// <summary>
+        /// Minimal AsyncAutoResetEvent for .NET Framework.
+        /// </summary>
+        private sealed class AsyncAutoResetEvent
+        {
+            private static readonly Task s_completed = Task.FromResult(true);
+            private readonly object _mutex = new object();
+            private TaskCompletionSource<bool> _tcs = new TaskCompletionSource<bool>();
+
+            public Task WaitAsync(CancellationToken ct)
+            {
+                lock (_mutex)
+                {
+                    if (_tcs.Task.IsCompleted)
+                    {
+                        _tcs = new TaskCompletionSource<bool>();
+                        return s_completed;
+                    }
+
+                    // Register cancellation against the current waiter
+                    var tcs = _tcs;
+                    if (ct.CanBeCanceled)
+                    {
+                        ct.Register(() =>
+                        {
+                            try { tcs.TrySetCanceled(); } catch { }
+                        });
+                    }
+                    return tcs.Task;
+                }
+            }
+
+            public void Set()
+            {
+                lock (_mutex)
+                {
+                    if (!_tcs.Task.IsCompleted)
+                    {
+                        _tcs.TrySetResult(true);
+                    }
+                    else
+                    {
+                        // already signaled; keep it signaled for the next waiter
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/Software/Visual_Studio/Tango.Telemetry/Helpers/JsonFlattener.cs b/Software/Visual_Studio/Tango.Telemetry/Helpers/JsonFlattener.cs
new file mode 100644
index 000000000..1355a8fc4
--- /dev/null
+++ b/Software/Visual_Studio/Tango.Telemetry/Helpers/JsonFlattener.cs
@@ -0,0 +1,89 @@
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Reflection;
+
+namespace Tango.Telemetry.Helpers
+{
+    internal static class JsonFlattener
+    {
+        public static string FlattenObjectToFlatJson(object obj, Formatting format)
+        {
+            var flat = new JObject();
+            FlattenRecursive(obj, flat, prefix: null);
+            return flat.ToString(format);
+        }
+
+        private static void FlattenRecursive(object obj, JObject target, string prefix)
+        {
+            if (obj == null)
+                return;
+
+            var type = obj.GetType();
+            if (type == typeof(JObject))
+            {
+                foreach (var prop in ((JObject)obj).Properties())
+                {
+                    FlattenRecursive(prop.Value, target, Combine(prefix, prop.Name));
+                }
+                return;
+            }
+
+            if (obj is JValue jVal)
+            {
+                target[Combine(prefix, "Value")] = JToken.FromObject(jVal.Value);
+                return;
+            }
+
+            if (obj is JToken jToken && jToken.Type == JTokenType.Object)
+            {
+                foreach (var prop in ((JObject)jToken).Properties())
+                {
+                    FlattenRecursive(prop.Value, target, Combine(prefix, prop.Name));
+                }
+                return;
+            }
+
+            foreach (var prop in type.GetProperties(BindingFlags.Public | BindingFlags.Instance))
+            {
+                if (!prop.CanRead) continue;
+
+                var value = prop.GetValue(obj);
+
+                if (value == null) continue;
+
+                var valueType = value.GetType();
+
+                if (IsSimpleType(valueType))
+                {
+                    target[Combine(prefix, prop.Name)] = JToken.FromObject(value);
+                }
+                else if (value is IEnumerable enumerable && !(value is string))
+                {
+                    int index = 0;
+                    foreach (var item in enumerable)
+                    {
+                        FlattenRecursive(item, target, Combine(prefix, $"{prop.Name}_{index}"));
+                        index++;
+                    }
+                }
+                else
+                {
+                    FlattenRecursive(value, target, Combine(prefix, prop.Name));
+                }
+            }
+        }
+
+        private static string Combine(string prefix, string name)
+        {
+            return string.IsNullOrEmpty(prefix) ? name : $"{prefix}_{name}";
+        }
+
+        private static bool IsSimpleType(Type type)
+        {
+            return type.IsPrimitive || type.IsValueType || type == typeof(string);
+        }
+    }
+}