Software/Visual_Studio/PPC/Tango.PPC.Common/Synchronization/DefaultMachineDataSynchronizer.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Timers;
using Tango.BL;
using Tango.PPC.Common.Web;
using System.Data.Entity;
using Tango.BL.DTO;
using Tango.PPC.Common.Connection;
using Tango.BL.Builders;
using Tango.Settings;
using Tango.Core;
using Tango.PPC.Common.Authentication;
using Tango.Logging;
using System.Diagnostics;
using Tango.BL.Enumerations;
using Tango.PPC.Common.Application;
using Tango.Core.DI;

namespace Tango.PPC.Common.Synchronization
{
    public class DefaultMachineDataSynchronizer : ExtendedObject, IMachineDataSynchronizer
    {
        private Timer _synchTimer;
        private PPCWebClient client;
        private IMachineProvider _machineProvider;
        private IAuthenticationProvider _authenticationProvider;
        private List<LogItemBase> _logs;
        private bool _synchronizedOnce;

        [TangoInject(TangoInjectMode.WhenAvailable)]
        private IPPCApplicationManager _appManager;

        public event EventHandler<SynchronizationStatusChangedEventArgs> CurrentStatusChanged;
        public event EventHandler SynchronizationStarted;
        public event EventHandler<SynchronizationEndedEventArgs> SynchronizationEnded;

        public int MaxJobs { get; set; }
        public int MaxJobRuns { get; set; }
        public int MaxMachinesEvents { get; set; }
        public int MaxOfflineUpdates { get; set; }

        private SynchronizationStatus _currentStatus;
        public SynchronizationStatus CurrentStatus
        {
            get { return _currentStatus; }
            private set { _currentStatus = value; RaisePropertyChangedAuto(); }
        }

        private SynchronizationStatus _lastStatus;
        public SynchronizationStatus LastStatus
        {
            get { return _lastStatus; }
            private set { _lastStatus = value; RaisePropertyChangedAuto(); }
        }

        public SynchronizedObservableCollection<SynchronizationStatus> StatusHistory { get; private set; }

        public TimeSpan Interval { get; set; }

        private bool _isEnabled;
        public bool IsEnabled
        {
            get { return _isEnabled; }
            set { _isEnabled = value; OnEnableChanged(); RaisePropertyChangedAuto(); }
        }

        private bool _isSynchronizing;
        public bool IsSynchronizing
        {
            get { return _isSynchronizing; }
            set { _isSynchronizing = value; RaisePropertyChangedAuto(); }
        }

        public DefaultMachineDataSynchronizer()
        {
            StatusHistory = new SynchronizedObservableCollection<SynchronizationStatus>();

            MaxJobs = 10;
            MaxJobRuns = 10;
            MaxMachinesEvents = 10;
            MaxOfflineUpdates = 10;

            var settings = SettingsManager.Default.GetOrCreate<PPCSettings>();
            Interval = settings.SynchronizationInterval;

            _synchTimer = new Timer(Interval.TotalMilliseconds);
            _synchTimer.Elapsed += _synchTimer_Elapsed;
            _synchTimer.Enabled = true;

            ExecuteNewStatus(TimeSpan.FromMinutes(2));
            LastStatus = CurrentStatus;
        }

        public DefaultMachineDataSynchronizer(PPCWebClient ppcWebClient, IMachineProvider machineProvider, IAuthenticationProvider authenticationProvider) : this()
        {
            _logs = new List<LogItemBase>();
            _machineProvider = machineProvider;
            client = new PPCWebClient(ppcWebClient, TimeSpan.FromMinutes(10));
            _authenticationProvider = authenticationProvider;

            LogManager.NewLog += LogManager_NewLog;
        }

        private void LogManager_NewLog(object sender, LogItemBase e)
        {
            if (IsSynchronizing)
            {
                _logs.Add(e);
            }
        }

        private String GetLogsStringAndClear()
        {
            String logsString = String.Join(Environment.NewLine, _logs.ToList().Select(x => x.ToString()));
            _logs.Clear();
            return logsString;
        }

        private void OnEnableChanged()
        {
            _synchTimer.Interval = Interval.TotalMilliseconds;

            if (IsEnabled)
            {
                CurrentStatus.State = SynchronizationState.Pending;
            }
            else
            {
                CurrentStatus.State = SynchronizationState.Disabled;
            }
        }

        private void _synchTimer_Elapsed(object sender, ElapsedEventArgs e)
        {
            _synchTimer.Interval = Interval.TotalMilliseconds;

            try
            {
                Synchronize().GetAwaiter().GetResult();
            }
            catch { }
        }

        private async Task<UploadMachineDataRequest> CreateUploadMachineDataRequest(bool syncJobs, bool syncDiagnostics)
        {
            UploadMachineDataRequest request = new UploadMachineDataRequest();

            using (ObservablesContext db = ObservablesContext.CreateDefault())
            {
                if (syncJobs)
                {
                    LogManager.Log("Checking Jobs...");

                    var jobs = await new JobsCollectionBuilder(db).Set(x => !x.IsSynchronized).WithSegments().WithBrushStops().Query(x => x.Take(MaxJobs).OrderByDescending(z => z.LastUpdated)).BuildListAsync();
                    List<JobDTO> dtos = new List<JobDTO>();

                    foreach (var job in jobs)
                    {
                        var dto = JobDTO.FromObservable(job);
                        request.Jobs.Add(dto);
                    }
                }

                if (syncDiagnostics)
                {
                    LogManager.Log("Checking Job Runs...");

                    var jobRuns = await db.JobRuns.Where(x => !x.IsSynchronized).Take(MaxJobRuns).OrderByDescending(x => x.LastUpdated).ToListAsync();
                    List<JobRunDTO> dtos = new List<JobRunDTO>();

                    foreach (var jobRun in jobRuns)
                    {
                        var dto = JobRunDTO.FromObservable(jobRun);
                        request.JobRuns.Add(dto);
                    }
                }

                if (syncDiagnostics)
                {
                    LogManager.Log("Checking Events...");

                    var machineEvents = await db.MachinesEvents.Where(x => !x.IsSynchronized).Take(MaxMachinesEvents).OrderByDescending(x => x.LastUpdated).ToListAsync();
                    List<MachinesEventDTO> dtos = new List<MachinesEventDTO>();

                    foreach (var machineEvent in machineEvents)
                    {
                        machineEvent.IsSynchronized = true;
                        var dto = MachinesEventDTO.FromObservable(machineEvent);
                        request.MachineEvents.Add(dto);
                    }
                }

                if (syncDiagnostics)
                {
                    LogManager.Log("Checking Offline Updates...");

                    var tangoUpdates = await db.TangoUpdates.Where(x => !x.IsSynchronized && (x.Status == (int)TangoUpdateStatuses.OfflineUpdateCompleted || x.Status == (int)TangoUpdateStatuses.OfflineUpdateFailed || x.Status == (int)TangoUpdateStatuses.OfflineFirmwareUpgradeCompleted || x.Status == (int)TangoUpdateStatuses.OfflineFirmwareUpgradeFailed)).Take(MaxOfflineUpdates).OrderByDescending(x => x.LastUpdated).ToListAsync();
                    List<TangoUpdateDTO> dtos = new List<TangoUpdateDTO>();

                    foreach (var tangoUpdate in tangoUpdates)
                    {
                        tangoUpdate.IsSynchronized = true;
                        var dto = TangoUpdateDTO.FromObservable(tangoUpdate);
                        request.OfflineUpdates.Add(dto);
                    }
                }
            }

            return request;
        }

        private async Task FinalizeMachineDataUpload(UploadMachineDataRequest request, UploadMachineDataResponse response)
        {
            using (ObservablesContext db = ObservablesContext.CreateDefault())
            {
                //Finalize jobs
                foreach (var job in request.Jobs)
                {
                    var failedJob = response.FailedJobs.SingleOrDefault(x => x.Guid == job.Guid);

                    if (failedJob == null)
                    {
                        var dbJob = await db.Jobs.SingleOrDefaultAsync(x => x.Guid == job.Guid);
                        dbJob.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - Job '{job.Name}' cannot be stored on the server due to the following reason:\n{failedJob.Reason}", LogCategory.Error);
                    }
                }

                //Finalize job runs
                foreach (var jobRun in request.JobRuns)
                {
                    var failedJobRun = response.FailedJobRuns.SingleOrDefault(x => x.Guid == jobRun.Guid);

                    if (failedJobRun == null)
                    {
                        var dbJobRun = await db.JobRuns.SingleOrDefaultAsync(x => x.Guid == jobRun.Guid);
                        dbJobRun.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - JobRun '{jobRun.ID}' cannot be stored on the server due to the following reason:\n{failedJobRun.Reason}", LogCategory.Error);
                    }
                }

                //Finalize machine events
                foreach (var machineEvent in request.MachineEvents)
                {
                    var failedMachineEvent = response.FailedMachineEvents.SingleOrDefault(x => x.Guid == machineEvent.Guid);

                    if (failedMachineEvent == null)
                    {
                        var dbMachineEvent = await db.MachinesEvents.SingleOrDefaultAsync(x => x.Guid == machineEvent.Guid);
                        dbMachineEvent.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - Event '{machineEvent.ID}' cannot be stored on the server due to the following reason:\n{failedMachineEvent.Reason}", LogCategory.Error);
                    }
                }

                //Finalize tango updates
                foreach (var tangoUpdate in request.OfflineUpdates)
                {
                    var failedTangoUpdate = response.FailedOfflineUpdates.SingleOrDefault(x => x.Guid == tangoUpdate.Guid);

                    if (failedTangoUpdate == null)
                    {
                        var dbTangoUpdate = await db.TangoUpdates.SingleOrDefaultAsync(x => x.Guid == tangoUpdate.Guid);
                        dbTangoUpdate.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - TangoUpdate '{tangoUpdate.ID}' cannot be stored on the server due to the following reason:\n{failedTangoUpdate.Reason}", LogCategory.Error);
                    }
                }

                await db.SaveChangesAsync();
            }
        }

        private async Task<DownloadMachineDataResponse> DownloadMachineData(bool syncJobs, bool syncDiagnostics)
        {
            return await client.DownloadMachineData(new DownloadMachineDataRequest()
            {
                RequestJobs = syncJobs,
                RequestJobRuns = syncDiagnostics,
                RequestMachineEvents = syncDiagnostics,
                MaxJobs = MaxJobs,
                MaxJobRuns = MaxJobRuns,
                MaxMachinesEvents = MaxMachinesEvents,
            });
        }

        private async Task<NotifyMachineDataDownloadCompletedRequest> InsertReplaceMachineData(DownloadMachineDataResponse response)
        {
            NotifyMachineDataDownloadCompletedRequest request = new NotifyMachineDataDownloadCompletedRequest();

            //Insert/Replace Jobs.
            if (response.Jobs.Count > 0)
            {
                LogManager.Log("Inserting/Replacing Jobs...");
            }
            foreach (var dto in response.Jobs)
            {
                using (ObservablesContext db = ObservablesContext.CreateDefault())
                {
                    try
                    {
                        var job = dto.ToObservable();

                        job.ID = 0;
                        job.UserGuid = _authenticationProvider.CurrentUser.Guid;
                        job.CustomerGuid = null;
                        job.IsSynchronized = true;

                        var existingJob = await db.Jobs.SingleOrDefaultAsync(x => x.Guid == job.Guid);

                        if (existingJob == null)
                        {
                            db.Jobs.Add(job);
                            await db.SaveChangesAsync();
                        }
                        else if (job.LastUpdated > existingJob.LastUpdated)
                        {
                            existingJob.Delete(db);
                            db.Jobs.Add(job);
                            await db.SaveChangesAsync();
                        }

                        request.SynchronizedJobs.Add(job.Guid);
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log($"Synchronization Error - Job '{dto.Name}' cannot be stored locally due to the following reason:\n{ex.FlattenMessage()}", LogCategory.Error);
                    }
                }
            }

            //Insert JobRuns.
            if (response.JobRuns.Count > 0)
            {
                LogManager.Log("Inserting/Replacing Job Runs...");
            }
            foreach (var dto in response.JobRuns)
            {
                using (ObservablesContext db = ObservablesContext.CreateDefault())
                {
                    try
                    {
                        var run = dto.ToObservable();
                        run.ID = 0;
                        run.IsSynchronized = true;

                        if (await db.JobRuns.SingleOrDefaultAsync(x => x.Guid == run.Guid) == null)
                        {
                            db.JobRuns.Add(run);
                            await db.SaveChangesAsync();
                        }

                        request.SynchronizedJobRuns.Add(run.Guid);
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log($"Synchronization Error - JobRun '{dto.ID}' cannot be stored locally due to the following reason:\n{ex.FlattenMessage()}", LogCategory.Error);
                    }
                }
            }

            //Insert MachineEvents.
            if (response.MachineEvents.Count > 0)
            {
                LogManager.Log("Inserting/Replacing Events...");
            }
            foreach (var dto in response.MachineEvents)
            {
                using (ObservablesContext db = ObservablesContext.CreateDefault())
                {
                    try
                    {
                        var ev = dto.ToObservable();
                        ev.ID = 0;
                        ev.UserGuid = _authenticationProvider.CurrentUser.Guid;
                        ev.IsSynchronized = true;

                        if (await db.MachinesEvents.SingleOrDefaultAsync(x => x.Guid == ev.Guid) == null)
                        {
                            db.MachinesEvents.Add(ev);
                            await db.SaveChangesAsync();
                        }

                        request.SynchronizedMachineEvents.Add(ev.Guid);
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log($"Synchronization Error - Event '{dto.ID}' cannot be stored locally due to the following reason:\n{ex.FlattenMessage()}", LogCategory.Error);
                    }
                }
            }

            return request;
        }

        public async Task Synchronize()
        {
            _synchronizedOnce = true;

            if (!IsEnabled || IsSynchronizing) return;

            var settings = SettingsManager.Default.GetOrCreate<PPCSettings>();

            var syncJobs = settings.SynchronizeJobs;
            var syncDiagnostics = settings.SynchronizeDiagnostics;

            if (!syncJobs && !syncDiagnostics) return;

            IsSynchronizing = true;
            SynchronizationStarted?.Invoke(this, new EventArgs());

            _logs.Clear();

            _synchTimer.Stop();

            LogManager.Log("Starting machine data synchronization...");
            LogManager.Log($"Synchronization interval: {Interval}.");

            CurrentStatus.StartDateTime = DateTime.Now;
            UpdateCurrentStatus(SynchronizationState.Synchronizing, "Starting synchronization...");

            Stopwatch watch = new Stopwatch();
            watch.Start();

            String notifyToken = null;

            int newChangedJobs = 0;
            int newJobRuns = 0;
            int newMachineEvents = 0;

            try
            {
                LogManager.Log("Authenticating with machine service...");

                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Authenticating with machine service...");

                if (!this.client.IsAuthenticated)
                {
                    await this.client.Login(new LoginRequest()
                    {
                        Mode = LoginMode.Machine,
                        SerialNumber = _machineProvider.Machine.SerialNumber,
                    });
                }

                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Preparing machine data for upload...");
                LogManager.Log("Preparing machine data for upload...");
                var request = await CreateUploadMachineDataRequest(syncJobs, syncDiagnostics);
                request.ApplicationVersion = _appManager.Version.ToString();
                request.FirmwareVersion = _appManager.FirmwareVersion.ToString();

                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Uploading machine data...");
                LogManager.Log($"Uploading machine data:\nJobs: {request.Jobs.Count}\nJob Runs: {request.JobRuns.Count}\nEvents: {request.MachineEvents.Count}\nOffline Updates: {request.OfflineUpdates.Count}");
                var response = await this.client.UploadMachineData(request);
                notifyToken = response.NotifyCompletedToken;
                LogManager.Log($"Upload response received:\nFailed Jobs: {response.FailedJobs.Count}\nFailed Job Runs: {response.FailedJobRuns.Count}\nFailed Events: {response.FailedMachineEvents.Count}\nFailed Offline Updates: {response.FailedOfflineUpdates.Count}");
                LogManager.Log("Finalizing upload...");
                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Finalizing upload...");
                await FinalizeMachineDataUpload(request, response);

                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Downloading machine data from service...");
                LogManager.Log("Downloading machine data from server...");
                var downloadResponse = await DownloadMachineData(syncJobs, syncDiagnostics);

                newChangedJobs = downloadResponse.Jobs.Count;
                newJobRuns = downloadResponse.JobRuns.Count;
                newMachineEvents = downloadResponse.MachineEvents.Count;

                LogManager.Log($"Download response received:\nJobs: {downloadResponse.Jobs.Count}\nJob Runs: {downloadResponse.JobRuns.Count}\nEvents: {downloadResponse.MachineEvents.Count}");
                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Updating local database...");
                LogManager.Log("Updating local database...");
                var notifyRequest = await InsertReplaceMachineData(downloadResponse);
                LogManager.Log($"Finalizing download:\nSynchronized Jobs: {notifyRequest.SynchronizedJobs.Count}\nSynchronized Job Runs: {notifyRequest.SynchronizedJobRuns.Count}\nSynchronized Events: {notifyRequest.SynchronizedMachineEvents.Count}");
                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Finalizing download...");
                var notifyResponse = await this.client.NotifyMachineDataDownloadCompleted(notifyRequest);

                if (notifyToken != null)
                {
                    try
                    {
                        await client.NotifyUpdateCompleted(new MachineUpdateCompletedRequest()
                        {
                            Token = notifyToken,
                            Status = TangoUpdateStatuses.SynchronizationCompleted,
                        });
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log(ex, "Synchronization completed successfully but an error occurred when trying to notify about the completion.");
                    }
                }

                LogManager.Log("Machine data synchronization completed successfully.");
                UpdateCurrentStatus(SynchronizationState.Completed, "Synchronization completed successfully.", null, watch.Elapsed);
            }
            catch (Exception ex)
            {
                if (notifyToken != null)
                {
                    try
                    {
                        await client.NotifyUpdateCompleted(new MachineUpdateCompletedRequest()
                        {
                            Token = notifyToken,
                            Status = TangoUpdateStatuses.SynchronizationFailed,
                            FailedReason = ex.FlattenMessage(),
                            FailedLog = GetLogsStringAndClear(),
                        });
                    }
                    catch (Exception ee)
                    {
                        LogManager.Log(ee, "Synchronization completed successfully but an error occurred when trying to notify about the completion.");
                    }
                }

                UpdateCurrentStatus(SynchronizationState.Failed, "Synchronization failed.", ex.FlattenMessage(), watch.Elapsed);
                throw LogManager.Log(ex, "Error occurred while synchronizing machine data.");
            }
            finally
            {
                watch.Stop();
                LogManager.Log($"Synchronization duration: {watch.Elapsed}.");
                LastStatus = CurrentStatus;
                CreateNewStatus();
                IsSynchronizing = false;
                SynchronizationEnded?.Invoke(this, new SynchronizationEndedEventArgs()
                {
                    NewChangedJobs = newChangedJobs,
                    NewJobRuns = newJobRuns,
                    NewMachineEvents = newMachineEvents,
                });
            }

            _synchTimer.Start();
        }

        private void CreateNewStatus()
        {
            CurrentStatus = new SynchronizationStatus();
            CurrentStatus.State = SynchronizationState.Pending;
            CurrentStatus.StartDateTime = DateTime.Now.Add(Interval);
            StatusHistory.Insert(0, CurrentStatus);
        }

        private async void ExecuteNewStatus(TimeSpan delay)
        {
            CurrentStatus = new SynchronizationStatus();
            CurrentStatus.State = SynchronizationState.Pending;
            CurrentStatus.StartDateTime = DateTime.Now.Add(delay);
            StatusHistory.Insert(0, CurrentStatus);
            await Task.Delay(delay);
            try
            {
                if (!_synchronizedOnce)
                {
                    await Synchronize();
                }
            }
            catch { }
        }

        private void UpdateCurrentStatus(SynchronizationState state, String message, String errorReason = null, TimeSpan? duration = null)
        {
            CurrentStatus.State = state;
            CurrentStatus.Message = message;
            CurrentStatus.ErrorReason = errorReason;
            CurrentStatus.Duration = duration;
            CurrentStatusChanged?.Invoke(this, new SynchronizationStatusChangedEventArgs()
            {
                Status = CurrentStatus,
            });
        }
    }
}