aboutsummaryrefslogtreecommitdiffstats
path: root/Software/Visual_Studio/PPC/Tango.PPC.Common/Synchronization/DefaultMachineDataSynchronizer.cs
blob: 5a951e9faf766c820cba5421d296c7ef50c17e9c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
function modelConvert(model, outname)
%% script for converting Piotr's matlab model into YAML format

outfile = fopen(outname, 'w');

fprintf(outfile, '%%YAML:1.0\n\n');

fprintf(outfile, ['options:\n'...
                  '    numberOfTrees: 8\n'...
                  '    numberOfTreesToEvaluate: 4\n'...
                  '    selfsimilarityGridSize: 5\n'...
                  '    stride: 2\n'...
                  '    shrinkNumber: 2\n'...
                  '    patchSize: 32\n'...
                  '    patchInnerSize: 16\n'...
                  '    numberOfGradientOrientations: 4\n'...
                  '    gradientSmoothingRadius: 0\n'...
                  '    regFeatureSmoothingRadius: 2\n'...
                  '    ssFeatureSmoothingRadius: 8\n'...
                  '    gradientNormalizationRadius: 4\n\n']);

fprintf(outfile, 'childs:\n');
printToYML(outfile, model.child', 0);

fprintf(outfile, 'featureIds:\n');
printToYML(outfile, model.fids', 0);

fprintf(outfile, 'thresholds:\n');
printToYML(outfile, model.thrs', 0);

N = 1000;
fprintf(outfile, 'edgeBoundaries:\n');
printToYML(outfile, model.eBnds, N);

fprintf(outfile, 'edgeBins:\n');
printToYML(outfile, model.eBins, N);

fclose(outfile);
gzip(outname);

end

function printToYML(outfile, A, N)
%% append matrix A to outfile as
%%    - [a11, a12, a13, a14, ..., a1n]
%%    - [a21, a22, a23, a24, ..., a2n]
%%    ...
%%
%% if size(A, 2) == 1, A is printed by N elemnent per row

    if (length(size(A)) ~= 2)
        error('printToYML: second-argument matrix should have two dimensions');
    end

    if (size(A,2) ~= 1)
        for i=1:size(A,1)
            fprintf(outfile, '    - [');
            fprintf(outfile, '%d,', A(i, 1:end-1));
            fprintf(outfile, '%d]\n', A(i, end));
        end
    else
        len = length(A);
        for i=1:ceil(len/N)
            first = (i-1)*N + 1;
             last = min(i*N, len) - 1;

            fprintf(outfile, '    - [');
            fprintf(outfile, '%d,', A(first:last));
            fprintf(outfile, '%d]\n', A(last + 1));
        end
    end
    fprintf(outfile, '\n');
end
600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Timers;
using Tango.BL;
using Tango.PPC.Common.Web;
using System.Data.Entity;
using Tango.BL.DTO;
using Tango.PPC.Common.Connection;
using Tango.BL.Builders;
using Tango.Settings;
using Tango.Core;
using Tango.PPC.Common.Authentication;
using Tango.Logging;
using System.Diagnostics;
using Tango.BL.Enumerations;
using Tango.PPC.Common.Application;
using Tango.Core.DI;

namespace Tango.PPC.Common.Synchronization
{
    public class DefaultMachineDataSynchronizer : ExtendedObject, IMachineDataSynchronizer
    {
        private Timer _synchTimer;
        private PPCWebClient client;
        private IMachineProvider _machineProvider;
        private IAuthenticationProvider _authenticationProvider;
        private List<LogItemBase> _logs;
        private bool _synchronizedOnce;

        [TangoInject(TangoInjectMode.WhenAvailable)]
        private IPPCApplicationManager _appManager;

        public event EventHandler<SynchronizationStatusChangedEventArgs> CurrentStatusChanged;
        public event EventHandler SynchronizationStarted;
        public event EventHandler<SynchronizationEndedEventArgs> SynchronizationEnded;

        public int MaxJobs { get; set; }
        public int MaxJobRuns { get; set; }
        public int MaxMachinesEvents { get; set; }
        public int MaxOfflineUpdates { get; set; }
        public int MaxDataStoreItems { get; set; }

        private SynchronizationStatus _currentStatus;
        public SynchronizationStatus CurrentStatus
        {
            get { return _currentStatus; }
            private set { _currentStatus = value; RaisePropertyChangedAuto(); }
        }

        private SynchronizationStatus _lastStatus;
        public SynchronizationStatus LastStatus
        {
            get { return _lastStatus; }
            private set { _lastStatus = value; RaisePropertyChangedAuto(); }
        }

        public SynchronizedObservableCollection<SynchronizationStatus> StatusHistory { get; private set; }

        public TimeSpan Interval { get; set; }

        private bool _isEnabled;
        public bool IsEnabled
        {
            get { return _isEnabled; }
            set { _isEnabled = value; OnEnableChanged(); RaisePropertyChangedAuto(); }
        }

        private bool _isSynchronizing;
        public bool IsSynchronizing
        {
            get { return _isSynchronizing; }
            set { _isSynchronizing = value; RaisePropertyChangedAuto(); }
        }

        public DefaultMachineDataSynchronizer()
        {
            StatusHistory = new SynchronizedObservableCollection<SynchronizationStatus>();

            MaxJobs = 10;
            MaxJobRuns = 10;
            MaxMachinesEvents = 10;
            MaxOfflineUpdates = 10;
            MaxDataStoreItems = 100;

            var settings = SettingsManager.Default.GetOrCreate<PPCSettings>();
            Interval = settings.SynchronizationInterval;

            _synchTimer = new Timer(Interval.TotalMilliseconds);
            _synchTimer.Elapsed += _synchTimer_Elapsed;
            _synchTimer.Enabled = true;

            ExecuteNewStatus(TimeSpan.FromMinutes(2));
            LastStatus = CurrentStatus;
        }

        public DefaultMachineDataSynchronizer(PPCWebClient ppcWebClient, IMachineProvider machineProvider, IAuthenticationProvider authenticationProvider) : this()
        {
            _logs = new List<LogItemBase>();
            _machineProvider = machineProvider;
            client = new PPCWebClient(ppcWebClient, TimeSpan.FromMinutes(10));
            _authenticationProvider = authenticationProvider;

            LogManager.NewLog += LogManager_NewLog;
        }

        private void LogManager_NewLog(object sender, LogItemBase e)
        {
            if (IsSynchronizing)
            {
                _logs.Add(e);
            }
        }

        private String GetLogsStringAndClear()
        {
            String logsString = String.Join(Environment.NewLine, _logs.ToList().Select(x => x.ToString()));
            _logs.Clear();
            return logsString;
        }

        private void OnEnableChanged()
        {
            _synchTimer.Interval = Interval.TotalMilliseconds;

            if (IsEnabled)
            {
                CurrentStatus.State = SynchronizationState.Pending;
            }
            else
            {
                CurrentStatus.State = SynchronizationState.Disabled;
            }
        }

        private void _synchTimer_Elapsed(object sender, ElapsedEventArgs e)
        {
            _synchTimer.Interval = Interval.TotalMilliseconds;

            try
            {
                Synchronize().GetAwaiter().GetResult();
            }
            catch { }
        }

        private async Task<UploadMachineDataRequest> CreateUploadMachineDataRequest(bool syncJobs, bool syncDiagnostics)
        {
            UploadMachineDataRequest request = new UploadMachineDataRequest();

            using (ObservablesContext db = ObservablesContext.CreateDefault())
            {
                if (syncJobs)
                {
                    LogManager.Log("Checking Jobs...");

                    var jobs = await new JobsCollectionBuilder(db).Set(x => !x.IsSynchronized).WithSegments().WithBrushStops().Query(x => x.Take(MaxJobs).OrderByDescending(z => z.LastUpdated)).BuildListAsync();
                    List<JobDTO> dtos = new List<JobDTO>();

                    foreach (var job in jobs)
                    {
                        var dto = JobDTO.FromObservable(job);
                        request.Jobs.Add(dto);
                    }
                }

                if (syncDiagnostics)
                {
                    LogManager.Log("Checking Job Runs...");

                    var jobRuns = await db.JobRuns.Where(x => !x.IsSynchronized).Take(MaxJobRuns).OrderByDescending(x => x.LastUpdated).ToListAsync();
                    List<JobRunDTO> dtos = new List<JobRunDTO>();

                    foreach (var jobRun in jobRuns)
                    {
                        var dto = JobRunDTO.FromObservable(jobRun);
                        request.JobRuns.Add(dto);
                    }
                }

                if (syncDiagnostics)
                {
                    LogManager.Log("Checking Events...");

                    var machineEvents = await db.MachinesEvents.Where(x => !x.IsSynchronized).Take(MaxMachinesEvents).OrderByDescending(x => x.LastUpdated).ToListAsync();
                    List<MachinesEventDTO> dtos = new List<MachinesEventDTO>();

                    foreach (var machineEvent in machineEvents)
                    {
                        machineEvent.IsSynchronized = true;
                        var dto = MachinesEventDTO.FromObservable(machineEvent);
                        request.MachineEvents.Add(dto);
                    }
                }

                if (syncDiagnostics)
                {
                    LogManager.Log("Checking Offline Updates...");

                    var tangoUpdates = await db.TangoUpdates.Where(x => !x.IsSynchronized && (x.Status == (int)TangoUpdateStatuses.OfflineUpdateCompleted || x.Status == (int)TangoUpdateStatuses.OfflineUpdateFailed || x.Status == (int)TangoUpdateStatuses.OfflineFirmwareUpgradeCompleted || x.Status == (int)TangoUpdateStatuses.OfflineFirmwareUpgradeFailed)).Take(MaxOfflineUpdates).OrderByDescending(x => x.LastUpdated).ToListAsync();
                    List<TangoUpdateDTO> dtos = new List<TangoUpdateDTO>();

                    foreach (var tangoUpdate in tangoUpdates)
                    {
                        tangoUpdate.IsSynchronized = true;
                        var dto = TangoUpdateDTO.FromObservable(tangoUpdate);
                        request.OfflineUpdates.Add(dto);
                    }
                }

                {   //Always synchronize data store items
                    LogManager.Log("Checking Data Store Items...");

                    var dataStoreItems = await db.DataStoreItems.Where(x => !x.IsSynchronized).Take(MaxDataStoreItems).ToListAsync();
                    List<DataStoreItemDTO> dtos = new List<DataStoreItemDTO>();

                    foreach (var item in dataStoreItems)
                    {
                        item.IsSynchronized = true;
                        var dto = DataStoreItemDTO.FromObservable(item);
                        dto.MachineGuid = null;
                        request.DataStoreItems.Add(dto);
                    }
                }
            }

            return request;
        }

        private async Task FinalizeMachineDataUpload(UploadMachineDataRequest request, UploadMachineDataResponse response)
        {
            using (ObservablesContext db = ObservablesContext.CreateDefault())
            {
                //Finalize jobs
                foreach (var job in request.Jobs)
                {
                    var failedJob = response.FailedJobs.SingleOrDefault(x => x.Guid == job.Guid);

                    if (failedJob == null)
                    {
                        var dbJob = await db.Jobs.SingleOrDefaultAsync(x => x.Guid == job.Guid);
                        dbJob.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - Job '{job.Name}' cannot be stored on the server due to the following reason:\n{failedJob.Reason}", LogCategory.Error);
                    }
                }

                //Finalize job runs
                foreach (var jobRun in request.JobRuns)
                {
                    var failedJobRun = response.FailedJobRuns.SingleOrDefault(x => x.Guid == jobRun.Guid);

                    if (failedJobRun == null)
                    {
                        var dbJobRun = await db.JobRuns.SingleOrDefaultAsync(x => x.Guid == jobRun.Guid);
                        dbJobRun.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - JobRun '{jobRun.ID}' cannot be stored on the server due to the following reason:\n{failedJobRun.Reason}", LogCategory.Error);
                    }
                }

                //Finalize machine events
                foreach (var machineEvent in request.MachineEvents)
                {
                    var failedMachineEvent = response.FailedMachineEvents.SingleOrDefault(x => x.Guid == machineEvent.Guid);

                    if (failedMachineEvent == null)
                    {
                        var dbMachineEvent = await db.MachinesEvents.SingleOrDefaultAsync(x => x.Guid == machineEvent.Guid);
                        dbMachineEvent.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - Event '{machineEvent.ID}' cannot be stored on the server due to the following reason:\n{failedMachineEvent.Reason}", LogCategory.Error);
                    }
                }

                //Finalize tango updates
                foreach (var tangoUpdate in request.OfflineUpdates)
                {
                    var failedTangoUpdate = response.FailedOfflineUpdates.SingleOrDefault(x => x.Guid == tangoUpdate.Guid);

                    if (failedTangoUpdate == null)
                    {
                        var dbTangoUpdate = await db.TangoUpdates.SingleOrDefaultAsync(x => x.Guid == tangoUpdate.Guid);
                        dbTangoUpdate.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - TangoUpdate '{tangoUpdate.ID}' cannot be stored on the server due to the following reason:\n{failedTangoUpdate.Reason}", LogCategory.Error);
                    }
                }

                //Finalize data store items
                foreach (var dataStoreItem in request.DataStoreItems)
                {
                    var failedDataStoreItem = response.FailedDataStoreItems.SingleOrDefault(x => x.Guid == dataStoreItem.Guid);

                    if (failedDataStoreItem == null)
                    {
                        var dbDataStoreItem = await db.DataStoreItems.SingleOrDefaultAsync(x => x.Guid == dataStoreItem.Guid);
                        dbDataStoreItem.IsSynchronized = true;
                    }
                    else
                    {
                        LogManager.Log($"Synchronization Error - DataStoreItem '{dataStoreItem.Key}' cannot be stored on the server due to the following reason:\n{failedDataStoreItem.Reason}", LogCategory.Error);
                    }
                }

                await db.SaveChangesAsync();
            }
        }

        private async Task<DownloadMachineDataResponse> DownloadMachineData(bool syncJobs, bool syncDiagnostics)
        {
            return await client.DownloadMachineData(new DownloadMachineDataRequest()
            {
                RequestJobs = syncJobs,
                RequestJobRuns = syncDiagnostics,
                RequestMachineEvents = syncDiagnostics,
                RequestDataStoreItems = true,
                MaxJobs = MaxJobs,
                MaxJobRuns = MaxJobRuns,
                MaxMachinesEvents = MaxMachinesEvents,
                MaxDataStoreItems = MaxDataStoreItems,
            });
        }

        private async Task<NotifyMachineDataDownloadCompletedRequest> InsertReplaceMachineData(DownloadMachineDataResponse response)
        {
            NotifyMachineDataDownloadCompletedRequest request = new NotifyMachineDataDownloadCompletedRequest();

            //Insert/Replace Jobs.
            if (response.Jobs.Count > 0)
            {
                LogManager.Log("Inserting/Replacing Jobs...");
            }
            foreach (var dto in response.Jobs)
            {
                using (ObservablesContext db = ObservablesContext.CreateDefault())
                {
                    try
                    {
                        var job = dto.ToObservable();

                        job.ID = 0;
                        job.UserGuid = null;
                        job.CustomerGuid = null;
                        job.IsSynchronized = true;

                        var existingJob = await db.Jobs.SingleOrDefaultAsync(x => x.Guid == job.Guid);

                        if (existingJob == null)
                        {
                            db.Jobs.Add(job);
                            await db.SaveChangesAsync();
                        }
                        else if (job.LastUpdated > existingJob.LastUpdated)
                        {
                            existingJob.Delete(db);
                            db.Jobs.Add(job);
                            await db.SaveChangesAsync();
                        }

                        request.SynchronizedJobs.Add(job.Guid);
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log($"Synchronization Error - Job '{dto.Name}' cannot be stored locally due to the following reason:\n{ex.FlattenMessage()}", LogCategory.Error);
                    }
                }
            }

            //Insert/Update Data Store Items.
            if (response.DataStoreItems.Count > 0)
            {
                LogManager.Log("Inserting/Updating Data Store Items...");
            }
            foreach (var dto in response.DataStoreItems)
            {
                using (ObservablesContext db = ObservablesContext.CreateDefault())
                {
                    try
                    {
                        var dataStoreItem = dto.ToObservable();

                        dataStoreItem.ID = 0;
                        dataStoreItem.MachineGuid = null;
                        dataStoreItem.IsSynchronized = true;

                        var existingItem = db.DataStoreItems.SingleOrDefault(x => x.Guid == dataStoreItem.Guid);

                        if (existingItem == null)
                        {
                            db.DataStoreItems.Add(dataStoreItem);
                            db.SaveChanges();
                        }
                        else if (dataStoreItem.LastUpdated >= existingItem.LastUpdated)
                        {
                            existingItem.DataType = dataStoreItem.DataType;
                            existingItem.Value = dataStoreItem.Value;
                            existingItem.IsSynchronized = true;
                            existingItem.LastUpdated = dataStoreItem.LastUpdated;
                            db.SaveChanges();
                        }

                        request.SynchronizedDataStoreItems.Add(dataStoreItem.Guid);
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log($"Synchronization Error - DataStoreItem '{dto.Key}' cannot be stored locally due to the following reason:\n{ex.FlattenMessage()}", LogCategory.Error);
                    }
                }
            }

            //Insert JobRuns.
            if (response.JobRuns.Count > 0)
            {
                LogManager.Log("Inserting/Replacing Job Runs...");
            }
            foreach (var dto in response.JobRuns)
            {
                using (ObservablesContext db = ObservablesContext.CreateDefault())
                {
                    try
                    {
                        var run = dto.ToObservable();
                        run.ID = 0;
                        run.IsSynchronized = true;

                        if (await db.JobRuns.SingleOrDefaultAsync(x => x.Guid == run.Guid) == null)
                        {
                            db.JobRuns.Add(run);
                            await db.SaveChangesAsync();
                        }

                        request.SynchronizedJobRuns.Add(run.Guid);
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log($"Synchronization Error - JobRun '{dto.ID}' cannot be stored locally due to the following reason:\n{ex.FlattenMessage()}", LogCategory.Error);
                    }
                }
            }

            //Insert MachineEvents.
            if (response.MachineEvents.Count > 0)
            {
                LogManager.Log("Inserting/Replacing Events...");
            }
            foreach (var dto in response.MachineEvents)
            {
                using (ObservablesContext db = ObservablesContext.CreateDefault())
                {
                    try
                    {
                        var ev = dto.ToObservable();
                        ev.ID = 0;
                        ev.UserGuid = null;
                        ev.IsSynchronized = true;

                        if (await db.MachinesEvents.SingleOrDefaultAsync(x => x.Guid == ev.Guid) == null)
                        {
                            db.MachinesEvents.Add(ev);
                            await db.SaveChangesAsync();
                        }

                        request.SynchronizedMachineEvents.Add(ev.Guid);
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log($"Synchronization Error - Event '{dto.ID}' cannot be stored locally due to the following reason:\n{ex.FlattenMessage()}", LogCategory.Error);
                    }
                }
            }

            return request;
        }

        public async Task Synchronize()
        {
            _synchronizedOnce = true;

            if (!IsEnabled || IsSynchronizing) return;

            var settings = SettingsManager.Default.GetOrCreate<PPCSettings>();

            var syncJobs = settings.SynchronizeJobs;
            var syncDiagnostics = settings.SynchronizeDiagnostics;

            IsSynchronizing = true;
            SynchronizationStarted?.Invoke(this, new EventArgs());

            _logs.Clear();

            _synchTimer.Stop();

            LogManager.Log("Starting machine data synchronization...");
            LogManager.Log($"Synchronization interval: {Interval}.");

            CurrentStatus.StartDateTime = DateTime.Now;
            UpdateCurrentStatus(SynchronizationState.Synchronizing, "Starting synchronization...");

            Stopwatch watch = new Stopwatch();
            watch.Start();

            String notifyToken = null;

            int newChangedJobs = 0;
            int newJobRuns = 0;
            int newMachineEvents = 0;

            try
            {
                LogManager.Log("Authenticating with machine service...");

                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Authenticating with machine service...");

                if (!this.client.IsAuthenticated)
                {
                    await this.client.Login(new LoginRequest()
                    {
                        Mode = LoginMode.Machine,
                        MachineGuid = _machineProvider.Machine.Guid,
                    });
                }

                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Preparing machine data for upload...");
                LogManager.Log("Preparing machine data for upload...");
                var request = await CreateUploadMachineDataRequest(syncJobs, syncDiagnostics);
                request.ApplicationVersion = _appManager.Version.ToString();
                request.FirmwareVersion = _appManager.FirmwareVersion.ToString();

                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Uploading machine data...");
                LogManager.Log($"Uploading machine data:\nJobs: {request.Jobs.Count}\nJob Runs: {request.JobRuns.Count}\nEvents: {request.MachineEvents.Count}\nOffline Updates: {request.OfflineUpdates.Count}");
                var response = await this.client.UploadMachineData(request);
                notifyToken = response.NotifyCompletedToken;
                LogManager.Log($"Upload response received:\nFailed Jobs: {response.FailedJobs.Count}\nFailed Job Runs: {response.FailedJobRuns.Count}\nFailed Events: {response.FailedMachineEvents.Count}\nFailed Offline Updates: {response.FailedOfflineUpdates.Count}");
                LogManager.Log("Finalizing upload...");
                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Finalizing upload...");
                await FinalizeMachineDataUpload(request, response);

                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Downloading machine data from service...");
                LogManager.Log("Downloading machine data from server...");
                var downloadResponse = await DownloadMachineData(syncJobs, syncDiagnostics);

                newChangedJobs = downloadResponse.Jobs.Count;
                newJobRuns = downloadResponse.JobRuns.Count;
                newMachineEvents = downloadResponse.MachineEvents.Count;

                LogManager.Log($"Download response received:\nJobs: {downloadResponse.Jobs.Count}\nJob Runs: {downloadResponse.JobRuns.Count}\nEvents: {downloadResponse.MachineEvents.Count}");
                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Updating local database...");
                LogManager.Log("Updating local database...");
                var notifyRequest = await InsertReplaceMachineData(downloadResponse);
                LogManager.Log($"Finalizing download:\nSynchronized Jobs: {notifyRequest.SynchronizedJobs.Count}\nSynchronized Job Runs: {notifyRequest.SynchronizedJobRuns.Count}\nSynchronized Events: {notifyRequest.SynchronizedMachineEvents.Count}");
                UpdateCurrentStatus(SynchronizationState.Synchronizing, "Finalizing download...");
                var notifyResponse = await this.client.NotifyMachineDataDownloadCompleted(notifyRequest);

                if (notifyToken != null)
                {
                    try
                    {
                        await client.NotifyUpdateCompleted(new MachineUpdateCompletedRequest()
                        {
                            Token = notifyToken,
                            Status = TangoUpdateStatuses.SynchronizationCompleted,
                        });
                    }
                    catch (Exception ex)
                    {
                        LogManager.Log(ex, "Synchronization completed successfully but an error occurred when trying to notify about the completion.");
                    }
                }

                LogManager.Log("Machine data synchronization completed successfully.");
                UpdateCurrentStatus(SynchronizationState.Completed, "Synchronization completed successfully.", null, watch.Elapsed);
            }
            catch (Exception ex)
            {
                if (notifyToken != null)
                {
                    try
                    {
                        await client.NotifyUpdateCompleted(new MachineUpdateCompletedRequest()
                        {
                            Token = notifyToken,
                            Status = TangoUpdateStatuses.SynchronizationFailed,
                            FailedReason = ex.FlattenMessage(),
                            FailedLog = GetLogsStringAndClear(),
                        });
                    }
                    catch (Exception ee)
                    {
                        LogManager.Log(ee, "Synchronization completed successfully but an error occurred when trying to notify about the completion.");
                    }
                }

                UpdateCurrentStatus(SynchronizationState.Failed, "Synchronization failed.", ex.FlattenMessage(), watch.Elapsed);
                throw LogManager.Log(ex, "Error occurred while synchronizing machine data.");
            }
            finally
            {
                watch.Stop();
                LogManager.Log($"Synchronization duration: {watch.Elapsed}.");
                LastStatus = CurrentStatus;
                CreateNewStatus();
                IsSynchronizing = false;
                SynchronizationEnded?.Invoke(this, new SynchronizationEndedEventArgs()
                {
                    NewChangedJobs = newChangedJobs,
                    NewJobRuns = newJobRuns,
                    NewMachineEvents = newMachineEvents,
                });
            }

            _synchTimer.Start();
        }

        private void CreateNewStatus()
        {
            CurrentStatus = new SynchronizationStatus();
            CurrentStatus.State = SynchronizationState.Pending;
            CurrentStatus.StartDateTime = DateTime.Now.Add(Interval);
            StatusHistory.Insert(0, CurrentStatus);
        }

        private async void ExecuteNewStatus(TimeSpan delay)
        {
            CurrentStatus = new SynchronizationStatus();
            CurrentStatus.State = SynchronizationState.Pending;
            CurrentStatus.StartDateTime = DateTime.Now.Add(delay);
            StatusHistory.Insert(0, CurrentStatus);
            await Task.Delay(delay);
            try
            {
                if (!_synchronizedOnce)
                {
                    await Synchronize();
                }
            }
            catch { }
        }

        private void UpdateCurrentStatus(SynchronizationState state, String message, String errorReason = null, TimeSpan? duration = null)
        {
            CurrentStatus.State = state;
            CurrentStatus.Message = message;
            CurrentStatus.ErrorReason = errorReason;
            CurrentStatus.Duration = duration;
            CurrentStatusChanged?.Invoke(this, new SynchronizationStatusChangedEventArgs()
            {
                Status = CurrentStatus,
            });
        }
    }
}