using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; using Microsoft.Azure.Management.AppService.Fluent; using Microsoft.Azure.Management.Fluent; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; using Tango.AzureUtils.Database; using Tango.BL.Entities; using Tango.BL.Enumerations; using Tango.Core.IO; namespace Tango.AzureUtils.Storage { public class StorageManager : AzureUtilsComponentBase { private String _account; private CloudBlobClient _client; private DatabaseManager _databaseManager; public StorageManager(IAzure azure) : base(azure) { _databaseManager = CreateManager(); } public Task Connect(String account) { OnProgress(AzureUtilsStage.Storage, $"Connecting to storage account..."); _account = account; var targetAccount = CloudStorageAccount.Parse(_account); _client = targetAccount.CreateCloudBlobClient(); return Task.FromResult(true); } public CloudBlobClient GetClient() { return _client; } private async Task CreateEmptyBlob(CloudBlobContainer container, String name) { OnProgress(AzureUtilsStage.Storage, $"Creating blob '{name}'..."); CloudBlockBlob targetBlob = container.GetBlockBlobReference(name); using (MemoryStream ms = new MemoryStream()) { await targetBlob.UploadFromStreamAsync(ms);//Empty memory stream. Will create an empty blob. } return targetBlob; } public async Task CreateContainer(String name) { OnProgress(AzureUtilsStage.Storage, $"Creating storage container '{name}'..."); var container = _client.GetContainerReference(name); await container.CreateIfNotExistsAsync(); return container; } public async Task RemoveContainer(String name) { OnProgress(AzureUtilsStage.Storage, $"Removing storage container '{name}'..."); var container = _client.GetContainerReference(name); await container.DeleteAsync(); } public async Task RemoveBlob(String containerName, String blobName) { OnProgress(AzureUtilsStage.Storage, $"Removing blob '{blobName}'..."); var container = _client.GetContainerReference(containerName); var blob = container.GetBlockBlobReference(blobName); await blob.DeleteAsync(); } public async Task UpgradeTangoStorage(IWebAppBase sourceApp, IWebAppBase targetApp, MachineTypes machineType, String versionTag) { OnProgress(AzureUtilsStage.Storage, $"Retrieving source and target settings..."); var sourceSettings = await sourceApp.GetMachineServiceSettingsAsync(); var targetSettings = await targetApp.GetMachineServiceSettingsAsync(); var latestTangoVersion = await _databaseManager.GetLatestTangoVersion(sourceApp, machineType, versionTag); OnProgress(AzureUtilsStage.Storage, $"Upgrading {machineType.ToDescription()} version storage..."); var sourceAccount = CloudStorageAccount.Parse(sourceSettings.STORAGE_ACCOUNT); var sourceClient = sourceAccount.CreateCloudBlobClient(); var targetAccount = CloudStorageAccount.Parse(targetSettings.STORAGE_ACCOUNT); var targetClient = targetAccount.CreateCloudBlobClient(); var sourceTangoContainer = sourceClient.GetContainerReference(sourceSettings.TANGO_VERSIONS_CONTAINER); var targetTangoContainer = targetClient.GetContainerReference(targetSettings.TANGO_VERSIONS_CONTAINER); var sourceTangoBlob = sourceTangoContainer.GetBlockBlobReference(latestTangoVersion.BlobName); var sourceTangoInstallerBlob = sourceTangoContainer.GetBlockBlobReference(latestTangoVersion.InstallerBlobName); var targetTangoBlob = await CreateEmptyBlob(targetTangoContainer, sourceTangoBlob.Name); var targetTangoInstallerBlob = await CreateEmptyBlob(targetTangoContainer, sourceTangoInstallerBlob.Name); await Task.Factory.StartNew(() => { targetTangoBlob.StartCopy(sourceTangoBlob); targetTangoInstallerBlob.StartCopy(sourceTangoInstallerBlob); }); } public async Task UpgradeFSEStorage(IWebAppBase sourceApp, IWebAppBase targetApp, FSEBuildVariants build) { OnProgress(AzureUtilsStage.Storage, $"Retrieving source and target settings..."); var sourceSettings = await sourceApp.GetMachineServiceSettingsAsync(); var targetSettings = await targetApp.GetMachineServiceSettingsAsync(); var latestFSEVersion = await _databaseManager.GetLatestFSEVersion(sourceApp, build); OnProgress(AzureUtilsStage.Storage, $"Upgrading {build} version storage..."); var sourceAccount = CloudStorageAccount.Parse(sourceSettings.STORAGE_ACCOUNT); var sourceClient = sourceAccount.CreateCloudBlobClient(); var targetAccount = CloudStorageAccount.Parse(targetSettings.STORAGE_ACCOUNT); var targetClient = targetAccount.CreateCloudBlobClient(); var sourceFSEContainer = sourceClient.GetContainerReference(sourceSettings.FSE_VERSIONS_CONTAINER); var targetFSEContainer = targetClient.GetContainerReference(targetSettings.FSE_VERSIONS_CONTAINER); //var sourceFSEBlob = sourceFSEContainer.GetBlockBlobReference(latestPPCVersion.BlobName); var sourceFSEInstallerBlob = sourceFSEContainer.GetBlockBlobReference(latestFSEVersion.InstallerBlobName); //var targetFSEBlob = await CreateEmptyBlob(targetFSEContainer, sourceFSEBlob.Name); var targetFSEInstallerBlob = await CreateEmptyBlob(targetFSEContainer, sourceFSEInstallerBlob.Name); await Task.Factory.StartNew(() => { //targetFSEBlob.StartCopy(sourceFSEBlob); targetFSEInstallerBlob.StartCopy(sourceFSEInstallerBlob); }); } public async Task UpgradeMachineStudioStorage(IWebAppBase sourceApp, IWebAppBase targetApp) { OnProgress(AzureUtilsStage.Storage, $"Retrieving source and target settings..."); var latestMachineStudioVersion = await _databaseManager.GetLatestMachineStudioVersion(sourceApp); var sourceSettings = await sourceApp.GetMachineServiceSettingsAsync(); var targetSettings = await targetApp.GetMachineServiceSettingsAsync(); OnProgress(AzureUtilsStage.Storage, $"Upgrading Machine Studio version storage..."); var sourceAccount = CloudStorageAccount.Parse(sourceSettings.STORAGE_ACCOUNT); var sourceClient = sourceAccount.CreateCloudBlobClient(); var targetAccount = CloudStorageAccount.Parse(targetSettings.STORAGE_ACCOUNT); var targetClient = targetAccount.CreateCloudBlobClient(); var sourceMachineStudioContainer = sourceClient.GetContainerReference(sourceSettings.MACHINE_STUDIO_VERSIONS_CONTAINER); var targetMachineStudioContainer = targetClient.GetContainerReference(targetSettings.MACHINE_STUDIO_VERSIONS_CONTAINER); var sourceMachineStudioBlob = sourceMachineStudioContainer.GetBlockBlobReference(latestMachineStudioVersion.BlobName); var sourceMachineStudioInstallerBlob = sourceMachineStudioContainer.GetBlockBlobReference(latestMachineStudioVersion.InstallerBlobName); var targetMachineStudioBlob = await CreateEmptyBlob(targetMachineStudioContainer, sourceMachineStudioBlob.Name); var targetMachineStudioInstallerBlob = await CreateEmptyBlob(targetMachineStudioContainer, sourceMachineStudioInstallerBlob.Name); await Task.Factory.StartNew(() => { targetMachineStudioBlob.StartCopy(sourceMachineStudioBlob); targetMachineStudioInstallerBlob.StartCopy(sourceMachineStudioInstallerBlob); }); } public async Task DowngradeMachineStudioStorage(IWebAppBase app) { OnProgress(AzureUtilsStage.Storage, $"Retrieving settings..."); var latestMachineStudioVersion = await _databaseManager.GetLatestMachineStudioVersion(app); var settings = await app.GetMachineServiceSettingsAsync(); await RemoveBlob(settings.MACHINE_STUDIO_VERSIONS_CONTAINER, latestMachineStudioVersion.BlobName); await RemoveBlob(settings.MACHINE_STUDIO_VERSIONS_CONTAINER, latestMachineStudioVersion.InstallerBlobName); OnCompleted("Latest Machine Studio storage blobs removed."); } public async Task DowngradeTangoStorage(IWebAppBase app, MachineTypes machineType, String versionTag) { OnProgress(AzureUtilsStage.Storage, $"Retrieving settings..."); var latestTangoVersion = await _databaseManager.GetLatestTangoVersion(app, machineType, versionTag); var settings = await app.GetMachineServiceSettingsAsync(); await RemoveBlob(settings.TANGO_VERSIONS_CONTAINER, latestTangoVersion.BlobName); await RemoveBlob(settings.TANGO_VERSIONS_CONTAINER, latestTangoVersion.InstallerBlobName); OnCompleted($"Latest {machineType.ToDescription()} storage blobs removed."); } public async Task ValidateTangoStorageUpgrade(IWebAppBase sourceApp, IWebAppBase targetApp, MachineTypes machineType, String versionTag) { OnProgress(AzureUtilsStage.Validating, $"Validating {machineType.ToDescription()} storage upgrade..."); var sourceSettings = await sourceApp.GetMachineServiceSettingsAsync(); var targetSettings = await targetApp.GetMachineServiceSettingsAsync(); var latestSourceTangoVersion = await _databaseManager.GetLatestTangoVersion(sourceApp, machineType, versionTag); var latestTargetTangoVersion = await _databaseManager.GetLatestTangoVersion(targetApp, machineType, versionTag); var targetAccount = CloudStorageAccount.Parse(targetSettings.STORAGE_ACCOUNT); var targetClient = targetAccount.CreateCloudBlobClient(); var targetTangoContainer = targetClient.GetContainerReference(targetSettings.TANGO_VERSIONS_CONTAINER); //Check version binaries blob not exists on the target. var targetTangoBlob = targetTangoContainer.GetBlockBlobReference(latestSourceTangoVersion.BlobName); if (await targetTangoBlob.ExistsAsync()) { throw new ValidationException($"{machineType.ToDescription()} Block blob '{latestSourceTangoVersion.BlobName}' already exists on the target storage."); } //Check PPC installer blob not exists on the target. var targetTangoInstallerBlob = targetTangoContainer.GetBlockBlobReference(latestSourceTangoVersion.InstallerBlobName); if (await targetTangoInstallerBlob.ExistsAsync()) { throw new ValidationException($"{machineType.ToDescription()} Block blob '{latestSourceTangoVersion.InstallerBlobName}' already exists on the target storage."); } } public async Task ValidateFSEStorageUpgrade(IWebAppBase sourceApp, IWebAppBase targetApp, FSEBuildVariants build) { OnProgress(AzureUtilsStage.Validating, $"Validating {build} storage upgrade..."); var sourceSettings = await sourceApp.GetMachineServiceSettingsAsync(); var targetSettings = await targetApp.GetMachineServiceSettingsAsync(); var latestSourceFSEVersion = await _databaseManager.GetLatestFSEVersion(sourceApp, build); var latestTargetFSEVersion = await _databaseManager.GetLatestFSEVersion(targetApp, build); var targetAccount = CloudStorageAccount.Parse(targetSettings.STORAGE_ACCOUNT); var targetClient = targetAccount.CreateCloudBlobClient(); var targetFSEContainer = targetClient.GetContainerReference(targetSettings.FSE_VERSIONS_CONTAINER); //Check FSE binaries blob not exists on the target. var targetFSEBlob = targetFSEContainer.GetBlockBlobReference(latestSourceFSEVersion.BlobName); if (await targetFSEBlob.ExistsAsync()) { throw new ValidationException($"{build} Block blob '{latestSourceFSEVersion.BlobName}' already exists on the target storage."); } //Check FSE installer blob not exists on the target. var targetFSEInstallerBlob = targetFSEContainer.GetBlockBlobReference(latestSourceFSEVersion.InstallerBlobName); if (await targetFSEInstallerBlob.ExistsAsync()) { throw new ValidationException($"{build} Block blob '{latestSourceFSEVersion.InstallerBlobName}' already exists on the target storage."); } } public async Task ValidateMachineStudioStorageUpgrade(IWebAppBase sourceApp, IWebAppBase targetApp) { OnProgress(AzureUtilsStage.Validating, "Validating machine studio storage upgrade..."); var sourceSettings = await sourceApp.GetMachineServiceSettingsAsync(); var targetSettings = await targetApp.GetMachineServiceSettingsAsync(); var latestSourceMachineStudioVersion = await _databaseManager.GetLatestMachineStudioVersion(sourceApp); var latestTargetMachineStudioVersion = await _databaseManager.GetLatestMachineStudioVersion(targetApp); var targetAccount = CloudStorageAccount.Parse(targetSettings.STORAGE_ACCOUNT); var targetClient = targetAccount.CreateCloudBlobClient(); var targetMachineStudioContainer = targetClient.GetContainerReference(targetSettings.MACHINE_STUDIO_VERSIONS_CONTAINER); //Check machine studio binaries blob not exists on the target. var targetMachineStudioBlob = targetMachineStudioContainer.GetBlockBlobReference(latestSourceMachineStudioVersion.BlobName); if (await targetMachineStudioBlob.ExistsAsync()) { throw new ValidationException($"Machine Studio Block blob '{latestSourceMachineStudioVersion.BlobName}' already exists on the target storage."); } //Check machine studio installer blob not exists on the target. var targetMachineStudioInstallerBlob = targetMachineStudioContainer.GetBlockBlobReference(latestSourceMachineStudioVersion.InstallerBlobName); if (await targetMachineStudioInstallerBlob.ExistsAsync()) { throw new ValidationException($"Machine Studio Block blob '{latestSourceMachineStudioVersion.InstallerBlobName}' already exists on the target storage."); } } public async Task DownloadLatestTangoVersion(IWebAppBase app, MachineTypes machineType, String filePath, String versionTag) { OnProgress(AzureUtilsStage.Storage, $"Retrieving source and target settings..."); var settings = await app.GetMachineServiceSettingsAsync(); TangoVersion version = await _databaseManager.GetLatestTangoVersion(app, machineType, versionTag); OnProgress(AzureUtilsStage.Storage, $"Downloading {machineType.ToDescription()} version '{version.Version}'..."); var account = CloudStorageAccount.Parse(settings.STORAGE_ACCOUNT); var client = account.CreateCloudBlobClient(); var container = client.GetContainerReference(settings.TANGO_VERSIONS_CONTAINER); var blob = container.GetBlockBlobReference(version.BlobName); await blob.FetchAttributesAsync(); var length = blob.Properties.Length; using (FileStreamWrapper st = new FileStreamWrapper(filePath, FileMode.Create, (progress) => { OnProgress(AzureUtilsStage.Storage, $"Downloading {machineType.ToDescription()} version '{version.VersionAndTag}'...", progress, length, false); })) { await blob.DownloadToStreamAsync(st); } } public async Task ReplaceLatestTangoVersion(IWebAppBase app, MachineTypes machineType, String filePath, String versionTag) { OnProgress(AzureUtilsStage.Storage, $"Retrieving source and target settings..."); var settings = await app.GetMachineServiceSettingsAsync(); TangoVersion version = await _databaseManager.GetLatestTangoVersion(app, machineType, versionTag); OnProgress(AzureUtilsStage.Storage, $"Uploading {machineType.ToDescription()} version '{version.VersionAndTag}'..."); var account = CloudStorageAccount.Parse(settings.STORAGE_ACCOUNT); var client = account.CreateCloudBlobClient(); var container = client.GetContainerReference(settings.TANGO_VERSIONS_CONTAINER); var blob = container.GetBlockBlobReference(version.BlobName); FileStreamWrapper st = null; using (st = new FileStreamWrapper(filePath, FileMode.Open, (progress) => { OnProgress(AzureUtilsStage.Storage, $"Uploading {machineType.ToDescription()} version '{version.Version}'...", progress, st.Length, false); })) { await blob.UploadFromStreamAsync(st); } } } }