Dynamic developer instructions using Azure table.

11 files changed, 208 insertions, 7 deletions

diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Controllers/ChatController.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Controllers/ChatController.cs
index 396651e3f..950e70aa5 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Controllers/ChatController.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Controllers/ChatController.cs
@@ -20,14 +20,16 @@ namespace Tango.Portal.Chat.Web.Controllers
         private readonly KustoQueryService _adx;
         private readonly LlmClient _llm;
         private readonly ChatMessageLogger _logger;
+        private readonly AIInstructionService _instructionService;
 
-        public ChatController(SchemaRegistry schema, KqlGuard guard, KustoQueryService adx, LlmClient llm, ChatMessageLogger logger)
+        public ChatController(SchemaRegistry schema, KqlGuard guard, KustoQueryService adx, LlmClient llm, ChatMessageLogger logger, AIInstructionService instructionService)
         {
             _schema = schema;
             _guard = guard;
             _adx = adx;
             _llm = llm;
             _logger = logger;
+            _instructionService = instructionService;
         }
 
         [HttpPost("ask")]
@@ -47,6 +49,13 @@ namespace Tango.Portal.Chat.Web.Controllers
                 var sessionUser = SessionUtils.GetSessionUser(HttpContext);
                 var sessionId = HttpContext.Session.Id;
 
+                // Handle SYSTEM commands for roy@twine-s.com
+                if (req.Question.StartsWith("SYSTEM:", StringComparison.OrdinalIgnoreCase) && 
+                    sessionUser?.Email?.Equals("roy@twine-s.com", StringComparison.OrdinalIgnoreCase) == true)
+                {
+                    return await HandleSystemCommandAsync(req.Question, sessionUser.Email, ct);
+                }
+
                 // Log the question
                 _ = Task.Run(async () =>
                 {
@@ -66,7 +75,7 @@ namespace Tango.Portal.Chat.Web.Controllers
                 }, ct);
 
                 var schemaJson = _schema.GetSchemaJson();
-                var plannerPrompt = _schema.GetPlannerPrompt();
+                var plannerPrompt = await _schema.GetPlannerPromptAsync();
                 var plotySample = _schema.GetPlotySample();
 
                 // 1) Ask the model for KQL
@@ -108,6 +117,7 @@ namespace Tango.Portal.Chat.Web.Controllers
                             response,
                             plan.Assistant,
                             plan.Provider.ToString(),
+                            plan.Assumptions,
                             ct);
                     }
                     catch
@@ -140,6 +150,7 @@ namespace Tango.Portal.Chat.Web.Controllers
                             errorResponse,
                             "error",
                             "Unknown",
+                            null,
                             ct);
                     }
                     catch
@@ -290,5 +301,41 @@ namespace Tango.Portal.Chat.Web.Controllers
                 Ploty = ploty ?? String.Empty
             };
         }
+
+        private async Task<ActionResult<ChatResponse>> HandleSystemCommandAsync(string question, string userEmail, CancellationToken ct)
+        {
+            var instruction = question.Substring(7).Trim(); // Remove "SYSTEM:" prefix
+
+            if (instruction.Equals("delete", StringComparison.OrdinalIgnoreCase))
+            {
+                var success = await _instructionService.DeleteLastInstructionAsync();
+                return new ChatResponse
+                {
+                    Answer = success 
+                        ? "The last instruction has been successfully deleted."
+                        : "No instructions found to delete or deletion failed.",
+                    ThreadId = null
+                };
+            }
+            else if (!string.IsNullOrWhiteSpace(instruction))
+            {
+                var success = await _instructionService.AddInstructionAsync(instruction, userEmail);
+                return new ChatResponse
+                {
+                    Answer = success
+                        ? "The new instruction has been successfully added."
+                        : "Failed to add the instruction.",
+                    ThreadId = null
+                };
+            }
+            else
+            {
+                return new ChatResponse
+                {
+                    Answer = "Invalid SYSTEM command. Use 'SYSTEM: <instruction>' to add or 'SYSTEM: delete' to remove the last instruction.",
+                    ThreadId = null
+                };
+            }
+        }
     }
 }
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Data/planner_prompt.txt b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Data/planner_prompt.txt
index 40a5b2ed3..fa40f30e7 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Data/planner_prompt.txt
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Data/planner_prompt.txt
@@ -20,6 +20,7 @@ SCHEMA USE
 - When asked to query by months ago, convert number of months to days (e.g last to months = StartTime >= ago(60d)).
 - When joining tables, also join by environment if both sides have ENVIRONMENT (e.g. SitesTable | join kind=inner (OrganizationsTable) on $left.ORGANIZATION_GUID == $right.GUID and $left.ENVIRONMENT == $right.ENVIRONMENT).
 - When joining tables, this example for correct syntax: EventsTable | join kind=inner (EventTypesTable) on $left.EventTypeGuid == $right.GUID.
+- When joining tables and asked to filter by environment, apply the filter to both sides of the join (e.g: MachinesTable | where ENVIRONMENT == 'PROD' | summarize MachineCount=count() by ORGANIZATION_GUID | join kind=inner (OrganizationsTable | where ENVIRONMENT == "PROD") on $left.ORGANIZATION_GUID == $right.GUID | top 5 by MachineCount desc | project OrganizationName=NAME, MachineCount).
 - If you are joining and want to project two columns with the same name append '1' to the end of the second table name. (e.g: SitesTable | join kind=inner (OrganizationsTable) on $left.ORGANIZATION_GUID == $right.GUID | project Site = NAME, Organization = NAME1).
 - To get machine's organization name example: MachinesTable | where SERIAL_NUMBER == '30001' | join kind=inner (OrganizationsTable) on $left.ORGANIZATION_GUID == $right.GUID | project OrganizationName = NAME1.
 - To get machine's site name example: MachinesTable | where SERIAL_NUMBER == '30001' | join kind=inner (SitesTable) on $left.SITE_GUID == $right.GUID | project SiteName = NAME1.
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/AIInstruction.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/AIInstruction.cs
new file mode 100644
index 000000000..52f566e00
--- /dev/null
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/AIInstruction.cs
@@ -0,0 +1,16 @@
+using Azure;
+using Azure.Data.Tables;
+
+namespace Tango.Portal.Chat.Web.Models
+{
+    public class AIInstruction : ITableEntity
+    {
+        public string PartitionKey { get; set; } = "Instructions";
+        public string RowKey { get; set; } = string.Empty;
+        public DateTimeOffset? Timestamp { get; set; }
+        public ETag ETag { get; set; }
+        public string Instruction { get; set; } = string.Empty;
+        public DateTime CreatedAt { get; set; }
+        public string CreatedBy { get; set; } = string.Empty;
+    }
+}
\ No newline at end of file
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/ChatConversationMessage.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/ChatConversationMessage.cs
index 3c056a922..865935230 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/ChatConversationMessage.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/ChatConversationMessage.cs
@@ -13,5 +13,6 @@ namespace Tango.Portal.Chat.Web.Models
         public string Classification { get; set; } = string.Empty;
         public string Message { get; set; } = string.Empty;
         public string Provider { get; set; } = string.Empty;
+        public string Assumptions { get; set; } = string.Empty;
     }
 }
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/Options.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/Options.cs
index 6c6a537d6..9ae05da3d 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/Options.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Models/Options.cs
@@ -35,4 +35,9 @@ namespace Tango.Portal.Chat.Web.Services
         public string ClientId { get; set; } = string.Empty;
         public string ClientSecret { get; set; } = string.Empty;
     }
+
+    public sealed class AzureStorageOptions
+    {
+        public string ConnectionString { get; set; } = string.Empty;
+    }
 }
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Program.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Program.cs
index 41402c527..380a9607b 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Program.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Program.cs
@@ -15,6 +15,10 @@ builder.Services.AddSingleton<KustoQueryService>();
 builder.Services.AddSingleton<ChatMessageLogger>();
 builder.Services.AddSingleton<SchemaRegistry>();
 builder.Services.AddSingleton<KqlGuard>();
+
+// Azure Storage config
+builder.Services.Configure<AzureStorageOptions>(builder.Configuration.GetSection("AzureStorage"));
+builder.Services.AddSingleton<AIInstructionService>();
 builder.Services.AddSession();
 
 // Simple HTTP client for LLM
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/AIInstructionService.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/AIInstructionService.cs
new file mode 100644
index 000000000..5fcb4bf66
--- /dev/null
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/AIInstructionService.cs
@@ -0,0 +1,97 @@
+using Azure.Data.Tables;
+using Microsoft.Extensions.Options;
+using Tango.Portal.Chat.Web.Models;
+
+namespace Tango.Portal.Chat.Web.Services
+{
+    public sealed class AIInstructionService
+    {
+        private readonly TableClient _tableClient;
+        private readonly ILogger<AIInstructionService> _logger;
+
+        public AIInstructionService(IOptions<AzureStorageOptions> options, ILogger<AIInstructionService> logger)
+        {
+            _logger = logger;
+            var serviceClient = new TableServiceClient(options.Value.ConnectionString);
+            _tableClient = serviceClient.GetTableClient("AIInstructions");
+            
+            // Create table if it doesn't exist
+            _tableClient.CreateIfNotExists();
+        }
+
+        public async Task<bool> AddInstructionAsync(string instruction, string createdBy)
+        {
+            try
+            {
+                var aiInstruction = new AIInstruction
+                {
+                    RowKey = Guid.NewGuid().ToString(),
+                    Instruction = instruction,
+                    CreatedAt = DateTime.UtcNow,
+                    CreatedBy = createdBy
+                };
+
+                await _tableClient.AddEntityAsync(aiInstruction);
+                _logger.LogInformation("Added AI instruction by {CreatedBy}: {Instruction}", createdBy, instruction);
+                return true;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to add AI instruction: {Instruction}", instruction);
+                return false;
+            }
+        }
+
+        public async Task<bool> DeleteLastInstructionAsync()
+        {
+            try
+            {
+                var instructions = await GetAllInstructionsAsync();
+                var lastInstruction = instructions.OrderByDescending(i => i.CreatedAt).FirstOrDefault();
+                
+                if (lastInstruction == null)
+                {
+                    _logger.LogWarning("No instructions found to delete");
+                    return false;
+                }
+
+                await _tableClient.DeleteEntityAsync(lastInstruction.PartitionKey, lastInstruction.RowKey);
+                _logger.LogInformation("Deleted last AI instruction: {Instruction}", lastInstruction.Instruction);
+                return true;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to delete last AI instruction");
+                return false;
+            }
+        }
+
+        public async Task<List<AIInstruction>> GetAllInstructionsAsync()
+        {
+            try
+            {
+                var instructions = new List<AIInstruction>();
+                await foreach (var instruction in _tableClient.QueryAsync<AIInstruction>())
+                {
+                    instructions.Add(instruction);
+                }
+                return instructions.OrderBy(i => i.CreatedAt).ToList();
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to retrieve AI instructions");
+                return new List<AIInstruction>();
+            }
+        }
+
+        public async Task<string> GetInstructionsTextAsync()
+        {
+            var instructions = await GetAllInstructionsAsync();
+            if (!instructions.Any())
+                return string.Empty;
+
+            var instructionTexts = instructions.Select(i => i.Instruction);
+            return string.Join("\n", instructionTexts);
+        }
+    }
+}
\ No newline at end of file
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/ChatMessageLogger.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/ChatMessageLogger.cs
index 4eed2182d..2ae47467a 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/ChatMessageLogger.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/ChatMessageLogger.cs
@@ -66,13 +66,14 @@ namespace Tango.Portal.Chat.Web.Services
                 Role = "user",
                 Classification = "question",
                 Message = question,
-                Provider = string.Empty
+                Provider = string.Empty,
+                Assumptions = string.Empty
             };
 
             await LogMessageAsync(message, ct);
         }
 
-        public async Task LogAnswerAsync(string sessionId, string userEmail, string userName, ChatResponse response, string assistantType, string provider, CancellationToken ct = default)
+        public async Task LogAnswerAsync(string sessionId, string userEmail, string userName, ChatResponse response, string assistantType, string provider, List<string>? assumptions, CancellationToken ct = default)
         {
             var answerMessage = BuildAnswerMessage(response);
             
@@ -85,7 +86,8 @@ namespace Tango.Portal.Chat.Web.Services
                 Role = "assistant",
                 Classification = assistantType,
                 Message = answerMessage,
-                Provider = provider
+                Provider = provider,
+                Assumptions = assumptions != null && assumptions.Count > 0 ? string.Join("; ", assumptions) : string.Empty
             };
 
             await LogMessageAsync(message, ct);
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/SchemaRegistry.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/SchemaRegistry.cs
index 1c48b93a2..88612f33b 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/SchemaRegistry.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/SchemaRegistry.cs
@@ -6,11 +6,14 @@ namespace Tango.Portal.Chat.Web.Services
     {
         private readonly IWebHostEnvironment _env;
         private readonly ILogger<SchemaRegistry> _log;
+        private readonly AIInstructionService _instructionService;
         private string? _cached;
 
-        public SchemaRegistry(IWebHostEnvironment env, ILogger<SchemaRegistry> log)
+        public SchemaRegistry(IWebHostEnvironment env, ILogger<SchemaRegistry> log, AIInstructionService instructionService)
         {
-            _env = env; _log = log;
+            _env = env; 
+            _log = log;
+            _instructionService = instructionService;
         }
 
         public string GetSchemaJson()
@@ -29,8 +32,29 @@ namespace Tango.Portal.Chat.Web.Services
             return _cached!;
         }
 
+        public async Task<string> GetPlannerPromptAsync()
+        {
+            var path = Path.Combine(_env.ContentRootPath, "Data", "planner_prompt.txt");
+            if (!File.Exists(path))
+            {
+                _log.LogWarning("Planner prompt file not found at {Path}. Returning empty prompt.", path);
+                return string.Empty;
+            }
+            
+            var basePrompt = File.ReadAllText(path);
+            var aiInstructions = await _instructionService.GetInstructionsTextAsync();
+            
+            if (!string.IsNullOrWhiteSpace(aiInstructions))
+            {
+                return $"{basePrompt}\n\nAdditional Instructions:\n{aiInstructions}";
+            }
+            
+            return basePrompt;
+        }
+
         public string GetPlannerPrompt()
         {
+            // Keep synchronous version for backward compatibility
             var path = Path.Combine(_env.ContentRootPath, "Data", "planner_prompt.txt");
             if (!File.Exists(path))
             {
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Tango.Portal.Chat.Web.csproj b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Tango.Portal.Chat.Web.csproj
index 3d74069d8..8480aca38 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Tango.Portal.Chat.Web.csproj
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Tango.Portal.Chat.Web.csproj
@@ -20,6 +20,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Azure.Kusto.Data" Version="12.2.3" />
     <PackageReference Include="Azure.Identity" Version="1.12.0" />
+    <PackageReference Include="Azure.Data.Tables" Version="12.8.3" />
   </ItemGroup>
   <ItemGroup>
     <Content Update="wwwroot\favicon.ico">
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/appsettings.json b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/appsettings.json
index 70f38ae11..260d4b834 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/appsettings.json
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/appsettings.json
@@ -25,5 +25,8 @@
     "ClientId": "ec612854-7abc-457b-808a-5d0c5ba80c57",
     "ClientSecret": "C6n8Q~-NgsAQ6yYJwoNABkcVUNSm2~8-8xNgaa32"
   },
+  "AzureStorage": {
+    "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=tangostorage;AccountKey=S4z/D+Yg6mwMis+bs/VpcDLA9yE1iZaYq23shQlRIi2KmM9E7JY8zdZjeAPOPdG3gONHoNDEpsgH6D4cqQ/bsA==;EndpointSuffix=core.windows.net"
+  },
   "AllowedHosts": "*"
 }
\ No newline at end of file