Multiple Improvements.

3 files changed, 29 insertions, 101 deletions

diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/KqlGuard.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/KqlGuard.cs
index b248fb502..729aaa435 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/KqlGuard.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/KqlGuard.cs
@@ -5,7 +5,7 @@ namespace Tango.Portal.Chat.Web.Services
     public sealed class KqlGuard
     {
         private static readonly string[] Banned = new[] {
-            "externaldata", "evaluate", "cluster(", "database(", "ingest", "print", "datatable", "delete", "drop", "truncate", "update", "set", "declare", "let", "materializedview", "mv-merge", "alter", "create", "append", "ingestiontime()", ".show", ".set", ".clear", ".drop", ".alter"
+            "externaldata", "evaluate", "cluster(", "database(", "ingest", "datatable", "delete", "drop", "truncate", "update", "set", "materializedview", "mv-merge", "alter", "create", "append", "ingestiontime()", ".show", ".set", ".clear", ".drop", ".alter"
         };
 
         public KqlValidationResult Validate(string kql)
@@ -13,8 +13,11 @@ namespace Tango.Portal.Chat.Web.Services
             var text = kql.ToLowerInvariant();
 
             foreach (var token in Banned)
-                if (text.Contains(token))
+            {
+                var pattern = $@"\b{Regex.Escape(token)}\b";
+                if (Regex.IsMatch(text, pattern, RegexOptions.IgnoreCase))
                     return KqlValidationResult.Fail($"Query uses banned token: {token}");
+            }
 
             // Ensure only allowed tables are referenced (quick heuristic)
             //var tableNames = new HashSet<string>(allowTables.Select(t => t.ToLowerInvariant()));
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/LlmClient.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/LlmClient.cs
index 1577b7e08..fc970d52e 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/LlmClient.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/LlmClient.cs
@@ -31,102 +31,7 @@ namespace Tango.Portal.Chat.Web.Services
             _opt = opt.Value;
         }
 
-        public async Task<ProposeKqlResult> ProposeKqlAsync(string question, string schemaJson, CancellationToken ct = default)
-        {
-            var system = string.Join("\n", new[] {
-                "You are a Kusto (KQL) assistant for Azure Data Explorer.",
-                "Use ONLY the tables/columns provided in the SCHEMA JSON that follows.",
-                "ALWAYS try to query for the least amount of data neccessary to answer the question.",
-                "Return a JSON object with fields: assistant, kql, parameters, parameterTypes (optional), assumptions, why.",
-                "When asked to query by months ago, convert number of months to days (e.g last to months = StartTime >= ago(60d))",
-                "When joining tables, this example for correct syntax: EventsTable | join kind=inner (EventTypesTable) on $left.EventTypeGuid == $right.GUID.",
-                "When querying MachinesTable you can fetch the machine's Organization and Site by joining the latest record from JobRunsTable by SerialNumber.",
-                "Output raw JSON ONLY (no code fences).",
-                @"Classify the user's question into exactly one of:
-                  - ""data"": requires querying telemetry via KQL (numbers, trends, counts, rates, top-N, timelines).
-                  - ""docs"": architectural/how-to/design/definitions/“what is/how do we” that do not require live data. place data or docs in the assistant field you return. 
-                  - ""none"": conversational content like Thank you/Hi/How are you. Or questions about why did you chose to execute the previouse KQL query the way you did and maybe ask for correction. In this case answer nicely and try to have a good conversation without going out of context.",
-                "If the question is classified as 'docs', set kql to an empty string and parameters to an empty object.",
-
-
-            });
-
-            var schemaBlock = $"SCHEMA:\n{schemaJson}";
-            var user = $"Question: {question}\n\n{schemaBlock}";
-
-            var payload = new
-            {
-                model = _opt.Model,
-                temperature = _opt.Temperature,
-                response_format = new { type = "json_object" },
-                messages = new object[] {
-                    new { role = "system", content = system },
-                    new { role = "user", content = user }
-                }
-            };
-
-            using var req = new HttpRequestMessage(HttpMethod.Post, _opt.Endpoint);
-            var json = JsonSerializer.Serialize(payload);
-            req.Content = new StringContent(json, Encoding.UTF8, "application/json");
-
-            if (_opt.IsAzure) req.Headers.Add("api-key", _opt.ApiKey);
-            else req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _opt.ApiKey);
-
-            using var resp = await _http.SendAsync(req, ct);
-            resp.EnsureSuccessStatusCode();
-            var body = await resp.Content.ReadAsStringAsync(ct);
-
-            var root = JsonNode.Parse(body)!.AsObject();
-            var content = root["choices"]![0]!["message"]!["content"]?.ToString() ?? "{}";
-            content = StripCodeFences(content);
-
-            var opts = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
-            opts.Converters.Add(new FlexibleStringListConverter());
-            try
-            {
-                var result = JsonSerializer.Deserialize<ProposeKqlResult>(content, opts);
-                if (result != null) return result;
-            }
-            catch (JsonException)
-            {
-                // fall back
-            }
-
-            // Lenient mapping
-            var node = JsonNode.Parse(content) as JsonObject ?? new JsonObject();
-            var kql = node["kql"]?.ToString() ?? string.Empty;
-
-            var parameters = new Dictionary<string, string>();
-            if (node["parameters"] is JsonObject pObj)
-            {
-                foreach (var kv in pObj)
-                    parameters[kv.Key] = kv.Value?.ToString() ?? string.Empty;
-            }
-
-            List<string>? assumptions = null;
-            if (node.TryGetPropertyValue("assumptions", out var aNode) && aNode is not null)
-            {
-                assumptions = new List<string>();
-                if (aNode is JsonArray arr)
-                {
-                    foreach (var el in arr) assumptions.Add(el?.ToString() ?? string.Empty);
-                }
-                else
-                {
-                    assumptions.Add(aNode.ToString());
-                }
-            }
-
-            return new ProposeKqlResult
-            {
-                Kql = kql,
-                Parameters = parameters,
-                Assumptions = assumptions,
-                Why = node["why"]?.ToString()
-            };
-        }
-
-        public async Task<ProposeKqlResult> ProposeKqlAsync(String plannerPrompt,
+        public async Task<ProposeKqlResult> ProposeKqlAsync(String plannerPrompt, String plotySample,
     string question, string schemaJson, IEnumerable<ChatMessage>? history, CancellationToken ct = default)
         {
             var messages = new List<object> { new { role = "system", content = plannerPrompt } };
@@ -134,11 +39,11 @@ namespace Tango.Portal.Chat.Web.Services
             if (history != null)
             {
                 foreach (var m in history.TakeLast(6))
-                    messages.Add(new { role = m.Role, content = m.Content, usedKql = m.UsedKql });
+                    messages.Add(new { role = m.Role, content = CapString(m.Content, 1000), usedKql = m.UsedKql });
             }
 
             var schemaBlock = $"SCHEMA:\n{schemaJson}";
-            messages.Add(new { role = "user", content = $"Question: {question}\n\n{schemaBlock}" });
+            messages.Add(new { role = "user", content = $"Question: {question}\n\n{schemaBlock}\n\n{plotySample}" });
 
             var payload = new
             {
@@ -390,7 +295,7 @@ namespace Tango.Portal.Chat.Web.Services
                 // Poll until completed (same as your existing loop)
                 while (true)
                 {
-                    await Task.Delay(600, ct);
+                    await Task.Delay(2000, ct);
                     using var gReq = new HttpRequestMessage(HttpMethod.Get, $"https://api.openai.com/v1/threads/{threadId}/runs/{runId}");
                     AddOpenAIHeaders(gReq);
                     var gBody = await ReadBodyOrThrowAsync(await _http.SendAsync(gReq, ct), ct);
@@ -437,6 +342,15 @@ namespace Tango.Portal.Chat.Web.Services
             return Regex.Replace(s, @"\s*【[^】]*】", string.Empty);
         }
 
+        private static string CapString(string input, int maxLength)
+        {
+            if (string.IsNullOrEmpty(input) || maxLength <= 0)
+                return string.Empty;
+
+            if (input.Length <= maxLength)
+                return input;
 
+            return input.Substring(0, maxLength) + "…";
+        }
     }
 }
diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/SchemaRegistry.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/SchemaRegistry.cs
index 3ba6ad0c5..1c48b93a2 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/SchemaRegistry.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Services/SchemaRegistry.cs
@@ -39,5 +39,16 @@ namespace Tango.Portal.Chat.Web.Services
             }
             return File.ReadAllText(path);
         }
+
+        public string GetPlotySample()
+        {
+            var path = Path.Combine(_env.ContentRootPath, "Data", "ploty_sample.txt");
+            if (!File.Exists(path))
+            {
+                _log.LogWarning("Ploty sample file not found at {Path}. Returning empty prompt.", path);
+                return string.Empty;
+            }
+            return File.ReadAllText(path);
+        }
     }
 }