Multiple Improvements.

1 files changed, 135 insertions, 73 deletions

diff --git a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Controllers/ChatController.cs b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Controllers/ChatController.cs
index c204df7c7..da91d31e0 100644
--- a/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Controllers/ChatController.cs
+++ b/Software/Visual_Studio_22/Tango.Portal.Chat.Web/Controllers/ChatController.cs
@@ -4,6 +4,10 @@ using Tango.Portal.Chat.Web.Models;
 using Tango.Portal.Chat.Web.Services;
 using Kusto.Data.Data;
 using Microsoft.AspNetCore.Mvc;
+using Newtonsoft.Json.Linq;
+using System.Text.Json.Nodes;
+using System.Collections;
+using Tango.Portal.Chat.Web.Utils;
 
 namespace Tango.Portal.Chat.Web.Controllers
 {
@@ -32,80 +36,26 @@ namespace Tango.Portal.Chat.Web.Controllers
             {
                 var schemaJson = _schema.GetSchemaJson();
                 var plannerPrompt = _schema.GetPlannerPrompt();
+                var plotySample = _schema.GetPlotySample();
 
                 // 1) Ask the model for KQL
-                var plan = await _llm.ProposeKqlAsync(plannerPrompt, req.Question, schemaJson, req.History, ct);
+                var plan = await _llm.ProposeKqlAsync(plannerPrompt, plotySample, req.Question, schemaJson, req.History, ct);
 
                 if (plan.Assistant == "data")
                 {
-                    // 2) Guardrail validation
-                    var val = _guard.Validate(plan.Kql);
-                    if (!val.IsOk) return BadRequest(new { error = "Invalid KQL", details = val.Error, plan });
-
-                    // 4) Execute in ADX
-                    DataTable table;
-                    try
-                    {
-                        table = await _adx.QueryAsync(plan.Kql, plan.Parameters, ct);
-                    }
-                    catch (Exception ex)
-                    {
-                        // Return error to the client so they can iterate
-                        return new ChatResponse
-                        {
-                            Answer = $"Seems like my kusto query ran into some issue..\n{ex.Message}",
-                            ThreadId = req.ThreadId,
-                            UsedKql = plan.Kql
-                        };
-                    }
-
-                    // 5) Build compact facts (limit rows/cols)
-                    var preview = ToPreview(table, 200);
-                    var facts = JsonSerializer.Serialize(preview);
-
-                    // 6) Ask model for final answer
-                    //var answer = await _llm.AnswerFromFactsAsync(req.Question, facts, plan.Kql, ct);
-
-                    var run = await _llm.AnswerWithAssistantAsync(
-                        LlmClient.AssistantType.Data,
-                        req.Question,
-                        facts,
-                        plan.Kql,
-                        req.ThreadId,   // <-- reuse if provided
-                        ct);
-
-                    return new ChatResponse
-                    {
-                        Answer = run.Answer,
-                        UsedKql = plan.Kql,
-                        Preview = preview,
-                        ThreadId = run.ThreadId   // <-- echo back the thread id used/created
-                    };
+                    return await AnswerWithDataAssistant(req, plan, ct);
+                }
+                else if (plan.Assistant == "ploty")
+                {
+                    return await AnswerWithDataAssistant(req, plan, ct);
                 }
                 else if (plan.Assistant == "docs")
                 {
-                    // AFTER
-                    var run = await _llm.AnswerWithAssistantAsync(
-                        LlmClient.AssistantType.Docs,
-                        req.Question,
-                        string.Empty,
-                        plan.Kql,
-                        req.ThreadId,   // <-- reuse if provided
-                        ct);
-
-                    return new ChatResponse
-                    {
-                        Answer = run.Answer,
-                        ThreadId = run.ThreadId
-                    };
+                    return await AnswerWithDocsAssistant(req, plan, ct);
                 }
                 else
                 {
-                    return new ChatResponse
-                    {
-                        Answer = plan.ConversationAnswer,
-                        ThreadId = req.ThreadId
-                    };
+                    return AnswerWithPlannerConversation(req, plan);
                 }
             }
             catch (Exception ex)
@@ -118,19 +68,131 @@ namespace Tango.Portal.Chat.Web.Controllers
             }
         }
 
-        private static object ToPreview(DataTable dt, int maxRows)
+        private static ActionResult<ChatResponse> AnswerWithPlannerConversation(ChatRequest req, ProposeKqlResult plan)
+        {
+            return new ChatResponse
+            {
+                Answer = plan.ConversationAnswer,
+                ThreadId = req.ThreadId
+            };
+        }
+
+        private async Task<ActionResult<ChatResponse>> AnswerWithDocsAssistant(ChatRequest req, ProposeKqlResult plan, CancellationToken ct)
+        {
+            // AFTER
+            var run = await _llm.AnswerWithAssistantAsync(
+                LlmClient.AssistantType.Docs,
+                req.Question,
+                string.Empty,
+                plan.Kql,
+                req.ThreadId,   // <-- reuse if provided
+                ct);
+
+            return new ChatResponse
+            {
+                Answer = run.Answer,
+                ThreadId = run.ThreadId
+            };
+        }
+
+        private async Task<ActionResult<ChatResponse>> AnswerWithDataAssistant(ChatRequest req, ProposeKqlResult plan, CancellationToken ct)
         {
-            var cols = dt.Columns.Cast<DataColumn>().Select(c => c.ColumnName).ToArray();
-            var rows = new List<Dictionary<string, object?>>();
-            int count = 0;
-            foreach (DataRow r in dt.Rows)
+            // 2) Guardrail validation
+            var val = _guard.Validate(plan.Kql);
+            if (!val.IsOk)
+            {
+                // Return error to the client so they can iterate
+                return new ChatResponse
+                {
+                    Answer = $"The generated kusto query contains invalid tokens..\n{val.Error}",
+                    ThreadId = req.ThreadId,
+                    UsedKql = plan.Kql
+                };
+            }
+
+            // 4) Execute in ADX
+            DataTable table;
+            try
+            {
+                table = await _adx.QueryAsync(plan.Kql, plan.Parameters, ct);
+            }
+            catch (Exception ex)
+            {
+                // Return error to the client so they can iterate
+                return new ChatResponse
+                {
+                    Answer = $"Seems like my kusto query ran into some issue..\n{ex.Message}",
+                    ThreadId = req.ThreadId,
+                    UsedKql = plan.Kql
+                };
+            }
+
+            if (table.Columns.Contains("ploty") && table.Rows.Count > 0)
+            {
+                return AnswerWithPloty(req, plan, table);
+            }
+
+            // 5) Build compact facts (limit rows/cols)
+
+            if (table.Rows.Count <= 200)
+            {
+                return await AnswerWithDataAssistantInternal(req, plan, table, ct);
+            }
+            else
             {
-                if (count++ >= maxRows) break;
-                var d = new Dictionary<string, object?>();
-                foreach (var c in cols) d[c] = r[c];
-                rows.Add(d);
+                return AnswerWithMarkdownTable(req, plan, table);
             }
-            return new { columns = cols, rows };
+        }
+
+        private async Task<ActionResult<ChatResponse>> AnswerWithDataAssistantInternal(ChatRequest req, ProposeKqlResult plan, DataTable table, CancellationToken ct)
+        {
+            var preview = DataHelper.ToPreview(table, 200);
+            var facts = JsonSerializer.Serialize(preview);
+
+            // 6) Ask model for final answer
+            //var answer = await _llm.AnswerFromFactsAsync(req.Question, facts, plan.Kql, ct);
+
+            var run = await _llm.AnswerWithAssistantAsync(
+                LlmClient.AssistantType.Data,
+                req.Question,
+                facts,
+                plan.Kql,
+                req.ThreadId,   // <-- reuse if provided
+                ct);
+
+            return new ChatResponse
+            {
+                Answer = run.Answer,
+                UsedKql = plan.Kql,
+                Preview = preview,
+                ThreadId = run.ThreadId   // <-- echo back the thread id used/created
+            };
+        }
+
+        private static ActionResult<ChatResponse> AnswerWithMarkdownTable(ChatRequest req, ProposeKqlResult plan, DataTable table)
+        {
+            var markdown = DataHelper.ToMarkdownTable(table);
+
+            return new ChatResponse
+            {
+                Answer = $"The result set was too big for me to analyze so I'm just going to drop the entire result set.\n{markdown}",
+                UsedKql = plan.Kql,
+                Preview = markdown,
+                ThreadId = req.ThreadId   // <-- echo back the thread id used/created
+            };
+        }
+
+        private static ActionResult<ChatResponse> AnswerWithPloty(ChatRequest req, ProposeKqlResult plan, DataTable table)
+        {
+            String? ploty = table.Rows[0]["ploty"]?.ToString();
+
+            return new ChatResponse
+            {
+                Answer = plan.ConversationAnswer,
+                ThreadId = req.ThreadId,
+                UsedKql = plan.Kql,
+                Ploty = ploty ?? String.Empty
+            };
         }
     }
 }