Merge pull request #47 from wisedev-code/feat/perf_improvements_and_beyond

Feat/perf improvements and beyond

Author: wisedev-pstach

Examples/Examples/Agents/AgentConversationExample.cs

@@ -0,0 +1,72 @@
+using MaIN.Core.Hub;
+
+namespace Examples.Agents;
+
+public class AgentConversationExample : IExample
+{
+    private static readonly ConsoleColor UserColor = ConsoleColor.Magenta;
+    private static readonly ConsoleColor AgentColor = ConsoleColor.Green;
+    private static readonly ConsoleColor SystemColor = ConsoleColor.Yellow;
+
+    public async Task Start()
+    {
+        PrintColored("Agent conversation example is running!", SystemColor);
+        
+        PrintColored("Enter agent name: ", SystemColor, false);
+        var agentName = Console.ReadLine();
+        
+        PrintColored("Enter agent profile (example: 'Gentle and helpful assistant'): ", SystemColor, false);
+        var agentProfile = Console.ReadLine();
+        
+        PrintColored("Enter LLM model (ex: gemma3:4b, llama3.2:3b, yi:6b): ", SystemColor, false);
+        var model = Console.ReadLine()!;
+        var systemPrompt =
+            $"""
+             Your name is: {agentName}
+             You are: {agentProfile}
+             Always stay in your role.
+             """;
+
+        PrintColored($"Creating agent '{agentName}' with profile: '{agentProfile}' using model: '{model}'", SystemColor);
+        AIHub.Extensions.DisableLLamaLogs();
+        AIHub.Extensions.DisableNotificationsLogs();
+        var context = await AIHub.Agent()
+            .WithModel(model)
+            .WithInitialPrompt(systemPrompt)
+            .CreateAsync(interactiveResponse: true);
+        
+        bool conversationActive = true;
+        while (conversationActive)
+        {
+            PrintColored("You > ", UserColor, false);
+            string userMessage = Console.ReadLine()!;
+            
+            if (userMessage.ToLower() == "exit" || userMessage.ToLower() == "quit")
+            {
+                conversationActive = false;
+                continue;
+            }
+            
+            PrintColored($"{agentName} > ", AgentColor, false);
+            await context.ProcessAsync(userMessage);
+            
+            Console.WriteLine(); 
+        }
+        
+        PrintColored("Conversation ended. Goodbye!", SystemColor);
+    }
+    
+    private static void PrintColored(string message, ConsoleColor color, bool newLine = true)
+    {
+        Console.ForegroundColor = color;
+        if (newLine)
+        {
+            Console.WriteLine(message);
+        }
+        else
+        {
+            Console.Write(message);
+        }
+        Console.ResetColor();
+    }
+}

Examples/Examples/Agents/AgentWithApiDataSourceExample.cs

@@ -29,6 +29,5 @@ public async Task Start()
         var result = await context
             .ProcessAsync("I am looking for work as javascript developer");
         Console.WriteLine(result.Message.Content);
-
     }
 }

Examples/Examples/Program.cs

@@ -48,6 +48,7 @@ static void RegisterExamples(IServiceCollection services)
     services.AddTransient<ChatFromExistingExample>();
     services.AddTransient<ChatWithReasoningExample>();
     services.AddTransient<AgentExample>();
+    services.AddTransient<AgentConversationExample>();
     services.AddTransient<AgentWithRedirectExample>();
     services.AddTransient<MultiBackendAgentWithRedirectExample>();
     services.AddTransient<AgentWithRedirectImageExample>();
@@ -130,6 +131,7 @@ public class ExampleRegistry(IServiceProvider serviceProvider)
             ("\u25a0 Chat from Existing", _serviceProvider.GetRequiredService<ChatFromExistingExample>()),
             ("\u25a0 Chat with reasoning", _serviceProvider.GetRequiredService<ChatWithReasoningExample>()),
             ("\u25a0 Basic Agent", _serviceProvider.GetRequiredService<AgentExample>()),
+            ("\u25a0 Conversation Agent", _serviceProvider.GetRequiredService<AgentConversationExample>()),
             ("\u25a0 Agent with Redirect", _serviceProvider.GetRequiredService<AgentWithRedirectExample>()),
             ("\u25a0 Agent with Redirect (Multi backends)", _serviceProvider.GetRequiredService<MultiBackendAgentWithRedirectExample>()),
             ("\u25a0 Agent with Redirect Image", _serviceProvider.GetRequiredService<AgentWithRedirectImageExample>()),
@@ -141,7 +143,6 @@ public class ExampleRegistry(IServiceProvider serviceProvider)
             ("\u25a0 OpenAi Chat", _serviceProvider.GetRequiredService<ChatExampleOpenAi>()),
             ("\u25a0 OpenAi Chat with image", _serviceProvider.GetRequiredService<ChatWithImageGenOpenAiExample>()),
             ("\u25a0 OpenAi Agent with Web Data Source", _serviceProvider.GetRequiredService<AgentWithWebDataSourceOpenAiExample>())
-
         };
     }
 }

Releases/0.1.9.md

@@ -0,0 +1,14 @@
+# 0.1.9 release
+
+**Perf improvements**
+- (LLamaSharp) migrate from ChatSession to Conversation and BatchExecutor
+- Fixes for KernelMemory, now memory is properly disposed
+- New example (conversation agent)
+- CLI release (big perf improvement for infer)
+- Allow to enable or disable cache for model weights
+- Allow to disable buildin llama.cpp logs and MaIN notification
+
+New version of CLI can be downloaded from this page:
+https://maindoc.link/#/doc/cli
+
+*described changes are documented*

src/MaIN.Core.UnitTests/AgentContextTests.cs

@@ -133,7 +133,8 @@ public async Task CreateAsync_ShouldCallAgentServiceCreateAgent()
                 It.IsAny<bool>(), 
                 It.IsAny<bool>(), 
                 It.IsAny<InferenceParams>(),
-                It.IsAny<MemoryParams>()))
+                It.IsAny<MemoryParams>(),
+                It.IsAny<bool>()))
             .ReturnsAsync(agent);
 
         // Act
@@ -146,7 +147,8 @@ public async Task CreateAsync_ShouldCallAgentServiceCreateAgent()
                 It.Is<bool>(f => f == true), 
                 It.Is<bool>(r => r == false), 
                 It.IsAny<InferenceParams>(),
-                It.IsAny<MemoryParams>()),
+                It.IsAny<MemoryParams>(),
+                It.IsAny<bool>()),
             Times.Once);
         Assert.Equal(_agentContext, result);
     }

src/MaIN.Core/.nuspec

@@ -2,7 +2,7 @@
 <package>
   <metadata>
     <id>MaIN.NET</id>
-    <version>0.1.8</version>
+    <version>0.1.9</version>
     <authors>Wisedev</authors>
     <owners>Wisedev</owners>
     <icon>favicon.png</icon>

src/MaIN.Core/Hub/AiHub.cs

@@ -1,5 +1,7 @@
+using LLama.Native;
 using MaIN.Core.Hub.Contexts;
 using MaIN.Core.Interfaces;
+using MaIN.Services.Services.Abstract;
 
 namespace MaIN.Core.Hub;
 
@@ -17,8 +19,21 @@ internal static void Initialize(IAIHubServices services)
         throw new InvalidOperationException(
             "AIHub has not been initialized. Make sure to call AddAIHub() in your service configuration.");
 
-    public static ChatContext Chat() => new ChatContext(Services.ChatService);
-    public static AgentContext Agent() => new AgentContext(Services.AgentService);
-    public static FlowContext Flow() => new FlowContext(Services.FlowService, Services.AgentService);
+    public static ChatContext Chat() => new(Services.ChatService);
+    public static AgentContext Agent() => new(Services.AgentService);
+    public static FlowContext Flow() => new(Services.FlowService, Services.AgentService);
+
+    public abstract class Extensions
+    {
+        public static void DisableLLamaLogs()
+        {
+            NativeLogConfig.llama_log_set((_,_) => {});
+        }
+        
+        public static void DisableNotificationsLogs()
+        {
+            INotificationService.Disable = true;
+        }
+    }
 }
 

src/MaIN.Core/Hub/Contexts/AgentContext.cs

@@ -13,6 +13,7 @@ public class AgentContext
     private readonly IAgentService _agentService;
     private InferenceParams? _inferenceParams;
     private MemoryParams? _memoryParams;
+    private bool _disableCache;
     private Agent _agent;
 
     internal AgentContext(IAgentService agentService)
@@ -57,6 +58,12 @@ public AgentContext WithOrder(int order)
         _agent.Order = order;
         return this;
     }
+
+    public AgentContext DisableCache()
+    {
+        _disableCache = true;
+        return this;
+    }
     public AgentContext WithSource(IAgentSource source, AgentSourceType type)
     {
         _agent.Context.Source = new AgentSource()
@@ -127,13 +134,13 @@ public AgentContext WithBehaviour(string name, string instruction)
 
     public async Task<AgentContext> CreateAsync(bool flow = false, bool interactiveResponse = false)
     {
-        await _agentService.CreateAgent(_agent, flow, interactiveResponse, _inferenceParams, _memoryParams);
+        await _agentService.CreateAgent(_agent, flow, interactiveResponse, _inferenceParams, _memoryParams, _disableCache);
         return this;
     }
 
     public AgentContext Create(bool flow = false, bool interactiveResponse = false)
     {
-        _ = _agentService.CreateAgent(_agent, flow, interactiveResponse, _inferenceParams, _memoryParams).Result;
+        _ = _agentService.CreateAgent(_agent, flow, interactiveResponse, _inferenceParams, _memoryParams, _disableCache).Result;
         return this;
     }
 

src/MaIN.Core/Hub/Contexts/ChatContext.cs

@@ -1,6 +1,7 @@
 using MaIN.Domain.Configuration;
 using MaIN.Domain.Entities;
 using MaIN.Domain.Models;
+using MaIN.Services;
 using MaIN.Services.Constants;
 using MaIN.Services.Dtos;
 using MaIN.Services.Services.Abstract;
@@ -134,6 +135,12 @@ public ChatContext EnableVisual()
         _chat.Visual = true;
         return this;
     }
+
+    public ChatContext DisableCache()
+    {
+        _chat.Properties.AddProperty(ServiceConstants.Properties.DisableCacheProperty);
+        return this;
+    }
 
     public string GetChatId() => _chat.Id;
 
@@ -149,7 +156,7 @@ public async Task<ChatResult> CompleteAsync(
         _chat.Messages.Last().Files = _files;
         if(_preProcess)
         {
-            _chat.Messages.Last().Properties.Add(ServiceConstants.Messages.PreProcessProperty, string.Empty);
+            _chat.Messages.Last().Properties.AddProperty(ServiceConstants.Properties.PreProcessProperty);
         }
 
         if (!await ChatExists(_chat.Id))

src/MaIN.Domain/Entities/Chat.cs

@@ -1,3 +1,4 @@
+using LLama.Batched;
 using MaIN.Domain.Configuration;
 
 namespace MaIN.Domain.Entities;
@@ -15,6 +16,7 @@ public class Chat
     public Dictionary<string, string> Properties { get; init; } = [];
     public List<string> Memory { get; } = [];
     public BackendType? Backend { get; set; }
+    public Conversation.State? ConversationState { get; set; }
 
     public bool Interactive = false;
     public bool Translate = false;

src/MaIN.Domain/MaIN.Domain.csproj

@@ -7,4 +7,8 @@
         <IsPackable>false</IsPackable>
     </PropertyGroup>
 
+    <ItemGroup>
+      <PackageReference Include="LLamaSharp" Version="0.23.0" />
+    </ItemGroup>
+
 </Project>

src/MaIN.Infrastructure/Configuration/SqlExtensions.cs

@@ -49,6 +49,7 @@ [Type] NVARCHAR(MAX) NOT NULL,      -- Stored as JSON
                     [Properties] NVARCHAR(MAX) NULL,    -- Stored as JSON
                     [Visual] BIT NOT NULL DEFAULT 0,
                     [BackendType] INT NOT NULL DEFAULT 0,
+                    [ConvState] NVARCHAR(MAX) NULL,    -- Stored as JSON,
                     [InferenceParams] NVARCHAR(MAX) NULL,    -- Stored as JSON,
                     [MemoryParams] NVARCHAR(MAX) NULL,    -- Stored as JSON
                     [Interactive] BIT NOT NULL DEFAULT 0

src/MaIN.Infrastructure/Configuration/SqliteExtensions.cs

@@ -47,6 +47,7 @@ CREATE TABLE IF NOT EXISTS Chats (
             Properties TEXT,         -- Stored as JSON
             Visual INTEGER NOT NULL DEFAULT 0,
             BackendType INTEGER NOT NULL DEFAULT 0,
+            ConvState TEXT,         -- Stored as JSON
             InferenceParams TEXT,         -- Stored as JSON
             MemoryParams TEXT,         -- Stored as JSON
             Interactive INTEGER NOT NULL DEFAULT 0

src/MaIN.Infrastructure/MaIN.Infrastructure.csproj

@@ -13,7 +13,7 @@
       <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.0" />
       <PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.0-preview.5.24306.7" />
       <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
-      <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.1" />
+      <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.3" />
       <PackageReference Include="Microsoft.Extensions.Options" Version="9.0.0-preview.5.24306.7" />
       <PackageReference Include="MongoDB.Driver" Version="3.2.1" />
       <PackageReference Include="MongoDB.Driver.Core" Version="2.30.0" />

src/MaIN.Infrastructure/Models/ChatDocument.cs

@@ -18,4 +18,5 @@ public class ChatDocument
     public bool Translate { get; init; }
     public InferenceParamsDocument? InferenceParams { get; init; }
     public MemoryParamsDocument? MemoryParams { get; init; }
+    public object? ConvState { get; init; }
 }

src/MaIN.Infrastructure/Repositories/Sql/SqlChatRepository.cs

@@ -27,6 +27,9 @@ private ChatDocument MapChatDocument(dynamic row)
             Type = row.Type != null ? 
                 JsonSerializer.Deserialize<ChatTypeDocument>(row.Type.ToString(), _jsonOptions) : 
                 default,
+            ConvState = row.ConvState != null ? 
+                JsonSerializer.Deserialize<dynamic>(row.ConvState.ToString(), _jsonOptions) : 
+                default,
             InferenceParams = row.InferenceParams != null ? 
                 JsonSerializer.Deserialize<InferenceParamsDocument>(row.InferenceParams.ToString(), _jsonOptions) : 
                 default,
@@ -55,6 +58,7 @@ private object MapChatToParameters(ChatDocument chat)
             chat.Model,
             Messages = JsonSerializer.Serialize(chat.Messages, _jsonOptions),
             Type = JsonSerializer.Serialize(chat.Type, _jsonOptions),
+            ConvState = JsonSerializer.Serialize(chat.ConvState, _jsonOptions),
             InferenceParams = JsonSerializer.Serialize(chat.InferenceParams, _jsonOptions),
             MemoryParams = JsonSerializer.Serialize(chat.MemoryParams, _jsonOptions),
             Properties = JsonSerializer.Serialize(chat.Properties, _jsonOptions),
@@ -87,10 +91,10 @@ public async Task AddChat(ChatDocument chat)
         await connection.ExecuteAsync(@"
             INSERT INTO Chats (
                 Id, Name, Model, Messages, Type, Properties, 
-                Stream, Visual, InferenceParams, MemoryParams, Interactive
+                Stream, Visual, ConvState, InferenceParams, MemoryParams, Interactive
             ) VALUES (
                 @Id, @Name, @Model, @Messages, @Type, @Properties, 
-                 @Visual, @InferenceParams, @MemoryParams, @Interactive)", 
+                 @Visual, @ConvState, @InferenceParams, @MemoryParams, @Interactive)", 
             parameters);
     }
 
@@ -106,6 +110,7 @@ UPDATE Chats
                 Messages = @Messages,
                 Type = @Type,
                 Properties = @Properties,
+                ConvState = @ConvState,
                 Visual = @Visual
             WHERE Id = @Id",
             parameters);

src/MaIN.Infrastructure/Repositories/Sqlite/SqliteChatRepository.cs

@@ -25,6 +25,9 @@ private ChatDocument MapChatDocument(dynamic row)
             Type = row.Type != null ? 
                 JsonSerializer.Deserialize<ChatTypeDocument>(row.Type, _jsonOptions) : 
                 default,
+            ConvState = row.Type != null ? 
+                JsonSerializer.Deserialize<dynamic>(row.ConvState, _jsonOptions) : 
+                default,
             InferenceParams = row.Type != null ? 
                 JsonSerializer.Deserialize<InferenceParamsDocument>(row.InferenceParams, _jsonOptions) : 
                 default,
@@ -49,6 +52,7 @@ private object MapChatToParameters(ChatDocument chat)
             chat.Name,
             chat.Model,
             Messages = JsonSerializer.Serialize(chat.Messages, _jsonOptions),
+            ConvState = JsonSerializer.Serialize(chat.ConvState, _jsonOptions),
             InferenceParams = JsonSerializer.Serialize(chat.InferenceParams, _jsonOptions),
             MemoryParams = JsonSerializer.Serialize(chat.MemoryParams, _jsonOptions),
             Type = JsonSerializer.Serialize(chat.Type, _jsonOptions),
@@ -78,8 +82,8 @@ public async Task AddChat(ChatDocument chat)
     {
         var parameters = MapChatToParameters(chat);
         await connection.ExecuteAsync(@"
-            INSERT INTO Chats (Id, Name, Model, Messages, [Type], Properties, Visual, InferenceParams, MemoryParams, Interactive) VALUES (@Id, @Name, @Model, @Messages, @Type, @Properties, 
-                 @Visual, @InferenceParams, @MemoryParams, @Interactive)", parameters);
+            INSERT INTO Chats (Id, Name, Model, Messages, [Type], Properties, Visual, ConvState, InferenceParams, MemoryParams, Interactive) VALUES (@Id, @Name, @Model, @Messages, @Type, @Properties, 
+                 @Visual, @ConvState, @InferenceParams, @MemoryParams, @Interactive)", parameters);
     }
 
     public async Task UpdateChat(string id, ChatDocument chat)
@@ -92,6 +96,7 @@ UPDATE Chats
                 Messages = @Messages,
                 Type = @Type,
                 Properties = @Properties,
+                ConvState = @ConvState,
                 Visual = @Visual
             WHERE Id = @Id", parameters);
     }

src/MaIN.Services/Constants/ServiceConstants.cs

@@ -20,7 +20,12 @@ public static class ApiUrls
     public static class Messages
     {
         public const string GeneratedImageContent = "Generated Image:";
+    }
+
+    public static class Properties
+    {
         public const string PreProcessProperty = "Pre_Process";
+        public const string DisableCacheProperty = "DisableCache";
     }
 
     public static class Defaults