diff --git a/Releases/0.7.10.md b/Releases/0.7.10.md new file mode 100644 index 0000000..0d6c268 --- /dev/null +++ b/Releases/0.7.10.md @@ -0,0 +1,3 @@ +# 0.7.10 release + +- Fix changeOfValue not returning tokens while using .WithFiles() method. \ No newline at end of file diff --git a/src/MaIN.Core/.nuspec b/src/MaIN.Core/.nuspec index 78f647b..d345323 100644 --- a/src/MaIN.Core/.nuspec +++ b/src/MaIN.Core/.nuspec @@ -2,7 +2,7 @@ MaIN.NET - 0.7.9 + 0.7.10 Wisedev Wisedev favicon.png diff --git a/src/MaIN.Services/Services/Abstract/ILLMService.cs b/src/MaIN.Services/Services/Abstract/ILLMService.cs index 1004d68..1547c43 100644 --- a/src/MaIN.Services/Services/Abstract/ILLMService.cs +++ b/src/MaIN.Services/Services/Abstract/ILLMService.cs @@ -32,6 +32,7 @@ public interface ILLMService /// Task AskMemory(Chat chat, ChatMemoryOptions memoryOptions, + ChatRequestOptions requestOptions, CancellationToken cancellationToken = default); /// diff --git a/src/MaIN.Services/Services/LLMService/AnthropicService.cs b/src/MaIN.Services/Services/LLMService/AnthropicService.cs index d8ca83b..e7ec783 100644 --- a/src/MaIN.Services/Services/LLMService/AnthropicService.cs +++ b/src/MaIN.Services/Services/LLMService/AnthropicService.cs @@ -71,7 +71,7 @@ private void ValidateApiKey() if (HasFiles(lastMessage)) { var result = ChatHelper.ExtractMemoryOptions(lastMessage); - var memoryResult = await AskMemory(chat, result, cancellationToken); + var memoryResult = await AskMemory(chat, result, options, cancellationToken); resultBuilder.Append(memoryResult!.Message.Content); lastMessage.MarkProcessed(); UpdateSessionCache(chat.Id, resultBuilder.ToString(), options.CreateSession); @@ -531,7 +531,7 @@ private List BuildAnthropicMessages(List conversation) return messages; } - public async Task AskMemory(Chat chat, ChatMemoryOptions memoryOptions, CancellationToken cancellationToken = default) + public async Task AskMemory(Chat chat, ChatMemoryOptions memoryOptions, ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { throw new NotSupportedException("Embeddings are not supported by the Anthropic. Document reading requires embedding support."); } diff --git a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs index 0c71f22..ec617d5 100644 --- a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs +++ b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs @@ -48,6 +48,7 @@ protected override void ValidateApiKey() public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, + ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { var lastMsg = chat.Messages.Last(); diff --git a/src/MaIN.Services/Services/LLMService/GeminiService.cs b/src/MaIN.Services/Services/LLMService/GeminiService.cs index 786993c..b3698f8 100644 --- a/src/MaIN.Services/Services/LLMService/GeminiService.cs +++ b/src/MaIN.Services/Services/LLMService/GeminiService.cs @@ -1,4 +1,5 @@ -using MaIN.Domain.Configuration; +using System.Text; +using MaIN.Domain.Configuration; using MaIN.Services.Constants; using MaIN.Services.Services.Abstract; using MaIN.Services.Services.LLMService.Memory; @@ -8,6 +9,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using MaIN.Domain.Entities; +using MaIN.Domain.Models; using MaIN.Services.Utils; namespace MaIN.Services.Services.LLMService; @@ -70,6 +72,7 @@ protected override void ValidateApiKey() public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, + ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { if (!chat.Messages.Any()) @@ -88,7 +91,63 @@ protected override void ValidateApiKey() $"{userQuery} | For your next response only, please respond using exactly the following JSON format: \n{jsonGrammar}\n. Do not include any explanations, code blocks, or additional content. After this single JSON response, resume your normal conversational style."; } - var retrievedContext = await kernel.AskAsync(userQuery, cancellationToken: cancellationToken); + MemoryAnswer retrievedContext; + + if (requestOptions.InteractiveUpdates || requestOptions.TokenCallback != null) + { + var responseBuilder = new StringBuilder(); + + var searchOptions = new SearchOptions + { + Stream = true + }; + + await foreach (var chunk in kernel.AskStreamingAsync( + userQuery, + options: searchOptions, + cancellationToken: cancellationToken)) + { + if (!string.IsNullOrEmpty(chunk.Result)) + { + responseBuilder.Append(chunk.Result); + + var tokenValue = new LLMTokenValue + { + Text = chunk.Result, + Type = TokenType.Message + }; + + if (requestOptions.InteractiveUpdates) + { + await notificationService.DispatchNotification( + NotificationMessageBuilder.CreateChatCompletion(chat.Id, tokenValue, false), + ServiceConstants.Notifications.ReceiveMessageUpdate); + } + + requestOptions.TokenCallback?.Invoke(tokenValue); + } + } + + retrievedContext = new MemoryAnswer + { + Question = userQuery, + Result = responseBuilder.ToString(), + NoResult = responseBuilder.Length == 0 + }; + } + else + { + var searchOptions = new SearchOptions + { + Stream = false + }; + + retrievedContext = await kernel.AskAsync( + userQuery, + options: searchOptions, + cancellationToken: cancellationToken); + } + chat.Messages.Last().MarkProcessed(); await kernel.DeleteIndexAsync(cancellationToken: cancellationToken); return CreateChatResult(chat, retrievedContext.Result, []); diff --git a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs index 12ca5d2..79dbf30 100644 --- a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs +++ b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs @@ -41,6 +41,7 @@ protected override void ValidateApiKey() public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, + ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { var lastMsg = chat.Messages.Last(); diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index adfdbe8..f45bcea 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -58,7 +58,7 @@ public LLMService( if (ChatHelper.HasFiles(lastMsg)) { var memoryOptions = ChatHelper.ExtractMemoryOptions(lastMsg); - return await AskMemory(chat, memoryOptions, cancellationToken); + return await AskMemory(chat, memoryOptions, requestOptions, cancellationToken); } var model = KnownModels.GetModel(chat.Model); @@ -90,6 +90,7 @@ public Task CleanSessionCache(string? id) public async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, + ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { var model = KnownModels.GetModel(chat.Model); @@ -112,9 +113,64 @@ public Task CleanSessionCache(string? id) await memoryService.ImportDataToMemory((memory.km, memory.generator), memoryOptions, cancellationToken); var userMessage = chat.Messages.Last(); - var result = await memory.km.AskAsync( - userMessage.Content, - cancellationToken: cancellationToken); + + MemoryAnswer result; + + if (requestOptions.InteractiveUpdates || requestOptions.TokenCallback != null) + { + var responseBuilder = new StringBuilder(); + + var searchOptions = new SearchOptions + { + Stream = true + }; + + await foreach (var chunk in memory.km.AskStreamingAsync( + userMessage.Content, + options: searchOptions, + cancellationToken: cancellationToken)) + { + if (!string.IsNullOrEmpty(chunk.Result)) + { + responseBuilder.Append(chunk.Result); + + var tokenValue = new LLMTokenValue + { + Text = chunk.Result, + Type = TokenType.Message + }; + + if (requestOptions.InteractiveUpdates) + { + await notificationService.DispatchNotification( + NotificationMessageBuilder.CreateChatCompletion(chat.Id, tokenValue, false), + ServiceConstants.Notifications.ReceiveMessageUpdate); + } + + requestOptions.TokenCallback?.Invoke(tokenValue); + } + } + + result = new MemoryAnswer + { + Question = userMessage.Content, + Result = responseBuilder.ToString(), + NoResult = responseBuilder.Length == 0 + }; + } + else + { + var searchOptions = new SearchOptions + { + Stream = false + }; + + result = await memory.km.AskAsync( + userMessage.Content, + options: searchOptions, + cancellationToken: cancellationToken); + } + await memory.km.DeleteIndexAsync(cancellationToken: cancellationToken); if (disableCache) diff --git a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs index 8357a86..e2b527e 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs @@ -68,7 +68,7 @@ public abstract class OpenAiCompatibleService( if (HasFiles(lastMessage)) { var result = ChatHelper.ExtractMemoryOptions(lastMessage); - var memoryResult = await AskMemory(chat, result, cancellationToken); + var memoryResult = await AskMemory(chat, result, options, cancellationToken); resultBuilder.Append(memoryResult!.Message.Content); lastMessage.MarkProcessed(); UpdateSessionCache(chat.Id, resultBuilder.ToString(), options.CreateSession); @@ -438,6 +438,7 @@ await _notificationService.DispatchNotification( public virtual async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, + ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { if (!chat.Messages.Any()) @@ -455,8 +456,63 @@ await _notificationService.DispatchNotification( userQuery = $"{userQuery} | Respond only using the following JSON format: \n{jsonGrammar}\n. Do not add explanations, code tags, or any extra content."; } - var retrievedContext = await kernel.AskAsync(userQuery, cancellationToken: cancellationToken); + MemoryAnswer retrievedContext; + if (requestOptions.InteractiveUpdates || requestOptions.TokenCallback != null) + { + var responseBuilder = new StringBuilder(); + + var searchOptions = new SearchOptions + { + Stream = true + }; + + await foreach (var chunk in kernel.AskStreamingAsync( + userQuery, + options: searchOptions, + cancellationToken: cancellationToken)) + { + if (!string.IsNullOrEmpty(chunk.Result)) + { + responseBuilder.Append(chunk.Result); + + var tokenValue = new LLMTokenValue + { + Text = chunk.Result, + Type = TokenType.Message + }; + + if (requestOptions.InteractiveUpdates) + { + await notificationService.DispatchNotification( + NotificationMessageBuilder.CreateChatCompletion(chat.Id, tokenValue, false), + ServiceConstants.Notifications.ReceiveMessageUpdate); + } + + requestOptions.TokenCallback?.Invoke(tokenValue); + } + } + + retrievedContext = new MemoryAnswer + { + Question = userQuery, + Result = responseBuilder.ToString(), + NoResult = responseBuilder.Length == 0 + }; + } + else + { + var searchOptions = new SearchOptions + { + Stream = false + }; + + retrievedContext = await kernel.AskAsync( + userQuery, + options: searchOptions, + cancellationToken: cancellationToken); + } + await kernel.DeleteIndexAsync(cancellationToken: cancellationToken); return CreateChatResult(chat, retrievedContext.Result, []); } diff --git a/src/MaIN.Services/Services/LLMService/XaiService.cs b/src/MaIN.Services/Services/LLMService/XaiService.cs index 0b1cd5b..9d7095f 100644 --- a/src/MaIN.Services/Services/LLMService/XaiService.cs +++ b/src/MaIN.Services/Services/LLMService/XaiService.cs @@ -41,6 +41,7 @@ protected override void ValidateApiKey() public override async Task AskMemory( Chat chat, ChatMemoryOptions memoryOptions, + ChatRequestOptions requestOptions, CancellationToken cancellationToken = default) { var lastMsg = chat.Messages.Last(); diff --git a/src/MaIN.Services/Services/Steps/Commands/AnswerCommandHandler.cs b/src/MaIN.Services/Services/Steps/Commands/AnswerCommandHandler.cs index 75d097a..9dc6809 100644 --- a/src/MaIN.Services/Services/Steps/Commands/AnswerCommandHandler.cs +++ b/src/MaIN.Services/Services/Steps/Commands/AnswerCommandHandler.cs @@ -36,7 +36,7 @@ public class AnswerCommandHandler( { case KnowledgeUsage.UseMemory: result = await llmService.AskMemory(command.Chat, - new ChatMemoryOptions { Memory = command.Chat.Memory }); + new ChatMemoryOptions { Memory = command.Chat.Memory }, new ChatRequestOptions()); return result!.Message; case KnowledgeUsage.UseKnowledge: var isKnowledgeNeeded = await ShouldUseKnowledge(command.Knowledge, command.Chat); @@ -138,7 +138,7 @@ await notificationService.DispatchNotification(NotificationMessageBuilder.Create return result.Message; } - var knowledgeResult = await llmService.AskMemory(chat, memoryOptions); + var knowledgeResult = await llmService.AskMemory(chat, memoryOptions, new ChatRequestOptions()); chat.Messages.Last().Content = originalContent; return knowledgeResult?.Message; } diff --git a/src/MaIN.Services/Services/Steps/Commands/FetchCommandHandler.cs b/src/MaIN.Services/Services/Steps/Commands/FetchCommandHandler.cs index 9e49414..5818614 100644 --- a/src/MaIN.Services/Services/Steps/Commands/FetchCommandHandler.cs +++ b/src/MaIN.Services/Services/Steps/Commands/FetchCommandHandler.cs @@ -94,7 +94,7 @@ private async Task HandleFileSource(FetchCommand command, Dictionary HandleWebSource(FetchCommand command, Dictionary ProcessJsonResponse(Message response, FetchCommand c var result = await llmServiceFactory.CreateService(command.Chat.Backend ?? settings.BackendType).AskMemory(command.MemoryChat!, new ChatMemoryOptions { TextData = chunks - }); + }, new ChatRequestOptions()); result!.Message.Role = command.ResponseType == FetchResponseType.AS_System ? "System" : "Assistant"; var newMessage = result!.Message;