V18 / 8d896b8a86e5
MemorySearch.cs
Lilith/Core/Memory/MemorySearch.cs
Changed in this versionCS
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using Lilith.Core.Workspace;
using Lilith.Core.Workspace.Extensions;
namespace Lilith.Core.Memory;
// Called only under WorkspaceSession's process gate and exclusive workspace lease.
internal static class MemorySearch
{
private const int ChunkSize = 800, Stride = 680, MaximumChunks = 4000;
private const double MinimumSimilarity = 0.50;
private sealed record Chunk(string Path, int Offset, string Text, string Hash);
private sealed record Index(string Model, string Format, Dictionary<string, float[]> Vectors);
internal static async Task<WorkspaceResult> SearchAsync(WorkspaceOptions options, WorkspaceRequest request, IMemoryEmbeddings? embeddings, CancellationToken ct)
{
if (request.Query.Length is < 1 or > 200 || string.IsNullOrWhiteSpace(request.Query)) throw new ArgumentException("Supply a memory query up to 200 characters.");
if (request.Path is not ("" or "memory")) throw new ArgumentException("Memory retrieval is restricted to memory/.");
var chunks = new List<Chunk>();
string directory = options.ResolvePath("memory");
if (Directory.Exists(directory)) foreach (var file in directory.EnumerateEntries(2000).Order(StringComparer.Ordinal))
{
ct.ThrowIfCancellationRequested();
if (!File.Exists(file) || !file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) continue;
if (new FileInfo(file).Length > 80_000) throw new IOException("A memory fact exceeds the supported size.");
string text = await File.ReadAllTextAsync(file, new UTF8Encoding(false, true), ct);
if (text.Length > 20_000) throw new IOException("A memory fact exceeds 20,000 characters.");
string path = Path.GetRelativePath(options.RootDirectory, file).Replace('\\', '/');
for (int offset = 0; offset < text.Length; offset += Stride)
{
string excerpt = text.Substring(offset, Math.Min(ChunkSize, text.Length - offset));
string hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(path + "\n" + offset + "\n" + excerpt)));
chunks.Add(new(path, offset, excerpt, hash));
if (chunks.Count > MaximumChunks) throw new IOException("Memory index exceeds 4,000 chunks; reduce stored facts.");
if (offset + ChunkSize >= text.Length) break;
}
}
if (chunks.Count == 0) return Result([], request.Offset, "empty", null);
using var owned = embeddings is null ? new OllamaMemoryEmbeddings() : null;
embeddings ??= owned!;
var scores = new Dictionary<string, double>(StringComparer.Ordinal);
string mode = "hybrid", cachePath = Path.Combine(options.GetMetadataDirectory(), "memory-embeddings", "index.json");
string? warning = null;
try
{
cachePath.EnsureNoLinks();
string identity = await embeddings.IdentityAsync(ct);
Index? index = null;
if (File.Exists(cachePath) && new FileInfo(cachePath).Length <= 32_000_000)
try { index = JsonSerializer.Deserialize<Index>(await File.ReadAllTextAsync(cachePath, ct)); }
catch (JsonException) { /* Derived cache: rebuild from authoritative fact files. */ }
var query = await embeddings.EmbedAsync(["task: search result | query: " + request.Query], ct);
if (query.Length != 1 || !Valid(query[0])) throw new InvalidDataException("Embedding query response is invalid.");
var cached = index?.Model == identity && index.Format == "embeddinggemma-search-v1" && index.Vectors is not null ? index.Vectors : new(StringComparer.Ordinal);
var vectors = new Dictionary<string, float[]>(StringComparer.Ordinal);
foreach (var chunk in chunks)
if (cached.TryGetValue(chunk.Hash, out var vector) && Valid(vector) && vector.Length == query[0].Length) vectors[chunk.Hash] = vector;
var missing = chunks.Where(c => !vectors.ContainsKey(c.Hash)).ToArray();
for (int i = 0; i < missing.Length; i += 32)
{
var batch = missing.Skip(i).Take(32).ToArray();
var encoded = await embeddings.EmbedAsync(batch.Select(c => "title: " + Path.GetFileNameWithoutExtension(c.Path) + " | text: " + c.Text).ToArray(), ct);
if (encoded.Length != batch.Length || encoded.Any(v => !Valid(v) || v.Length != query[0].Length)) throw new InvalidDataException("Embedding response has invalid vectors.");
for (int n = 0; n < batch.Length; n++) vectors[batch[n].Hash] = encoded[n];
}
foreach (var chunk in chunks) scores[chunk.Hash] = Cosine(query[0], vectors[chunk.Hash]);
if (identity != await embeddings.IdentityAsync(ct)) throw new IOException("Embedding model changed during retrieval; retry.");
string serialized = JsonSerializer.Serialize(new Index(identity, "embeddinggemma-search-v1", vectors));
if (serialized.Length > 32_000_000) throw new IOException("Memory embedding cache exceeds its size bound.");
await cachePath.WriteAtomicallyAsync(serialized, ct);
}
catch (Exception ex) when (ex is HttpRequestException or IOException or JsonException or InvalidOperationException || ex is TaskCanceledException && !ct.IsCancellationRequested)
{
ct.ThrowIfCancellationRequested();
mode = "lexical_fallback"; warning = "Semantic retrieval unavailable; only text matches were searched. " + ex.Message;
scores.Clear();
}
var terms = Regex.Matches(request.Query.ToLowerInvariant(), @"[\p{L}\p{N}_]+")
.Select(m => m.Value).Where(t => t.Length > 2 && !StopWords.Contains(t)).Distinct().ToArray();
var ranked = chunks.Select(c =>
{
string searchable = Path.GetFileNameWithoutExtension(c.Path).Replace('_', ' ') + " " + c.Text;
double lexical = searchable.Contains(request.Query, StringComparison.OrdinalIgnoreCase) ? 1 :
terms.Length == 0 ? 0 : (double)terms.Count(t => Regex.IsMatch(searchable, @"\b" + Regex.Escape(t) + @"\b", RegexOptions.IgnoreCase)) / terms.Length;
double semantic = scores.GetValueOrDefault(c.Hash, -1);
return new { Chunk = c, Semantic = semantic, Lexical = lexical, Score = Math.Max(0, semantic) * .7 + lexical * .3 };
}).Where(x => x.Lexical > 0 || x.Semantic >= MinimumSimilarity)
.OrderByDescending(x => x.Score).ThenBy(x => x.Chunk.Path, StringComparer.Ordinal).ThenBy(x => x.Chunk.Offset)
.GroupBy(x => x.Chunk.Path, StringComparer.Ordinal).Select(g => g.First()).ToArray();
var matches = ranked.Skip(request.Offset).Take(5).Select(x => new
{
path = x.Chunk.Path, key = x.Chunk.Path[7..^4], excerpt = x.Chunk.Text,
offset = x.Chunk.Offset, content_hash = x.Chunk.Hash,
score = x.Score, similarity = x.Semantic < 0 ? (double?)null : x.Semantic, lexical = x.Lexical
}).ToArray();
return new("completed", matches.Length == 0 ? "No matching memory facts." : "Relevant saved memory candidates. Verify relevance; scores are not truth or confidence probabilities.",
Data: new { matches, next_offset = request.Offset + matches.Length < ranked.Length ? request.Offset + matches.Length : (int?)null, mode, warning });
}
private static WorkspaceResult Result(object[] matches, int offset, string mode, string? warning) => new("completed", "No memory facts are saved.", Data: new { matches, next_offset = (int?)null, mode, warning });
private static readonly HashSet<string> StopWords = ["the", "and", "for", "what", "which", "would", "could", "should", "you", "your", "with", "that", "this", "have", "does", "from", "about", "please", "remember"];
private static bool Valid(float[]? vector) => vector is { Length: >= 1 and <= 4096 } && vector.All(float.IsFinite) && vector.Any(v => v != 0);
private static double Cosine(float[] a, float[] b)
{
double dot = 0, aa = 0, bb = 0;
for (int i = 0; i < a.Length; i++) { dot += (double)a[i] * b[i]; aa += (double)a[i] * a[i]; bb += (double)b[i] * b[i]; }
return Math.Clamp(dot / Math.Sqrt(aa * bb), -1, 1);
}
}