ResearchModels.cs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

namespace Lilith.Core.WebResearch;

public sealed record SearchSource(string Url, string Title, string Engine);
public sealed record EngineOutcome(string Engine, string Status, string Message);
public sealed record SearchBatch(SearchSource[] Results, EngineOutcome[] Engines);
public sealed record WebPage(string Url, string Title, bool Readable, string Text, string Error, DateTimeOffset RetrievedAt);
public sealed record ResearchBrowserFrame(DateTimeOffset AtUtc, string Url, byte[]? Png, bool Closed = false, string Purpose = "Research");
public interface IResearchBrowser
{
    Task<SearchBatch> SearchAsync(string question, int candidateLimit, CancellationToken cancellationToken);
    Task<WebPage> ReadAsync(string url, CancellationToken cancellationToken);
}
public sealed record SearchEngine(string Name, string QueryUrl, string ResultSelector);
public sealed record BrowserResearchOptions(SearchEngine[] Engines, string? ChromeBinary = null, string? DriverDirectory = null, bool Headless = true)
{
    public static BrowserResearchOptions Default => new([
        new("Bing", "https://www.bing.com/search?q={query}&count=30", "li.b_algo h2 a"),
        new("DuckDuckGo", "https://duckduckgo.com/?q={query}", "a[data-testid='result-title-a'],a.result__a"),
        new("Google", "https://www.google.com/search?q={query}&num=30", "#search a:has(h3)")]);
    public static BrowserResearchOptions Load()
    {
        string? path = Environment.GetEnvironmentVariable("LILITH_WEB_CONFIG");
        var options = path is null ? Default : System.Text.Json.JsonSerializer.Deserialize<BrowserResearchOptions>(File.ReadAllText(path), new System.Text.Json.JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? throw new InvalidDataException("Invalid browser research configuration.");
        if (options.Engines is null || options.Engines.Length is < 1 or > 8 || options.Engines.Any(e => e is null || string.IsNullOrWhiteSpace(e.Name) || e.QueryUrl is null || !e.QueryUrl.Contains("{query}") || string.IsNullOrWhiteSpace(e.ResultSelector)) || options.Engines.Select(e => e.Name).Distinct(StringComparer.OrdinalIgnoreCase).Count() != options.Engines.Length) throw new InvalidDataException("Configure 1-8 uniquely named search engines with a {query} URL and result selector.");
        foreach (var engine in options.Engines) PublicWeb.ValidateUri(engine.QueryUrl.Replace("{query}", "test"));
        return options;
    }
}