web_fetch_operation

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

using System.Text.Json;
using System.Text.RegularExpressions;

namespace Lilith.Core.WebResearch;

internal static class WebOperations
{
    internal static readonly string[] Names = ["web_search_operation", "web_fetch_operation", "web_check_answer_operation"];
    internal static async Task<string> ExecuteAsync(IResearchBrowser browser, string name, IReadOnlyDictionary<string, string> inputs, CancellationToken ct)
    {
        string Get(string key) => inputs.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value) ? value : throw new ArgumentException("Missing web input: " + key);
        switch (name)
        {
            case "web_search_operation":
                string question = Get("question");
                if (question.Length > 2000 || !int.TryParse(Get("source_count"), out int count) || count is < 1 or > 20) throw new ArgumentException("Question must be at most 2000 characters; source_count must be 1-20.");
                return JsonSerializer.Serialize(await browser.SearchAsync(question, Math.Min(30, count * 3), ct), JsonOptions);
            case "web_fetch_operation":
                string url = PublicWeb.ValidateUri(Get("url")).AbsoluteUri;
                try { return JsonSerializer.Serialize(await browser.ReadAsync(url, ct), JsonOptions); }
                catch (Exception ex) when (ex is IOException or System.Net.Http.HttpRequestException or ArgumentException)
                { return JsonSerializer.Serialize(new WebPage(url, "", false, "", "Public page could not be read.", DateTimeOffset.UtcNow), JsonOptions); }
            case "web_check_answer_operation":
                using (var evidence = JsonDocument.Parse(Get("evidence")))
                {
                    var data = evidence.RootElement;
                    var urls = data.GetProperty("items").EnumerateArray().Where(i => i.TryGetProperty("readable", out var readable) && bool.TryParse(readable.ToString(), out bool valid) && valid)
                        .Select(i => PublicWeb.ValidateUri(i.GetProperty("url").GetString()!).AbsoluteUri).ToHashSet(StringComparer.Ordinal);
                    int requested = data.GetProperty("requested").GetInt32(), attempts = data.GetProperty("attempted").GetInt32();
                    if (urls.Count == 0) return "I could not verify an answer from readable public sources. Search engines or pages may be unavailable, blocked, or require a browser interaction. No current answer is claimed. Sources read: 0 of " + requested + ".";
                    string answer = Get("answer");
                    var links = Regex.Matches(answer, @"\[[^\]]*\]\(([^\s)]+)\)");
                    if (links.Count == 0 || links.Cast<Match>().Any(m => !Uri.TryCreate(m.Groups[1].Value, UriKind.Absolute, out var uri) || !urls.Contains(uri.AbsoluteUri)))
                        throw new InvalidDataException("Synthesis cited no verified source links, or cited a page that was not read.");
                    return answer + $"\n\nSources read: {urls.Count} of {requested} requested ({attempts} page attempts).";
                }
            default: throw new ArgumentException("Unknown research operation.");
        }
    }
    private static readonly JsonSerializerOptions JsonOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower };
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192

using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;

namespace Lilith.Core.WebResearch;

public sealed class SeleniumResearchBrowser(BrowserResearchOptions? options = null, Action<string, string>? captureSearchPage = null, Action<string>? reportLifecycle = null,
    Action<ResearchBrowserFrame>? captureFrame = null) : IResearchBrowser, IDisposable
{
    private readonly object _sync = new();
    private ChromeDriver? _driver;
    private PublicWebProxy? _proxy;
    private bool _disposed;
    private Timer? _captureTimer;
    private int _capturedFrames;
    private void Capture()
    {
        if (captureFrame is null || !Monitor.TryEnter(_sync)) return;
        try
        {
            if (_disposed || _driver is null || _capturedFrames >= 1200) return;
            var png = _driver.GetScreenshot().AsByteArray;
            captureFrame(new(DateTimeOffset.UtcNow, _driver.Url, png));
            _capturedFrames++;
        }
        catch (Exception ex) { Report("Browser capture unavailable: " + ex.GetType().Name); }
        finally { Monitor.Exit(_sync); }
    }
    private BrowserResearchOptions Options => options ?? BrowserResearchOptions.Load();
    private Task<T> UseAsync<T>(Func<ChromeDriver, T> action, CancellationToken ct) => Task.Run(() =>
    {
        lock (_sync)
        {
            ObjectDisposedException.ThrowIf(_disposed, this);
            ct.ThrowIfCancellationRequested();
            if (_driver is null)
            {
                _proxy = new PublicWebProxy();
                try
                {
                    _driver = Create(_proxy); Report("Selenium session opened: " + _driver.SessionId);
                    if (captureFrame is not null)
                    {
                        Capture();
                        _captureTimer = new Timer(_ => Capture(), null, 1000, 1000);
                    }
                }
                catch { _proxy.Dispose(); _proxy = null; throw; }
            }
            try { return action(_driver); }
            finally { Capture(); }
        }
    }, ct);

    public void Dispose()
    {
        lock (_sync)
        {
            if (_disposed) return;
            _disposed = true;
            _captureTimer?.Dispose();
            string? session = _driver?.SessionId?.ToString();
            try { _driver?.Dispose(); }
            finally
            {
                _driver = null; _proxy?.Dispose(); _proxy = null;
                if (session is not null) Report("Selenium session closed: " + session);
                if (session is not null && captureFrame is not null)
                    try { captureFrame(new(DateTimeOffset.UtcNow, "", null, Closed: true)); } catch { /* Capture cannot own disposal. */ }
            }
        }
    }
    private void Report(string message) { try { reportLifecycle?.Invoke(message); } catch { /* Diagnostics never own execution. */ } }
    private ChromeDriver Create(PublicWebProxy proxy)
    {
        Environment.SetEnvironmentVariable("SE_AVOID_STATS", "true");
        var settings = new ChromeOptions { PageLoadStrategy = PageLoadStrategy.Eager };
        string? binary = Options.ChromeBinary ?? Environment.GetEnvironmentVariable("LILITH_CHROME_BINARY") ?? new[] {
            Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "Google/Chrome/Application/chrome.exe"),
            Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), "Google/Chrome/Application/chrome.exe"),
            Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Google/Chrome/Application/chrome.exe") }.FirstOrDefault(File.Exists);
        if (binary is null || !File.Exists(binary)) throw new InvalidOperationException("An existing Chrome installation is required. Set LILITH_CHROME_BINARY if necessary.");
        settings.BinaryLocation = binary;
        if (Options.Headless) settings.AddArgument("--headless=new");
        settings.AddArguments("--disable-quic", "--disable-background-networking", "--no-first-run", "--force-webrtc-ip-handling-policy=disable_non_proxied_udp", "--proxy-server=http://127.0.0.1:" + proxy.Port, "--proxy-bypass-list=<-loopback>");
        if (captureFrame is null) settings.AddUserProfilePreference("profile.managed_default_content_settings.images", 2);
        if (captureFrame is not null) settings.AddArgument("--window-size=960,1080");
        settings.AddUserProfilePreference("profile.default_content_setting_values.automatic_downloads", 2);
        var service = Options.DriverDirectory is { Length: > 0 } directory ? ChromeDriverService.CreateDefaultService(directory) : ChromeDriverService.CreateDefaultService();
        service.HideCommandPromptWindow = true; service.SuppressInitialDiagnosticInformation = true;
        ChromeDriver? driver = null;
        try
        {
            driver = new ChromeDriver(service, settings, TimeSpan.FromSeconds(40));
            driver.ExecuteCdpCommand("Browser.setDownloadBehavior", new Dictionary<string, object?> { ["behavior"] = "deny" });
            driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(25);
            driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(5);
            return driver;
        }
        catch { driver?.Dispose(); service.Dispose(); throw; }
    }
    public Task<SearchBatch> SearchAsync(string question, int candidateLimit, CancellationToken ct) => UseAsync(driver =>
    {
        var found = new Dictionary<string, SearchSource>(StringComparer.Ordinal); var outcomes = new List<EngineOutcome>();
        foreach (var engine in Options.Engines)
        {
            ct.ThrowIfCancellationRequested();
            try
            {
                driver.Navigate().GoToUrl(engine.QueryUrl.Replace("{query}", Uri.EscapeDataString(question)));
                WaitFor(driver, engine.ResultSelector, ct);
                Capture();
                captureSearchPage?.Invoke(engine.Name, driver.PageSource);
                int count = 0;
                foreach (var anchor in driver.FindElements(By.CssSelector(engine.ResultSelector)).Take(60))
                {
                    string? href = anchor.GetAttribute("href");
                    if (href is null) continue;
                    string url;
                    try { url = PublicWeb.ValidateUri(Unwrap(href)).AbsoluteUri; } catch (ArgumentException) { continue; }
                    if (new Uri(url).Host == new Uri(engine.QueryUrl.Replace("{query}", "test")).Host) continue;
                    // Public result markup can arrive before the engine reveals
                    // its styled result panel. WebDriver's visible Text is empty
                    // then, although the actual link/title is already in the DOM.
                    string title = anchor.Text.Trim();
                    if (title.Length == 0) title = (anchor.GetDomProperty("textContent") ?? "").Trim();
                    if (title.Length == 0) continue;
                    count++; found.TryAdd(url, new(url, title[..Math.Min(title.Length, 200)], engine.Name));
                }
                string diagnostic = count > 0 ? "Public result links extracted." : DescribeUnavailable(driver);
                outcomes.Add(new(engine.Name, count > 0 ? "completed" : "unavailable", diagnostic));
            }
            catch (WebDriverException) { outcomes.Add(new(engine.Name, "unavailable", "Navigation failed or the engine blocked automated access.")); }
        }
        // Interleave engine results so a first engine cannot fill every selected slot.
        var queues = Options.Engines.Select(e => new Queue<SearchSource>(found.Values.Where(s => s.Engine == e.Name))).ToArray();
        var ordered = new List<SearchSource>();
        while (queues.Any(q => q.Count > 0)) foreach (var queue in queues) if (queue.TryDequeue(out var source)) ordered.Add(source);
        return new SearchBatch(ordered.Take(candidateLimit).ToArray(), outcomes.ToArray());
    }, ct);
    public Task<WebPage> ReadAsync(string url, CancellationToken ct) => UseAsync<WebPage>(driver =>
    {
        var uri = PublicWeb.ValidateUri(url); ct.ThrowIfCancellationRequested();
        try
        {
            driver.Navigate().GoToUrl(uri.AbsoluteUri); WaitFor(driver, "body", ct);
            Capture();
            string text = "";
            var readyUntil = DateTime.UtcNow.AddSeconds(8);
            do
            {
                ct.ThrowIfCancellationRequested();
                text = ((IJavaScriptExecutor)driver).ExecuteScript("return (document.querySelector('article,main,[role=main]') || document.body)?.innerText || '';")?.ToString() ?? "";
                if (text.Trim().Length >= 160) break;
                ct.WaitHandle.WaitOne(200);
            } while (DateTime.UtcNow < readyUntil);
            string final = PublicWeb.ValidateUri(driver.Url).AbsoluteUri;
            string lower = text[..Math.Min(text.Length, 1500)].ToLowerInvariant();
            bool blocked = new[] { "verify you are human", "unusual traffic", "access denied", "enable javascript and cookies", "checking your browser", "complete the captcha" }.Any(lower.Contains);
            if (blocked || text.Trim().Length < 160) return new(final, driver.Title, false, "", "Blocked, unreadable or too little public text.", DateTimeOffset.UtcNow);
            ct.ThrowIfCancellationRequested();
            return new(final, driver.Title[..Math.Min(driver.Title.Length, 200)], true, text[..Math.Min(text.Length, 16000)], "", DateTimeOffset.UtcNow);
        }
        catch (WebDriverException) { return new(uri.AbsoluteUri, "", false, "", "Page navigation failed or timed out.", DateTimeOffset.UtcNow); }
    }, ct);
    private static void WaitFor(ChromeDriver driver, string selector, CancellationToken ct)
    {
        var until = DateTime.UtcNow.AddSeconds(8);
        do { ct.ThrowIfCancellationRequested(); if (driver.FindElements(By.CssSelector(selector)).Count > 0) return; ct.WaitHandle.WaitOne(200); } while (DateTime.UtcNow < until);
    }
    private static string DescribeUnavailable(ChromeDriver driver)
    {
        string text = ((IJavaScriptExecutor)driver).ExecuteScript("return document.body?.innerText || '';")?.ToString() ?? "";
        string markupText = driver.FindElement(By.TagName("body")).GetDomProperty("textContent") ?? text;
        if (new[] { "unusual traffic", "verify you are human", "captcha", "checking your browser", "complete the following challenge", "bots use DuckDuckGo too" }.Any(s => markupText.Contains(s, StringComparison.OrdinalIgnoreCase)))
            return "The engine displayed an automated-access challenge; no results were extracted.";
        text = string.Join(" ", text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
        return $"No result links. Page: {driver.Title}; host: {new Uri(driver.Url).Host}; excerpt: {text[..Math.Min(text.Length, 600)]}";
    }
    private static string Unwrap(string href)
    {
        if (!Uri.TryCreate(href, UriKind.Absolute, out var uri)) return href;
        var query = uri.Query.TrimStart('?').Split('&').Select(p => p.Split('=', 2)).Where(p => p.Length == 2).GroupBy(p => p[0], StringComparer.OrdinalIgnoreCase).ToDictionary(g => g.Key, g => Uri.UnescapeDataString(g.First()[1]), StringComparer.OrdinalIgnoreCase);
        bool Host(string domain) => uri.Host.Equals(domain, StringComparison.OrdinalIgnoreCase) || uri.Host.EndsWith("." + domain, StringComparison.OrdinalIgnoreCase);
        if (Host("duckduckgo.com") && query.TryGetValue("uddg", out var target)) return target;
        if (Host("google.com") && uri.AbsolutePath == "/url" && query.TryGetValue("q", out target)) return target;
        if (Host("bing.com") && query.TryGetValue("u", out target) && target.StartsWith("a1"))
        {
            try { string encoded = target[2..].Replace('-', '+').Replace('_', '/'); return System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(encoded.PadRight((encoded.Length + 3) / 4 * 4, '='))); } catch (FormatException) { }
        }
        return href;
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

using System.Net;
using System.Net.Sockets;

namespace Lilith.Core.WebResearch;

public static class PublicWeb
{
    public static Uri ValidateUri(string value)
    {
        if (!Uri.TryCreate(value, UriKind.Absolute, out var uri) || uri.Scheme is not ("https" or "http") || !string.IsNullOrEmpty(uri.UserInfo) || !uri.IsDefaultPort || uri.HostNameType == UriHostNameType.Unknown) throw new ArgumentException("Only public HTTP(S) pages on standard ports are supported.");
        string host = uri.DnsSafeHost.TrimEnd('.').ToLowerInvariant();
        if (!host.Contains('.') || host.EndsWith(".localhost") || host.EndsWith(".local") || host.EndsWith(".internal") || host.EndsWith(".test") || host.EndsWith(".invalid") || host.EndsWith(".onion") || IPAddress.TryParse(host.Trim('[', ']'), out var ip) && !IsPublic(ip)) throw new ArgumentException("Private and local web destinations are not allowed.");
        return new UriBuilder(uri) { Fragment = "" }.Uri;
    }
    public static bool IsPublic(IPAddress address)
    {
        if (address.IsIPv4MappedToIPv6) address = address.MapToIPv4();
        byte[] b = address.GetAddressBytes();
        if (address.AddressFamily == AddressFamily.InterNetworkV6)
            return (b[0] & 0xe0) == 0x20 && !(b[0] == 0x20 && b[1] == 0x01 && b[2] == 0x0d && b[3] == 0xb8) && !(b[0] == 0x20 && b[1] == 0x02) && !(b[0] == 0x20 && b[1] == 0x01 && b[2] == 0 && b[3] == 0);
        return b[0] is not (0 or 10 or 127) && b[0] < 224 && !(b[0] == 100 && b[1] is >= 64 and <= 127) && !(b[0] == 169 && b[1] == 254) && !(b[0] == 172 && b[1] is >= 16 and <= 31) && !(b[0] == 192 && (b[1] == 168 || b[1] == 0 || b[1] == 2)) && !(b[0] == 198 && (b[1] is 18 or 19 || b[1] == 51 && b[2] == 100)) && !(b[0] == 203 && b[1] == 0 && b[2] == 113);
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Collections.Concurrent;

namespace Lilith.Core.WebResearch;

// All browser traffic, including redirects and subresources, passes through this
// proxy. DNS results are checked and the socket connects to the checked address.
internal sealed class PublicWebProxy : IDisposable
{
    private readonly TcpListener _listener = new(IPAddress.Loopback, 0);
    private readonly CancellationTokenSource _stop = new();
    private readonly ConcurrentDictionary<int, TcpClient> _clients = new();
    private int _next;
    internal int Port => ((IPEndPoint)_listener.LocalEndpoint).Port;
    internal PublicWebProxy() { _listener.Start(32); _ = AcceptAsync(); }
    private async Task AcceptAsync()
    {
        try
        {
            while (!_stop.IsCancellationRequested)
            {
                var client = await _listener.AcceptTcpClientAsync(_stop.Token);
                if (_clients.Count >= 64) { client.Dispose(); continue; }
                int id = Interlocked.Increment(ref _next); _clients[id] = client;
                _ = HandleAsync(id, client);
            }
        }
        catch (Exception) when (_stop.IsCancellationRequested) { }
    }
    private async Task HandleAsync(int id, TcpClient client)
    {
        using (client)
        using (var deadline = CancellationTokenSource.CreateLinkedTokenSource(_stop.Token))
        {
            deadline.CancelAfter(TimeSpan.FromSeconds(45)); var ct = deadline.Token;
            try
            {
                var stream = client.GetStream(); var header = new List<byte>(); var one = new byte[1];
                while (header.Count < 16384)
                {
                    if (await stream.ReadAsync(one, ct) == 0) return;
                    header.Add(one[0]);
                    if (header.Count >= 4 && header[^4] == 13 && header[^3] == 10 && header[^2] == 13 && header[^1] == 10) break;
                }
                string[] lines = Encoding.ASCII.GetString(header.ToArray()).Split("\r\n");
                string[] request = lines[0].Split(' ');
                if (request.Length != 3) return;
                bool tunnel = request[0] == "CONNECT";
                if (!tunnel && request[0] is not ("GET" or "HEAD")) throw new InvalidDataException("Read-only browser proxy.");
                var uri = PublicWeb.ValidateUri(tunnel ? "https://" + request[1] : request[1]);
                var addresses = await Dns.GetHostAddressesAsync(uri.DnsSafeHost, ct);
                if (addresses.Length == 0 || addresses.Any(ip => !PublicWeb.IsPublic(ip))) throw new InvalidDataException("Non-public DNS destination.");
                using var remote = new TcpClient();
                await remote.ConnectAsync(addresses[0], uri.Port, ct);
                var upstream = remote.GetStream();
                if (tunnel) await stream.WriteAsync(Encoding.ASCII.GetBytes("HTTP/1.1 200 Connection Established\r\n\r\n"), ct);
                else
                {
                    string outgoing = request[0] + " " + uri.PathAndQuery + " HTTP/1.1\r\n" + string.Join("\r\n", lines.Skip(1).Where(l => l.Length > 0 && !l.StartsWith("Proxy-", StringComparison.OrdinalIgnoreCase) && !l.StartsWith("Connection:", StringComparison.OrdinalIgnoreCase))) + "\r\nConnection: close\r\n\r\n";
                    await upstream.WriteAsync(Encoding.ASCII.GetBytes(outgoing), ct);
                }
                await Task.WhenAny(CopyAsync(stream, upstream, ct), CopyAsync(upstream, stream, ct));
                deadline.Cancel();
            }
            catch (Exception ex) when (ex is IOException or SocketException or OperationCanceledException or ArgumentException or InvalidDataException or ObjectDisposedException) { }
            finally { _clients.TryRemove(id, out _); }
        }
    }
    private static async Task CopyAsync(Stream from, Stream to, CancellationToken ct)
    {
        byte[] buffer = new byte[16384]; int total = 0;
        try
        {
            while (await from.ReadAsync(buffer, ct) is int read && read > 0)
            {
                total += read; if (total > 8_000_000) break;
                await to.WriteAsync(buffer.AsMemory(0, read), ct);
            }
        }
        catch (Exception ex) when (ex is IOException or OperationCanceledException or ObjectDisposedException) { }
    }
    public void Dispose()
    {
        _stop.Cancel(); _listener.Stop();
        foreach (var client in _clients.Values) client.Dispose();
        _stop.Dispose();
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

namespace Lilith.Core.WebResearch;

public sealed record SearchSource(string Url, string Title, string Engine);
public sealed record EngineOutcome(string Engine, string Status, string Message);
public sealed record SearchBatch(SearchSource[] Results, EngineOutcome[] Engines);
public sealed record WebPage(string Url, string Title, bool Readable, string Text, string Error, DateTimeOffset RetrievedAt);
public sealed record ResearchBrowserFrame(DateTimeOffset AtUtc, string Url, byte[]? Png, bool Closed = false, string Purpose = "Research");
public interface IResearchBrowser
{
    Task<SearchBatch> SearchAsync(string question, int candidateLimit, CancellationToken cancellationToken);
    Task<WebPage> ReadAsync(string url, CancellationToken cancellationToken);
}
public sealed record SearchEngine(string Name, string QueryUrl, string ResultSelector);
public sealed record BrowserResearchOptions(SearchEngine[] Engines, string? ChromeBinary = null, string? DriverDirectory = null, bool Headless = true)
{
    public static BrowserResearchOptions Default => new([
        new("Bing", "https://www.bing.com/search?q={query}&count=30", "li.b_algo h2 a"),
        new("DuckDuckGo", "https://duckduckgo.com/?q={query}", "a[data-testid='result-title-a'],a.result__a"),
        new("Google", "https://www.google.com/search?q={query}&num=30", "#search a:has(h3)")]);
    public static BrowserResearchOptions Load()
    {
        string? path = Environment.GetEnvironmentVariable("LILITH_WEB_CONFIG");
        var options = path is null ? Default : System.Text.Json.JsonSerializer.Deserialize<BrowserResearchOptions>(File.ReadAllText(path), new System.Text.Json.JsonSerializerOptions { PropertyNameCaseInsensitive = true }) ?? throw new InvalidDataException("Invalid browser research configuration.");
        if (options.Engines is null || options.Engines.Length is < 1 or > 8 || options.Engines.Any(e => e is null || string.IsNullOrWhiteSpace(e.Name) || e.QueryUrl is null || !e.QueryUrl.Contains("{query}") || string.IsNullOrWhiteSpace(e.ResultSelector)) || options.Engines.Select(e => e.Name).Distinct(StringComparer.OrdinalIgnoreCase).Count() != options.Engines.Length) throw new InvalidDataException("Configure 1-8 uniquely named search engines with a {query} URL and result selector.");
        foreach (var engine in options.Engines) PublicWeb.ValidateUri(engine.QueryUrl.Replace("{query}", "test"));
        return options;
    }
}