V11 / ca6f28eff240
SeleniumResearchBrowser.cs
Lilith/Core/WebResearch/SeleniumResearchBrowser.cs
Unchanged from the preceding versionCS
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
namespace Lilith.Core.WebResearch;
public sealed class SeleniumResearchBrowser(BrowserResearchOptions? options = null, Action<string, string>? captureSearchPage = null, Action<string>? reportLifecycle = null,
Action<ResearchBrowserFrame>? captureFrame = null) : IResearchBrowser, IDisposable
{
private readonly object _sync = new();
private ChromeDriver? _driver;
private PublicWebProxy? _proxy;
private bool _disposed;
private Timer? _captureTimer;
private int _capturedFrames;
private void Capture()
{
if (captureFrame is null || !Monitor.TryEnter(_sync)) return;
try
{
if (_disposed || _driver is null || _capturedFrames >= 1200) return;
var png = _driver.GetScreenshot().AsByteArray;
captureFrame(new(DateTimeOffset.UtcNow, _driver.Url, png));
_capturedFrames++;
}
catch (Exception ex) { Report("Browser capture unavailable: " + ex.GetType().Name); }
finally { Monitor.Exit(_sync); }
}
private BrowserResearchOptions Options => options ?? BrowserResearchOptions.Load();
private Task<T> UseAsync<T>(Func<ChromeDriver, T> action, CancellationToken ct) => Task.Run(() =>
{
lock (_sync)
{
ObjectDisposedException.ThrowIf(_disposed, this);
ct.ThrowIfCancellationRequested();
if (_driver is null)
{
_proxy = new PublicWebProxy();
try
{
_driver = Create(_proxy); Report("Selenium session opened: " + _driver.SessionId);
if (captureFrame is not null)
{
Capture();
_captureTimer = new Timer(_ => Capture(), null, 1000, 1000);
}
}
catch { _proxy.Dispose(); _proxy = null; throw; }
}
try { return action(_driver); }
finally { Capture(); }
}
}, ct);
public void Dispose()
{
lock (_sync)
{
if (_disposed) return;
_disposed = true;
_captureTimer?.Dispose();
string? session = _driver?.SessionId?.ToString();
try { _driver?.Dispose(); }
finally
{
_driver = null; _proxy?.Dispose(); _proxy = null;
if (session is not null) Report("Selenium session closed: " + session);
if (session is not null && captureFrame is not null)
try { captureFrame(new(DateTimeOffset.UtcNow, "", null, Closed: true)); } catch { /* Capture cannot own disposal. */ }
}
}
}
private void Report(string message) { try { reportLifecycle?.Invoke(message); } catch { /* Diagnostics never own execution. */ } }
private ChromeDriver Create(PublicWebProxy proxy)
{
Environment.SetEnvironmentVariable("SE_AVOID_STATS", "true");
var settings = new ChromeOptions { PageLoadStrategy = PageLoadStrategy.Eager };
string? binary = Options.ChromeBinary ?? Environment.GetEnvironmentVariable("LILITH_CHROME_BINARY") ?? new[] {
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles), "Google/Chrome/Application/chrome.exe"),
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), "Google/Chrome/Application/chrome.exe"),
Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "Google/Chrome/Application/chrome.exe") }.FirstOrDefault(File.Exists);
if (binary is null || !File.Exists(binary)) throw new InvalidOperationException("An existing Chrome installation is required. Set LILITH_CHROME_BINARY if necessary.");
settings.BinaryLocation = binary;
if (Options.Headless) settings.AddArgument("--headless=new");
settings.AddArguments("--disable-quic", "--disable-background-networking", "--no-first-run", "--force-webrtc-ip-handling-policy=disable_non_proxied_udp", "--proxy-server=http://127.0.0.1:" + proxy.Port, "--proxy-bypass-list=<-loopback>");
if (captureFrame is null) settings.AddUserProfilePreference("profile.managed_default_content_settings.images", 2);
if (captureFrame is not null) settings.AddArgument("--window-size=960,1080");
settings.AddUserProfilePreference("profile.default_content_setting_values.automatic_downloads", 2);
var service = Options.DriverDirectory is { Length: > 0 } directory ? ChromeDriverService.CreateDefaultService(directory) : ChromeDriverService.CreateDefaultService();
service.HideCommandPromptWindow = true; service.SuppressInitialDiagnosticInformation = true;
ChromeDriver? driver = null;
try
{
driver = new ChromeDriver(service, settings, TimeSpan.FromSeconds(40));
driver.ExecuteCdpCommand("Browser.setDownloadBehavior", new Dictionary<string, object?> { ["behavior"] = "deny" });
driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(25);
driver.Manage().Timeouts().AsynchronousJavaScript = TimeSpan.FromSeconds(5);
return driver;
}
catch { driver?.Dispose(); service.Dispose(); throw; }
}
public Task<SearchBatch> SearchAsync(string question, int candidateLimit, CancellationToken ct) => UseAsync(driver =>
{
var found = new Dictionary<string, SearchSource>(StringComparer.Ordinal); var outcomes = new List<EngineOutcome>();
foreach (var engine in Options.Engines)
{
ct.ThrowIfCancellationRequested();
try
{
driver.Navigate().GoToUrl(engine.QueryUrl.Replace("{query}", Uri.EscapeDataString(question)));
WaitFor(driver, engine.ResultSelector, ct);
Capture();
captureSearchPage?.Invoke(engine.Name, driver.PageSource);
int count = 0;
foreach (var anchor in driver.FindElements(By.CssSelector(engine.ResultSelector)).Take(60))
{
string? href = anchor.GetAttribute("href");
if (href is null) continue;
string url;
try { url = PublicWeb.ValidateUri(Unwrap(href)).AbsoluteUri; } catch (ArgumentException) { continue; }
if (new Uri(url).Host == new Uri(engine.QueryUrl.Replace("{query}", "test")).Host) continue;
// Public result markup can arrive before the engine reveals
// its styled result panel. WebDriver's visible Text is empty
// then, although the actual link/title is already in the DOM.
string title = anchor.Text.Trim();
if (title.Length == 0) title = (anchor.GetDomProperty("textContent") ?? "").Trim();
if (title.Length == 0) continue;
count++; found.TryAdd(url, new(url, title[..Math.Min(title.Length, 200)], engine.Name));
}
string diagnostic = count > 0 ? "Public result links extracted." : DescribeUnavailable(driver);
outcomes.Add(new(engine.Name, count > 0 ? "completed" : "unavailable", diagnostic));
}
catch (WebDriverException) { outcomes.Add(new(engine.Name, "unavailable", "Navigation failed or the engine blocked automated access.")); }
}
// Interleave engine results so a first engine cannot fill every selected slot.
var queues = Options.Engines.Select(e => new Queue<SearchSource>(found.Values.Where(s => s.Engine == e.Name))).ToArray();
var ordered = new List<SearchSource>();
while (queues.Any(q => q.Count > 0)) foreach (var queue in queues) if (queue.TryDequeue(out var source)) ordered.Add(source);
return new SearchBatch(ordered.Take(candidateLimit).ToArray(), outcomes.ToArray());
}, ct);
public Task<WebPage> ReadAsync(string url, CancellationToken ct) => UseAsync<WebPage>(driver =>
{
var uri = PublicWeb.ValidateUri(url); ct.ThrowIfCancellationRequested();
try
{
driver.Navigate().GoToUrl(uri.AbsoluteUri); WaitFor(driver, "body", ct);
Capture();
string text = "";
var readyUntil = DateTime.UtcNow.AddSeconds(8);
do
{
ct.ThrowIfCancellationRequested();
text = ((IJavaScriptExecutor)driver).ExecuteScript("return (document.querySelector('article,main,[role=main]') || document.body)?.innerText || '';")?.ToString() ?? "";
if (text.Trim().Length >= 160) break;
ct.WaitHandle.WaitOne(200);
} while (DateTime.UtcNow < readyUntil);
string final = PublicWeb.ValidateUri(driver.Url).AbsoluteUri;
string lower = text[..Math.Min(text.Length, 1500)].ToLowerInvariant();
bool blocked = new[] { "verify you are human", "unusual traffic", "access denied", "enable javascript and cookies", "checking your browser", "complete the captcha" }.Any(lower.Contains);
if (blocked || text.Trim().Length < 160) return new(final, driver.Title, false, "", "Blocked, unreadable or too little public text.", DateTimeOffset.UtcNow);
ct.ThrowIfCancellationRequested();
return new(final, driver.Title[..Math.Min(driver.Title.Length, 200)], true, text[..Math.Min(text.Length, 16000)], "", DateTimeOffset.UtcNow);
}
catch (WebDriverException) { return new(uri.AbsoluteUri, "", false, "", "Page navigation failed or timed out.", DateTimeOffset.UtcNow); }
}, ct);
private static void WaitFor(ChromeDriver driver, string selector, CancellationToken ct)
{
var until = DateTime.UtcNow.AddSeconds(8);
do { ct.ThrowIfCancellationRequested(); if (driver.FindElements(By.CssSelector(selector)).Count > 0) return; ct.WaitHandle.WaitOne(200); } while (DateTime.UtcNow < until);
}
private static string DescribeUnavailable(ChromeDriver driver)
{
string text = ((IJavaScriptExecutor)driver).ExecuteScript("return document.body?.innerText || '';")?.ToString() ?? "";
string markupText = driver.FindElement(By.TagName("body")).GetDomProperty("textContent") ?? text;
if (new[] { "unusual traffic", "verify you are human", "captcha", "checking your browser", "complete the following challenge", "bots use DuckDuckGo too" }.Any(s => markupText.Contains(s, StringComparison.OrdinalIgnoreCase)))
return "The engine displayed an automated-access challenge; no results were extracted.";
text = string.Join(" ", text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries));
return $"No result links. Page: {driver.Title}; host: {new Uri(driver.Url).Host}; excerpt: {text[..Math.Min(text.Length, 600)]}";
}
private static string Unwrap(string href)
{
if (!Uri.TryCreate(href, UriKind.Absolute, out var uri)) return href;
var query = uri.Query.TrimStart('?').Split('&').Select(p => p.Split('=', 2)).Where(p => p.Length == 2).GroupBy(p => p[0], StringComparer.OrdinalIgnoreCase).ToDictionary(g => g.Key, g => Uri.UnescapeDataString(g.First()[1]), StringComparer.OrdinalIgnoreCase);
bool Host(string domain) => uri.Host.Equals(domain, StringComparison.OrdinalIgnoreCase) || uri.Host.EndsWith("." + domain, StringComparison.OrdinalIgnoreCase);
if (Host("duckduckgo.com") && query.TryGetValue("uddg", out var target)) return target;
if (Host("google.com") && uri.AbsolutePath == "/url" && query.TryGetValue("q", out target)) return target;
if (Host("bing.com") && query.TryGetValue("u", out target) && target.StartsWith("a1"))
{
try { string encoded = target[2..].Replace('-', '+').Replace('_', '/'); return System.Text.Encoding.UTF8.GetString(Convert.FromBase64String(encoded.PadRight((encoded.Length + 3) / 4 * 4, '='))); } catch (FormatException) { }
}
return href;
}
}