V14 / fad4e1a110e5
ProjectRuntimeRunner.cs
Lilith/Core/Projects/ProjectRuntimeRunner.cs
Unchanged from the preceding versionCS
using System.Diagnostics;
using System.Text.Json;
using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using LogType = OpenQA.Selenium.LogType;
namespace Lilith.Core.Projects;
internal static class ProjectRuntimeRunner
{
private sealed class ProbeException(string message) : Exception(message);
internal static async Task<ProjectRuntimeReport> RunAsync(ProjectDraft draft, string folder, CancellationToken ct, Action<WebResearch.ResearchBrowserFrame>? capture = null, ProjectBrowserSession? browserSession = null)
{
using var deadline = CancellationTokenSource.CreateLinkedTokenSource(ct); deadline.CancelAfter(TimeSpan.FromMinutes(3)); ct = deadline.Token;
ProjectDraftStore.ValidateFiles(draft.Files, draft.Kind); ProjectDraftStore.ValidateTests(draft.Tests, draft.Kind);
Directory.CreateDirectory(folder);
var results = new List<ProjectCaseResult>(); var errors = new List<string>();
var build = await ProjectValidator.ValidateAsync(draft.Kind, new(draft.Files), draft.Architecture.Packages, folder, ct, additionalFiles: true);
await File.WriteAllTextAsync(Path.Combine(folder, "build.json"), ProjectJson.Serialize(build), ct);
if (!build.Passed) errors.AddRange(build.Errors.Append(build.BuildOutput).Where(s => s.Length > 0));
else if (draft.Kind == "console")
{
string dll = Directory.EnumerateFiles(build.BuildDirectory ?? throw new IOException("Missing host build directory."), "Application.dll", SearchOption.AllDirectories).First(p => p.Contains(Path.Combine("bin", "Release")) && !p.Contains(Path.DirectorySeparatorChar + "ref" + Path.DirectorySeparatorChar));
foreach (var test in draft.Tests)
{
ct.ThrowIfCancellationRequested();
var start = new ProcessStartInfo("dotnet") { WorkingDirectory = Path.GetDirectoryName(dll), UseShellExecute = false, RedirectStandardInput = true, RedirectStandardOutput = true, RedirectStandardError = true, CreateNoWindow = true };
start.ArgumentList.Add(dll);
using var process = Process.Start(start) ?? throw new IOException("Could not execute the approved application.");
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); timeout.CancelAfter(TimeSpan.FromSeconds(10));
async Task<string> ReadBounded(StreamReader reader)
{
var output = new System.Text.StringBuilder(); char[] buffer = new char[1024]; int count;
while ((count = await reader.ReadAsync(buffer.AsMemory(), timeout.Token)) > 0) { if (output.Length + count > 32000) throw new IOException("Application output exceeded 32000 characters."); output.Append(buffer, 0, count); }
return output.ToString();
}
var stdout = ReadBounded(process.StandardOutput); var stderr = ReadBounded(process.StandardError);
try
{
await process.StandardInput.WriteAsync(test.Stdin.AsMemory(), timeout.Token); process.StandardInput.Close();
// Faulted output capture must terminate the process instead of waiting for its pipe to unblock.
var capturedOutput = Task.WhenAll(stdout, stderr);
var exited = process.WaitForExitAsync(timeout.Token);
await Task.WhenAny(capturedOutput, exited); if (capturedOutput.IsFaulted) await capturedOutput;
await exited; await capturedOutput;
string actual = await stdout + "\n" + await stderr;
bool passed = process.ExitCode == test.ExitCode && (test.Contains ?? []).All(actual.Contains) && !(test.NotContains ?? []).Any(actual.Contains);
string observed = "Exit code: " + process.ExitCode + "\n" + actual;
File.WriteAllText(Path.Combine(folder, "case-" + test.Id + ".json"), ProjectJson.Serialize(new { test, exitCode = process.ExitCode, stdout = await stdout, stderr = await stderr }));
results.Add(new(test.Id, passed ? "passed" : "failed", ProjectJson.Serialize(new { test.ExitCode, test.Contains, test.NotContains }), observed));
}
catch (Exception ex) when (!ct.IsCancellationRequested)
{ results.Add(new(test.Id, "test_error", "Application exits with the expected output within 10 seconds", ex.Message)); }
finally { if (!process.HasExited) process.Kill(true); }
}
}
else await Task.Run(() => Browser(draft, folder, results, ct, capture, browserSession), ct);
bool passedAll = build.Passed && results.Count == draft.Tests.Length && results.All(r => r.Status == "passed");
var report = new ProjectRuntimeReport(passedAll, ProjectDraftStore.Hash(draft.Files), ProjectDraftStore.PlanHash(draft.Tests), results.ToArray(), errors.ToArray(), folder);
await File.WriteAllTextAsync(Path.Combine(folder, "runtime.json"), ProjectJson.Serialize(report), ct); return report;
}
private static void Browser(ProjectDraft draft, string folder, List<ProjectCaseResult> results, CancellationToken ct, Action<WebResearch.ResearchBrowserFrame>? capture, ProjectBrowserSession? browserSession)
{
string app = Path.Combine(folder, "app"); Directory.CreateDirectory(app);
foreach (var file in draft.Files) { string path = Path.Combine(app, file.Path); Directory.CreateDirectory(Path.GetDirectoryName(path)!); File.WriteAllText(path, file.Content); }
using var owned = browserSession is null ? new ProjectBrowserSession() : null;
var session = browserSession ?? owned!; session.Name = draft.Name; session.Capture = capture;
var browser = session.Get();
void Evaluate(string expression, bool assertion)
{
var value = JsonSerializer.SerializeToElement(browser.ExecuteCdpCommand("Runtime.evaluate", new() { ["expression"] = expression, ["returnByValue"] = true, ["awaitPromise"] = true, ["timeout"] = 1500 }));
if (value.TryGetProperty("exceptionDetails", out var exception)) throw new ProbeException("Browser probe error: " + exception.GetRawText());
if (assertion && (!value.GetProperty("result").TryGetProperty("value", out var result) || result.ValueKind != JsonValueKind.True)) throw new InvalidOperationException("Expected true; observed " + value.GetProperty("result").GetRawText() + ". Expression: " + expression);
}
void Capture() { if (capture is not null) capture(new(DateTimeOffset.UtcNow, "project://" + draft.Name + "/index.html", browser.GetScreenshot().AsByteArray, Purpose: "Project testing")); }
foreach (var test in draft.Tests)
{
ct.ThrowIfCancellationRequested(); var trace = new List<object>(); string status = "passed", actual = "All declared browser assertions passed.";
try
{
browser.ExecuteCdpCommand("Emulation.clearDeviceMetricsOverride", new());
browser.Manage().Window.Size = new(1440, 1080);
if (browser.Url.StartsWith(new Uri(app + Path.DirectorySeparatorChar).AbsoluteUri, StringComparison.Ordinal)) browser.ExecuteScript("try { localStorage.clear(); sessionStorage.clear(); } catch {} ");
_ = browser.Manage().Logs.GetLog(LogType.Browser);
browser.Navigate().GoToUrl(new Uri(Path.Combine(app, "index.html")).AbsoluteUri);
Capture();
foreach (var action in test.Actions!)
{
ct.ThrowIfCancellationRequested(); trace.Add(action);
if (!browser.Url.StartsWith(new Uri(app + Path.DirectorySeparatorChar).AbsoluteUri, StringComparison.Ordinal)) throw new IOException("Application navigated outside its approved local project.");
switch (action.Type)
{
case "click": browser.FindElement(By.CssSelector(action.Selector)).Click(); break;
case "type": var element = browser.FindElement(By.CssSelector(action.Selector)); element.Clear(); element.SendKeys(action.Value); break;
case "key": browser.FindElement(By.CssSelector(action.Selector.Length == 0 ? "body" : action.Selector)).SendKeys(action.Value switch { "ArrowUp" => Keys.ArrowUp, "ArrowDown" => Keys.ArrowDown, "ArrowLeft" => Keys.ArrowLeft, "ArrowRight" => Keys.ArrowRight, "Enter" => Keys.Enter, "Escape" => Keys.Escape, "Tab" => Keys.Tab, _ => action.Value }); break;
case "wait": if (!int.TryParse(action.Value, out int ms) || ms is < 0 or > 1000) throw new InvalidDataException("Wait must be 0-1000ms."); if (ct.WaitHandle.WaitOne(ms)) ct.ThrowIfCancellationRequested(); break;
case "viewport": browser.ExecuteCdpCommand("Emulation.setDeviceMetricsOverride", new() { ["width"] = action.Width, ["height"] = action.Height, ["deviceScaleFactor"] = 1, ["mobile"] = action.Width < 700 }); break;
case "script": Evaluate(action.Value, false); break;
case "assert_js": Evaluate(action.Value, true); break;
case "assert_text":
string text = browser.FindElement(By.CssSelector(action.Selector)).Text;
trace.Add(new { assertion = "text", expectedSubstring = action.Value, observed = text });
if (!text.Contains(action.Value, StringComparison.Ordinal)) throw new InvalidOperationException("Expected visible text containing " + action.Value + "; observed: " + text); break;
case "assert_visible": if (!browser.FindElement(By.CssSelector(action.Selector)).Displayed) throw new InvalidOperationException("Expected visible element: " + action.Selector); break;
}
if (action.Type is "click" or "key" or "type" or "wait" or "viewport") Capture();
}
var errors = browser.Manage().Logs.GetLog(LogType.Browser).Where(e => e.Level == LogLevel.Severe).Select(e => e.Message).ToArray();
if (errors.Length > 0) throw new InvalidOperationException(string.Join("\n", errors));
}
catch (Exception ex) when (!ct.IsCancellationRequested) { status = ex is InvalidDataException or ProbeException ? "test_error" : "failed"; actual = ex.Message; }
try { browser.GetScreenshot().SaveAsFile(Path.Combine(folder, "case-" + test.Id + ".png")); } catch (WebDriverException) { }
File.WriteAllText(Path.Combine(folder, "case-" + test.Id + ".json"), ProjectJson.Serialize(new { test, trace, status, actual }));
results.Add(new(test.Id, status, test.Requirement, actual));
}
}
}