V16 / 6df58cd3ed86
ProjectQualification.cs
Lilith/Core/Projects/ProjectQualification.cs
Changed in this versionCS
using Lilith.Core.Scenarios;
using Lilith.Core.Workspace;
using Lilith.Core.Workspace.Extensions;
namespace Lilith.Core.Projects;
internal static class ProjectQualification
{
private static string Clip(string value, int max) => value.Length <= max ? value : value[..max] + "\n[truncated; full evidence retained]";
internal static async Task<string> Scenario(ProviderClient provider, string name, IReadOnlyDictionary<string, string> inputs, CancellationToken ct)
{
var files = new ScenarioFileService();
var result = await new ScenarioExecutor(provider, files).RunXmlAsync(files.Bind(name, inputs), ct);
return result.Output;
}
internal static Dictionary<string, string> Inputs(ProjectDraft draft, string diagnostics = "No runtime evidence yet.") => new()
{
["kind"] = draft.Kind, ["prd"] = Clip(draft.Prd, 7000), ["request"] = Clip(draft.Request, 3000),
["source"] = ProjectIntegrationCodec.Display(new(draft.Files)), ["diagnostics"] = Clip(diagnostics, 5000)
};
internal static async Task<string> RunAsync(ProviderClient provider, ProjectDraftStore store, string id, CancellationToken ct,
Func<ProjectDraft, string, CancellationToken, Task<ProjectRuntimeReport>>? runner = null)
{
var draft = store.Load(id); ProjectDraftStore.ValidateTests(draft.Tests, draft.Kind);
if (draft.State is "published" or "awaiting_publication") throw new InvalidOperationException("Start a fresh qualification/edit session.");
string folder = store.Folder(id);
using var lease = new FileStream(Path.Combine(folder, "execution.lock"), FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
using var browserSession = new ProjectBrowserSession();
int firstRound = provider.Kanban?.OwnsQualification == true ? draft.Rounds + 1 : 1;
draft = draft with { State = "testing", Rounds = firstRound - 1, Runtime = null }; store.Save(draft);
try
{
for (int round = firstRound; round <= 10; round++)
{
provider.Kanban?.Boundary();
ct.ThrowIfCancellationRequested(); draft = store.Load(id) with { State = "testing", Rounds = round }; store.Save(draft);
string evidence = Path.Combine(folder, "runs", DateTime.UtcNow.ToString("yyyyMMdd-HHmmss") + "-" + Guid.NewGuid().ToString("N")[..8], "round-" + round);
evidence.EnsureNoLinks();
ProjectRuntimeReport report;
try { report = runner is null ? await ProjectRuntimeRunner.RunAsync(draft, evidence, ct, provider.HasProjectFrameCapture ? provider.CaptureProjectFrame : null, browserSession) : await runner(draft, evidence, ct); }
catch (Exception ex) when (!ct.IsCancellationRequested)
{
report = new(false, ProjectDraftStore.Hash(draft.Files), ProjectDraftStore.PlanHash(draft.Tests), [], ["Runtime infrastructure/test error: " + ex.Message], evidence);
Directory.CreateDirectory(evidence); File.WriteAllText(Path.Combine(evidence, "runtime.json"), ProjectJson.Serialize(report));
}
draft = draft with { Runtime = report with { Cases = report.Cases.Select(c => c with { Expected = Clip(c.Expected, 1000), Actual = Clip(c.Actual, 2000) }).ToArray(), Errors = report.Errors.Select(e => Clip(e, 2000)).ToArray() } }; store.Save(draft);
provider.Kanban?.RuntimeRound(report, round);
string diagnostics = ProjectJson.Serialize(new { report.Passed, report.Errors, cases = report.Cases.Where(c => c.Status != "passed").Select(c => c with { Actual = Clip(c.Actual, 1000) }).Take(10), report.Evidence }) + "\nPrevious repair/source-review issue: " + Clip(draft.SourceReview, 2000);
provider.ReportScenario(new(id, "project_test", "Round", $"Project {draft.Name}: round {round}/10, runtime {(report.Passed ? "passed" : "failed")}. Evidence: {evidence}", round, 10));
if (report.Passed)
{
try
{
string review = await Scenario(provider, "project_review_project", new Dictionary<string, string> {
["kind"] = draft.Kind, ["prd"] = Clip(draft.Prd + "\nRequested change: " + draft.Request, 10000), ["bundle"] = ProjectIntegrationCodec.Display(new(draft.Files)), ["diagnostics"] = Clip(diagnostics, 5000) }, ct);
var root = ProjectJson.Parse<System.Text.Json.JsonElement>(review);
bool clean = root.TryGetProperty("findings", out var findings) && findings.ValueKind == System.Text.Json.JsonValueKind.Array && findings.GetArrayLength() == 0;
draft = draft with { SourceReview = review }; store.Save(draft);
if (clean)
{
// Independent coverage agent may only append tests; every existing assertion is retained.
var coverageInputs = Inputs(draft, "All current tests passed. Append missing behavioral/boundary tests only; do not change existing cases. Current immutable tests: " + Clip(ProjectJson.Serialize(draft.Tests), 5000));
string additions = await Scenario(provider, "project_test_expand", coverageInputs, ct);
var extra = ProjectJson.Parse<ProjectTestPlan>(additions);
ProjectDraftStore.ValidateTests(extra.Cases, draft.Kind, allowEmpty: true);
if (extra.Cases.Length > 0)
{
additions = await Scenario(provider, "project_review_tests", new Dictionary<string,string> { ["kind"] = draft.Kind, ["prd"] = Clip(draft.Prd, 7000), ["request"] = Clip(draft.Request, 3000), ["source"] = ProjectIntegrationCodec.Display(new(draft.Files)), ["plan"] = additions }, ct);
extra = ProjectJson.Parse<ProjectTestPlan>(additions); ProjectDraftStore.ValidateTests(extra.Cases, draft.Kind, allowEmpty: true);
}
if (extra.Cases.Length == 0)
{
draft = draft with { State = "ready_to_publish" }; store.Save(draft);
return ProjectJson.Serialize(new { status = draft.State, draft.Id, rounds = round, checks = report.Cases.Length, message = "Declared runtime tests and independent source review passed on this revision; coverage is limited to these checks.", command = "/project-publish " + id, evidence });
}
if (extra.Cases.Any(t => draft.Tests.Any(old => old.Id == t.Id))) throw new InvalidDataException("Coverage must append NEW test IDs, not repeat existing cases.");
draft = store.AddTests(id, additions);
if (round == 10) break;
continue;
}
diagnostics += "\nIndependent source findings (not runtime evidence): " + Clip(review, 5000);
}
catch (Exception ex) when (!ct.IsCancellationRequested && ex is not Lilith.Core.Kanban.KanbanYieldException)
{
diagnostics += "\nIndependent review/test-plan error (do not change source merely to fix report formatting): " + ex.Message;
draft = store.Load(id) with { SourceReview = diagnostics }; store.Save(draft);
}
}
if (round == 10) break;
draft = draft with { State = "repairing" }; store.Save(draft);
try
{
string changes = await Scenario(provider, "project_repair", Inputs(draft, diagnostics), ct);
var proposed = ProjectJson.Parse<System.Text.Json.JsonElement>(changes);
if (proposed.TryGetProperty("test_issues", out var issues) && issues.ValueKind == System.Text.Json.JsonValueKind.Array && issues.GetArrayLength() > 0)
{
if (issues.GetArrayLength() > 10) throw new InvalidDataException("Report at most ten suspected test issues.");
foreach (var issue in issues.EnumerateArray())
{
string? testId = issue.GetProperty("id").GetString(), reason = issue.GetProperty("reason").GetString();
if (!draft.Tests.Any(t => t.Id == testId) || string.IsNullOrWhiteSpace(reason) || reason.Length > 1000) throw new InvalidDataException("Test issue requires an existing case ID and bounded reason.");
}
draft = store.Load(id) with { SourceReview = "AI-reported suspected test-plan issue (not independently verified). Frozen tests were not changed: " + issues.GetRawText() }; store.Save(draft);
break;
}
draft = store.Apply(id, changes);
}
catch (Exception ex) when (!ct.IsCancellationRequested && ex is not Lilith.Core.Kanban.KanbanYieldException)
{
// Do not silently skip failed repairs or restart the round budget.
draft = store.Load(id) with { SourceReview = "Repair/test-plan error: " + ex.Message };
store.Save(draft);
}
}
draft = store.Load(id) with { State = "failed" }; store.Save(draft);
return ProjectJson.Serialize(new { status = "failed", draft.Id, rounds = draft.Rounds, message = (draft.Rounds == 10 ? "Ten-round qualification limit reached." : "Qualification stopped with unresolved errors or a suspected frozen-test issue.") + " Nothing was published. Full errors and action traces are retained in the evidence folder.",
remaining = draft.Runtime is null ? null : new { draft.Runtime.SourceHash, draft.Runtime.PlanHash, errors = draft.Runtime.Errors.Take(10).Select(e => Clip(e, 500)), cases = draft.Runtime.Cases.Where(c => c.Status != "passed").Select(c => c with { Actual = Clip(c.Actual, 500), Expected = Clip(c.Expected, 200) }), draft.Runtime.Evidence },
unqualifiedTests = draft.Tests.Where(t => draft.Runtime is null || !draft.Runtime.Cases.Any(c => c.Id == t.Id && c.Status == "passed")).Select(t => t.Id), sourceReview = Clip(draft.SourceReview, 2000), evidence = folder });
}
catch (Lilith.Core.Kanban.KanbanYieldException) { store.Save(store.Load(id) with { State = "awaiting_test_approval" }); throw; }
catch (OperationCanceledException) { store.Save(store.Load(id) with { State = "cancelled", Runtime = null }); throw; }
catch (Exception ex) { store.Save(store.Load(id) with { State = "failed", SourceReview = ex.Message }); throw; }
}
internal static async Task<WorkspaceResult> PublishAsync(ProjectDraftStore store, string id, CancellationToken ct)
{
using var lease = new FileStream(Path.Combine(store.Folder(id), "execution.lock"), FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
var draft = store.Load(id);
if (draft.State != "ready_to_publish" || draft.Runtime is not { Passed: true } runtime || runtime.SourceHash != ProjectDraftStore.Hash(draft.Files) || runtime.PlanHash != ProjectDraftStore.PlanHash(draft.Tests)) throw new InvalidOperationException("Current source and complete test plan must pass before publication.");
string target = store.Workspace.Options.ResolvePath(draft.Name);
if (target.ComputeSignature() != (draft.OriginalSignature ?? "missing")) throw new IOException("Original project changed. Create a fresh edit session; nothing was overwritten.");
var files = draft.Files.ToList();
if (draft.Kind == "console") files.Add(new("Application.csproj", ProjectValidator.ProjectXml(draft.Architecture.Packages)));
files.Add(new("PRD.txt", draft.Prd));
var hashes = files.ToDictionary(f => f.Path, f => Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(f.Content))), StringComparer.Ordinal);
var manifest = new ProjectManifest(2, draft.Id, draft.Name, draft.Kind, draft.Prd, draft.Architecture.Contract, draft.Architecture.Packages, draft.Architecture.Files, [], new(true, draft.Kind, ["Current-revision build, declared runtime tests and independent source review"], [], "See retained runtime evidence."), hashes, DateTimeOffset.UtcNow, draft.SourceReview, runtime, draft.Tests);
files.Add(new("lilith-project.json", ProjectJson.Serialize(manifest)));
var bundle = new ProjectBundle(files.ToArray());
string payload = draft.OriginalSignature is null ? ProjectJson.Serialize(bundle) : ProjectJson.Serialize(new ProjectUpdateBundle(bundle, draft.OriginalManagedPaths));
var result = await store.Workspace.ExecuteAsync(new(draft.OriginalSignature is null ? "project_create" : "project_update", draft.Name, payload), ct);
store.Save(draft with { State = result.Status == "needs_approval" ? "awaiting_publication" : "published", ProposalId = result.ProposalId }); return result;
}
}