dirigence/dirigent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sync from monorepo @ 2452e92e

Browse Source
This commit is contained in:
Gabor Körber
2026-05-08 01:59:04 +02:00
commit b03dc15371
459 changed files with 129586 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/dirigent_archivist/tests/archive_filter_test.rs
+334
View File
@@ -0,0 +1,334 @@
//! Two-archive fanout tests exercising `ArchiveFilter` semantics.
//!
//! The primary backend is unfiltered; the secondary backend carries a
//! restricted filter. Writes should always reach the primary but only
//! fan out to the secondary when the session passes the filter.
#![cfg(feature = "test-utils")]
use std::collections::HashSet;
use std::sync::Arc;
use chrono::Utc;
use uuid::Uuid;
use dirigent_archivist::backend::mock::MockBackend;
use dirigent_archivist::backend::{ArchiveBackend, HealthStatus};
use dirigent_archivist::coordinator::Archivist;
use dirigent_archivist::registry::{
ArchiveFilter, ArchiveRegistration, FailureMode, WritePolicy,
};
use dirigent_archivist::types::{
ConnectorRecord, MessageRecord, RegisterSessionRequest,
};
fn reg(
name: &str,
backend: Arc<MockBackend>,
priority: u32,
filter: ArchiveFilter,
) -> Arc<ArchiveRegistration> {
Arc::new(
ArchiveRegistration::new(
name.into(),
"mock",
backend as Arc<dyn ArchiveBackend>,
/* write_active */ true,
FailureMode::Required,
priority,
/* enabled */ true,
WritePolicy::Inline,
/* writer */ None,
HealthStatus::Healthy,
)
.with_filter(filter),
)
}
/// Seed a connector into a MockBackend directly, bypassing the coordinator.
async fn seed_connector(backend: &MockBackend, connector_uid: Uuid, client_native_id: &str) {
use dirigent_archivist::backend::ConnectorRegistryBackend;
let rec = ConnectorRecord {
version: 1,
connector_uid,
r#type: "Mock".into(),
title: "Mock connector".into(),
client_native_id: client_native_id.into(),
alias_of: None,
created_at: Utc::now(),
metadata: serde_json::Value::Null,
fingerprint: None,
};
backend
.put_connector(rec)
.await
.expect("put_connector succeeds");
}
fn make_msg(session: Uuid, n: u32) -> MessageRecord {
MessageRecord {
version: 1,
message_id: Uuid::now_v7(),
session,
parent_id: None,
ts: Utc::now(),
role: "user".into(),
author: None,
content_md: format!("msg {}", n),
content_parts: None,
attachments: vec![],
metadata: serde_json::Value::Null,
}
}
#[tokio::test]
async fn secondary_archive_filters_by_exclude_connector() {
let primary_backend = Arc::new(MockBackend::new());
let secondary_backend = Arc::new(MockBackend::new());
let connector_a = Uuid::now_v7();
let connector_b = Uuid::now_v7();
// Connector A is excluded from the secondary.
let mut excluded = HashSet::new();
excluded.insert(connector_a);
let secondary_filter = ArchiveFilter {
exclude_connectors: excluded,
..Default::default()
};
// Seed connectors on primary (and on secondary so mapping writes that DO
// pass the filter don't fail for unrelated reasons).
seed_connector(&primary_backend, connector_a, "native/a").await;
seed_connector(&primary_backend, connector_b, "native/b").await;
let archivist = Archivist::from_registrations(vec![
reg("primary", primary_backend.clone(), 0, ArchiveFilter::default()),
reg("secondary", secondary_backend.clone(), 10, secondary_filter),
]);
// Register a session for each connector.
let resp_a = archivist
.register_session(
RegisterSessionRequest {
connector_uid: connector_a,
native_session_id: "sess-a".into(),
title: None,
custom_scroll_id: None,
metadata: serde_json::Value::Null,
completeness: Default::default(),
parent_scroll_id: None,
is_subagent: false,
continuation: None,
agent_id: None,
subagent_type: None,
spawning_tool_use_id: None,
},
None,
)
.await
.expect("register session a");
let scroll_a = resp_a.scroll_id;
let resp_b = archivist
.register_session(
RegisterSessionRequest {
connector_uid: connector_b,
native_session_id: "sess-b".into(),
title: None,
custom_scroll_id: None,
metadata: serde_json::Value::Null,
completeness: Default::default(),
parent_scroll_id: None,
is_subagent: false,
continuation: None,
agent_id: None,
subagent_type: None,
spawning_tool_use_id: None,
},
None,
)
.await
.expect("register session b");
let scroll_b = resp_b.scroll_id;
// Append 3 messages to each session.
archivist
.append_messages(
scroll_a,
vec![make_msg(scroll_a, 1), make_msg(scroll_a, 2), make_msg(scroll_a, 3)],
None,
)
.await
.expect("append to a");
archivist
.append_messages(
scroll_b,
vec![make_msg(scroll_b, 1), make_msg(scroll_b, 2), make_msg(scroll_b, 3)],
None,
)
.await
.expect("append to b");
// Primary sees every message.
assert_eq!(primary_backend.appended_count(scroll_a), 3);
assert_eq!(primary_backend.appended_count(scroll_b), 3);
// Secondary excludes connector_a: scroll_a is filtered out,
// scroll_b is replicated.
assert_eq!(
secondary_backend.appended_count(scroll_a),
0,
"secondary should NOT receive messages for the excluded connector"
);
assert_eq!(
secondary_backend.appended_count(scroll_b),
3,
"secondary should receive messages for the allowed connector"
);
// Session metadata fanout follows the same rule.
assert!(
primary_backend
.get_session(scroll_a)
.await
.unwrap()
.is_some(),
"primary has scroll_a"
);
assert!(
secondary_backend
.get_session(scroll_a)
.await
.unwrap()
.is_none(),
"secondary should NOT have scroll_a (excluded connector)"
);
assert!(
secondary_backend
.get_session(scroll_b)
.await
.unwrap()
.is_some(),
"secondary should have scroll_b (allowed connector)"
);
}
#[tokio::test]
async fn secondary_archive_filters_by_include_tag() {
let primary_backend = Arc::new(MockBackend::new());
let secondary_backend = Arc::new(MockBackend::new());
let connector = Uuid::now_v7();
seed_connector(&primary_backend, connector, "native/tagged").await;
let mut include = HashSet::new();
include.insert("prod".to_string());
let secondary_filter = ArchiveFilter {
include_tags: include,
..Default::default()
};
let archivist = Archivist::from_registrations(vec![
reg("primary", primary_backend.clone(), 0, ArchiveFilter::default()),
reg("secondary", secondary_backend.clone(), 10, secondary_filter),
]);
// Register two sessions on the same connector.
let prod_resp = archivist
.register_session(
RegisterSessionRequest {
connector_uid: connector,
native_session_id: "sess-prod".into(),
title: None,
custom_scroll_id: None,
metadata: serde_json::Value::Null,
completeness: Default::default(),
parent_scroll_id: None,
is_subagent: false,
continuation: None,
agent_id: None,
subagent_type: None,
spawning_tool_use_id: None,
},
None,
)
.await
.expect("register prod session");
let scroll_prod = prod_resp.scroll_id;
let dev_resp = archivist
.register_session(
RegisterSessionRequest {
connector_uid: connector,
native_session_id: "sess-dev".into(),
title: None,
custom_scroll_id: None,
metadata: serde_json::Value::Null,
completeness: Default::default(),
parent_scroll_id: None,
is_subagent: false,
continuation: None,
agent_id: None,
subagent_type: None,
spawning_tool_use_id: None,
},
None,
)
.await
.expect("register dev session");
let scroll_dev = dev_resp.scroll_id;
// Tag the prod session directly on the primary so the coordinator can
// see it on the next fanout metadata lookup. We mutate via the primary
// backend to avoid going through update_session_metadata (which doesn't
// expose a tag API).
{
use dirigent_archivist::backend::ArchiveBackend as _;
let mut md = primary_backend
.get_session(scroll_prod)
.await
.unwrap()
.expect("prod session on primary");
md.tags.push("prod".into());
primary_backend.put_session(md).await.unwrap();
}
// Append messages AFTER tagging — now the filter consults the tagged metadata.
archivist
.append_messages(
scroll_prod,
vec![
make_msg(scroll_prod, 1),
make_msg(scroll_prod, 2),
make_msg(scroll_prod, 3),
],
None,
)
.await
.expect("append prod");
archivist
.append_messages(
scroll_dev,
vec![make_msg(scroll_dev, 1), make_msg(scroll_dev, 2), make_msg(scroll_dev, 3)],
None,
)
.await
.expect("append dev");
// Primary keeps both.
assert_eq!(primary_backend.appended_count(scroll_prod), 3);
assert_eq!(primary_backend.appended_count(scroll_dev), 3);
// Secondary only keeps the tagged session.
assert_eq!(
secondary_backend.appended_count(scroll_prod),
3,
"secondary receives messages for the `prod`-tagged session"
);
assert_eq!(
secondary_backend.appended_count(scroll_dev),
0,
"secondary rejects the untagged session"
);
}
crates/dirigent_archivist/tests/fixtures/claude_minimal/projects/-home-user-myproj/abc12345-1234-1234-1234-abcdef123456.jsonl
Vendored
+2
View File
@@ -0,0 +1,2 @@
{"type":"user","uuid":"11111111-1111-7111-8111-111111111111","parentUuid":null,"timestamp":"2024-01-01T00:00:00Z","sessionId":"abc12345-1234-1234-1234-abcdef123456","cwd":"/home/user/myproj","version":"2.1.71","gitBranch":"main","isSidechain":false,"isMeta":false,"userType":"external","message":{"role":"user","content":"hello"}}
{"type":"assistant","uuid":"22222222-2222-7222-8222-222222222222","parentUuid":"11111111-1111-7111-8111-111111111111","timestamp":"2024-01-01T00:00:01Z","sessionId":"abc12345-1234-1234-1234-abcdef123456","cwd":"/home/user/myproj","version":"2.1.71","gitBranch":"main","isSidechain":false,"requestId":"req-001","message":{"model":"claude-3-5-sonnet","id":"msg-abc","type":"message","role":"assistant","content":[{"type":"text","text":"hi back"}],"stop_reason":"end_turn","usage":{"input_tokens":10,"output_tokens":5}}}
crates/dirigent_archivist/tests/import_claude_idempotency_test.rs
+153
View File
@@ -0,0 +1,153 @@
//! End-to-end test: import a Claude fixture twice, expect no duplication;
//! then append a new message and re-import, expect exactly 1 new message.
use camino::Utf8PathBuf;
use dirigent_archivist::{
backends::JsonlBackend,
import::{claude::import_claude_sessions, ImportProgressSink},
Archivist, SessionListQuery,
};
use std::sync::Arc;
use uuid::Uuid;
fn fixture_root() -> Utf8PathBuf {
Utf8PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap())
.join("tests/fixtures/claude_minimal")
}
/// Build a self-contained coordinator for a given archive root.
///
/// Uses `from_single_backend` so that parallel-test runs do not race on a
/// shared `.archives.json` in the tempdir's parent (which is what
/// `new_with_single_archive` would create).
async fn mk_archivist(root: std::path::PathBuf) -> dirigent_archivist::Result<Archivist> {
let backend = Arc::new(JsonlBackend::new(root).await?);
Archivist::from_single_backend("main".into(), backend).await
}
#[tokio::test]
async fn claude_import_twice_is_idempotent() -> dirigent_archivist::Result<()> {
let tmp = std::env::temp_dir().join(format!("claude_idem_{}", Uuid::now_v7()));
let archivist = mk_archivist(tmp.clone()).await?;
let fixture = fixture_root();
// First run — should import everything.
let stats1 = import_claude_sessions(&archivist, &fixture, None, &ImportProgressSink::noop(), &std::collections::HashMap::new()).await?;
assert!(
stats1.sessions_imported >= 1,
"expected at least one imported session, got stats {:?}",
stats1
);
assert!(
stats1.messages_written >= 2,
"expected >=2 messages written, got {:?}",
stats1
);
// Second run — should write nothing (fingerprint gate skips unchanged sessions).
let stats2 = import_claude_sessions(&archivist, &fixture, None, &ImportProgressSink::noop(), &std::collections::HashMap::new()).await?;
assert_eq!(
stats2.messages_written, 0,
"expected no re-write on second import, got {:?}",
stats2
);
assert_eq!(stats2.sessions_imported, 0);
assert!(
stats2.sessions_skipped >= 1,
"expected at least one skipped session, got {:?}",
stats2
);
// Verify on disk: no duplicate message_ids within any session.
let page = archivist
.list_sessions_paged(SessionListQuery::default().with_limit(200))
.await?;
for session in &page.items {
let messages = archivist.get_messages(session.scroll_id, None).await?;
let mut seen = std::collections::HashSet::new();
for m in &messages {
assert!(
seen.insert(m.message_id),
"duplicate message_id {} in session {}",
m.message_id,
session.scroll_id
);
}
}
let _ = tokio::fs::remove_dir_all(tmp).await;
Ok(())
}
#[tokio::test]
async fn claude_import_picks_up_additive_growth() -> dirigent_archivist::Result<()> {
// Copy the fixture to a mutable temp dir so we can append a message.
let tmp_src = std::env::temp_dir().join(format!("claude_grow_src_{}", Uuid::now_v7()));
let fixture = fixture_root();
copy_dir_recursive(&fixture.as_std_path().to_path_buf(), &tmp_src).await;
let tmp_arch = std::env::temp_dir().join(format!("claude_grow_arch_{}", Uuid::now_v7()));
let archivist = mk_archivist(tmp_arch.clone()).await?;
let src = Utf8PathBuf::from_path_buf(tmp_src.clone()).unwrap();
let _ = import_claude_sessions(&archivist, &src, None, &ImportProgressSink::noop(), &std::collections::HashMap::new()).await?;
// Append a new message to the existing JSONL.
let jsonl = find_jsonl(&tmp_src).expect("fixture jsonl not found");
let extra = r#"{"type":"user","uuid":"33333333-3333-7333-8333-333333333333","parentUuid":"22222222-2222-7222-8222-222222222222","timestamp":"2024-01-01T00:00:02Z","sessionId":"abc12345-1234-1234-1234-abcdef123456","cwd":"/home/user/myproj","version":"2.1.71","gitBranch":"main","isSidechain":false,"isMeta":false,"userType":"external","message":{"role":"user","content":"follow up"}}"#;
use tokio::io::AsyncWriteExt;
let mut f = tokio::fs::OpenOptions::new()
.append(true)
.open(&jsonl)
.await
.unwrap();
f.write_all(extra.as_bytes()).await.unwrap();
f.write_all(b"\n").await.unwrap();
drop(f);
let stats = import_claude_sessions(&archivist, &src, None, &ImportProgressSink::noop(), &std::collections::HashMap::new()).await?;
assert_eq!(
stats.messages_written, 1,
"expected 1 new message to be imported, got {:?}",
stats
);
assert_eq!(
stats.sessions_updated, 1,
"expected 1 session updated, got {:?}",
stats
);
let _ = tokio::fs::remove_dir_all(tmp_src).await;
let _ = tokio::fs::remove_dir_all(tmp_arch).await;
Ok(())
}
async fn copy_dir_recursive(src: &std::path::Path, dst: &std::path::Path) {
tokio::fs::create_dir_all(dst).await.unwrap();
let mut stack = vec![(src.to_path_buf(), dst.to_path_buf())];
while let Some((s, d)) = stack.pop() {
let mut entries = tokio::fs::read_dir(&s).await.unwrap();
while let Some(entry) = entries.next_entry().await.unwrap() {
let from = entry.path();
let to = d.join(entry.file_name());
if entry.file_type().await.unwrap().is_dir() {
tokio::fs::create_dir_all(&to).await.unwrap();
stack.push((from, to));
} else {
tokio::fs::copy(&from, &to).await.unwrap();
}
}
}
}
fn find_jsonl(dir: &std::path::Path) -> Option<std::path::PathBuf> {
for entry in walkdir::WalkDir::new(dir).into_iter().flatten() {
if entry.file_type().is_file()
&& entry.path().extension().and_then(|s| s.to_str()) == Some("jsonl")
{
return Some(entry.path().to_path_buf());
}
}
None
}
crates/dirigent_archivist/tests/import_progress_test.rs
+89
View File
@@ -0,0 +1,89 @@
//! Integration test: importer trait progress events fire in expected order.
//!
//! Drives a full `ChatGptImporter::import` against a fixture and asserts on
//! the `ImportProgressEvent` sequence observed on the paired receiver.
use std::sync::Arc;
use tempfile::TempDir;
use dirigent_archivist::{
backends::JsonlBackend,
coordinator::Archivist,
import::{
ImportConfig, ImportProgressEvent, ImportProgressSink, ImportTarget, ImporterRegistry,
},
};
#[tokio::test]
async fn progress_event_sequence_is_well_formed() {
// 1. Setup an in-memory archivist (JsonlBackend in tempdir).
let dir = TempDir::new().unwrap();
let backend = Arc::new(JsonlBackend::new(dir.path().to_path_buf()).await.unwrap());
let archivist = Archivist::from_single_backend("main".into(), backend)
.await
.unwrap();
let archivist = Arc::new(archivist);
// 2. Use the chatgpt fixture — a minimal conversations.json with a
// user + assistant message pair.
let fixture = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("../dirigent_chatgpt/tests/fixtures/minimal.json");
assert!(
fixture.exists(),
"chatgpt fixture missing at {}",
fixture.display()
);
let cfg = ImportConfig {
source: "chatgpt".into(),
params: {
let mut m = std::collections::BTreeMap::new();
m.insert("path".into(), serde_json::json!(fixture.display().to_string()));
m
},
};
// 3. Run the import with a channel sink.
let registry = ImporterRegistry::default();
let importer = registry.get("chatgpt").expect("chatgpt registered");
let (sink, mut rx) = ImportProgressSink::channel();
let archivist_for_job = archivist.clone();
let job = tokio::spawn(async move {
importer
.import(&cfg, &*archivist_for_job, ImportTarget::default(), sink)
.await
});
// 4. Collect all events until the sender side is dropped.
let mut events = Vec::new();
while let Some(evt) = rx.recv().await {
events.push(evt);
}
let stats = job.await.unwrap().expect("import");
// 5. Assertions on the event sequence.
// Must contain at least one SessionStarted before any SessionFinished.
let started_idx = events
.iter()
.position(|e| matches!(e, ImportProgressEvent::SessionStarted { .. }));
let finished_idx = events
.iter()
.position(|e| matches!(e, ImportProgressEvent::SessionFinished { .. }));
assert!(started_idx.is_some(), "expected a SessionStarted event");
assert!(finished_idx.is_some(), "expected a SessionFinished event");
assert!(
started_idx.unwrap() < finished_idx.unwrap(),
"SessionStarted must precede SessionFinished"
);
// Stats shows at least 2 messages written (chatgpt fixture has a user
// + assistant pair).
assert!(
stats.messages_written >= 2,
"expected messages to be written, got stats {:?}",
stats
);
assert_eq!(stats.sessions_imported, 1);
}
crates/dirigent_archivist/tests/integration_tests.rs
+2414
View File
File diff suppressed because it is too large Load Diff
crates/dirigent_archivist/tests/list_sessions_paged_test.rs
+364
View File
@@ -0,0 +1,364 @@
//! Tests for `Archivist::list_sessions_paged` — pagination, filters, cursor stability.
use chrono::{Duration, Utc};
use dirigent_archivist::{
backends::JsonlBackend, Archivist, RegisterConnectorRequest, RegisterSessionRequest,
Result, SessionListQuery,
};
use std::sync::Arc;
use uuid::Uuid;
/// Scaffold: create a coordinator backed by a single `JsonlBackend` in a
/// unique temp dir, returning the backend alongside it so tests can probe
/// disk paths via `backend.paths()`.
async fn mk_archivist() -> Result<(Archivist, Arc<JsonlBackend>, std::path::PathBuf)> {
let temp_dir = std::env::temp_dir().join(format!("paged_test_{}", Uuid::now_v7()));
let backend = Arc::new(JsonlBackend::new(temp_dir.clone()).await?);
let archivist =
Archivist::from_single_backend("main".into(), backend.clone()).await?;
Ok((archivist, backend, temp_dir))
}
/// Register a connector, return its UID.
async fn mk_connector(archivist: &Archivist, title: &str) -> Result<Uuid> {
let resp = archivist
.register_connector(
RegisterConnectorRequest {
r#type: "OpenCode".to_string(),
title: title.to_string(),
client_native_id: format!("{title}@local:{}", Uuid::now_v7()),
custom_uid: None,
metadata: serde_json::json!({}),
fingerprint: None,
},
None,
)
.await?;
Ok(resp.connector_uid)
}
/// Register a session and patch fields that `register_session` does not expose.
#[allow(clippy::too_many_arguments)]
async fn mk_session(
archivist: &Archivist,
backend: &JsonlBackend,
connector_uid: Uuid,
native_id: &str,
title: Option<&str>,
tags: Vec<String>,
model: Option<&str>,
project_id: Option<&str>,
no_update: bool,
) -> Result<Uuid> {
let mut metadata = serde_json::Map::new();
if let Some(m) = model {
metadata.insert("model".to_string(), serde_json::Value::String(m.to_string()));
}
if let Some(p) = project_id {
metadata.insert(
"project_id".to_string(),
serde_json::Value::String(p.to_string()),
);
}
let resp = archivist
.register_session(
RegisterSessionRequest {
connector_uid,
native_session_id: native_id.to_string(),
title: title.map(String::from),
custom_scroll_id: None,
metadata: serde_json::Value::Object(metadata),
completeness: Default::default(),
parent_scroll_id: None,
is_subagent: false,
continuation: None,
agent_id: None,
subagent_type: None,
spawning_tool_use_id: None,
},
None,
)
.await?;
let scroll_id = resp.scroll_id;
// Patch tags / no_update into session.json on disk.
if !tags.is_empty() || no_update {
let mut meta = archivist.get_session_metadata(scroll_id, None).await?;
meta.tags = tags;
meta.no_update = no_update;
let path = backend.paths().session_json(scroll_id);
dirigent_archivist::storage::json::write_json(&path, &meta)
.await
.map_err(dirigent_archivist::ArchivistError::Io)?;
}
Ok(scroll_id)
}
/// Overwrite a session's updated_at on disk for deterministic ordering.
async fn set_updated_at(
archivist: &Archivist,
backend: &JsonlBackend,
scroll_id: Uuid,
when: chrono::DateTime<chrono::Utc>,
) -> Result<()> {
let mut meta = archivist.get_session_metadata(scroll_id, None).await?;
meta.updated_at = when;
let path = backend.paths().session_json(scroll_id);
dirigent_archivist::storage::json::write_json(&path, &meta)
.await
.map_err(dirigent_archivist::ArchivistError::Io)?;
Ok(())
}
fn cleanup(path: std::path::PathBuf) {
let _ = std::fs::remove_dir_all(path);
}
#[tokio::test]
async fn list_sessions_paged_respects_limit() -> Result<()> {
let (archivist, backend, temp) = mk_archivist().await?;
let uid = mk_connector(&archivist, "connector-a").await?;
let base = Utc::now();
for i in 0..30 {
let scroll = mk_session(
&archivist,
&backend,
uid,
&format!("native-{i}"),
Some(&format!("title-{i}")),
Vec::new(),
None,
None,
false,
)
.await?;
set_updated_at(&archivist, &backend, scroll, base - Duration::seconds(i)).await?;
}
let page = archivist
.list_sessions_paged(SessionListQuery::default().with_connector(uid).with_limit(10))
.await?;
assert_eq!(page.items.len(), 10);
assert!(page.next_cursor.is_some());
cleanup(temp);
Ok(())
}
#[tokio::test]
async fn list_sessions_paged_end_of_list() -> Result<()> {
let (archivist, backend, temp) = mk_archivist().await?;
let uid = mk_connector(&archivist, "connector-b").await?;
let base = Utc::now();
for i in 0..5 {
let scroll = mk_session(
&archivist,
&backend,
uid,
&format!("native-{i}"),
Some(&format!("title-{i}")),
Vec::new(),
None,
None,
false,
)
.await?;
set_updated_at(&archivist, &backend, scroll, base - Duration::seconds(i)).await?;
}
let page = archivist
.list_sessions_paged(SessionListQuery::default().with_connector(uid).with_limit(100))
.await?;
assert_eq!(page.items.len(), 5);
assert!(page.next_cursor.is_none());
cleanup(temp);
Ok(())
}
#[tokio::test]
async fn list_sessions_paged_cursor_stability() -> Result<()> {
let (archivist, backend, temp) = mk_archivist().await?;
let uid = mk_connector(&archivist, "connector-c").await?;
let fixed = Utc::now();
for i in 0..6 {
let scroll = mk_session(
&archivist,
&backend,
uid,
&format!("native-{i}"),
Some(&format!("title-{i}")),
Vec::new(),
None,
None,
false,
)
.await?;
set_updated_at(&archivist, &backend, scroll, fixed).await?;
}
let p1 = archivist
.list_sessions_paged(SessionListQuery::default().with_connector(uid).with_limit(3))
.await?;
assert_eq!(p1.items.len(), 3);
assert!(p1.next_cursor.is_some());
let p2 = archivist
.list_sessions_paged(
SessionListQuery::default()
.with_connector(uid)
.with_limit(3)
.with_cursor(p1.next_cursor.clone()),
)
.await?;
assert_eq!(p2.items.len(), 3);
let ids1: std::collections::HashSet<_> = p1.items.iter().map(|s| s.scroll_id).collect();
let ids2: std::collections::HashSet<_> = p2.items.iter().map(|s| s.scroll_id).collect();
assert!(ids1.is_disjoint(&ids2), "page 1 and page 2 must not overlap");
assert!(p2.next_cursor.is_none());
cleanup(temp);
Ok(())
}
#[tokio::test]
async fn list_sessions_paged_title_filter() -> Result<()> {
let (archivist, backend, temp) = mk_archivist().await?;
let uid = mk_connector(&archivist, "connector-d").await?;
mk_session(&archivist, &backend, uid, "n1", Some("Alpha beta"), vec![], None, None, false).await?;
mk_session(&archivist, &backend, uid, "n2", Some("BETA only"), vec![], None, None, false).await?;
mk_session(&archivist, &backend, uid, "n3", Some("gamma"), vec![], None, None, false).await?;
mk_session(&archivist, &backend, uid, "n4", None, vec![], None, None, false).await?;
let page = archivist
.list_sessions_paged(
SessionListQuery::default()
.with_connector(uid)
.with_limit(50)
.with_title_query("beta"),
)
.await?;
let titles: Vec<_> = page.items.iter().filter_map(|s| s.title.clone()).collect();
assert_eq!(titles.len(), 2, "got {titles:?}");
assert!(titles.iter().any(|t| t == "Alpha beta"));
assert!(titles.iter().any(|t| t == "BETA only"));
cleanup(temp);
Ok(())
}
#[tokio::test]
async fn list_sessions_paged_tags_and() -> Result<()> {
let (archivist, backend, temp) = mk_archivist().await?;
let uid = mk_connector(&archivist, "connector-e").await?;
mk_session(
&archivist, &backend, uid, "n1", Some("s1"),
vec!["red".into(), "blue".into()], None, None, false,
).await?;
mk_session(
&archivist, &backend, uid, "n2", Some("s2"),
vec!["red".into()], None, None, false,
).await?;
mk_session(
&archivist, &backend, uid, "n3", Some("s3"),
vec!["blue".into()], None, None, false,
).await?;
let mut q = SessionListQuery::default().with_connector(uid).with_limit(50);
q.tags = vec!["red".into(), "blue".into()];
let page = archivist.list_sessions_paged(q).await?;
assert_eq!(page.items.len(), 1);
assert_eq!(page.items[0].title.as_deref(), Some("s1"));
cleanup(temp);
Ok(())
}
#[tokio::test]
async fn list_sessions_paged_model_filter() -> Result<()> {
let (archivist, backend, temp) = mk_archivist().await?;
let uid = mk_connector(&archivist, "connector-f").await?;
mk_session(&archivist, &backend, uid, "n1", Some("s1"), vec![], Some("claude-3-5-sonnet"), None, false).await?;
mk_session(&archivist, &backend, uid, "n2", Some("s2"), vec![], Some("gpt-4o"), None, false).await?;
mk_session(&archivist, &backend, uid, "n3", Some("s3"), vec![], None, None, false).await?;
let mut q = SessionListQuery::default().with_connector(uid).with_limit(50);
q.model_filter = Some("sonnet".into());
let page = archivist.list_sessions_paged(q).await?;
assert_eq!(page.items.len(), 1);
assert_eq!(page.items[0].title.as_deref(), Some("s1"));
cleanup(temp);
Ok(())
}
#[tokio::test]
async fn list_sessions_paged_include_hidden() -> Result<()> {
let (archivist, backend, temp) = mk_archivist().await?;
let uid = mk_connector(&archivist, "connector-g").await?;
mk_session(&archivist, &backend, uid, "n1", Some("visible"), vec![], None, None, false).await?;
mk_session(&archivist, &backend, uid, "n2", Some("hidden"), vec![], None, None, true).await?;
let visible_only = archivist
.list_sessions_paged(SessionListQuery::default().with_connector(uid).with_limit(50))
.await?;
assert_eq!(visible_only.items.len(), 1);
assert_eq!(visible_only.items[0].title.as_deref(), Some("visible"));
let all = archivist
.list_sessions_paged(
SessionListQuery::default()
.with_connector(uid)
.with_limit(50)
.with_include_hidden(true),
)
.await?;
assert_eq!(all.items.len(), 2);
cleanup(temp);
Ok(())
}
#[tokio::test]
async fn list_sessions_paged_project_scope() -> Result<()> {
let (archivist, backend, temp) = mk_archivist().await?;
let c1 = mk_connector(&archivist, "connector-h1").await?;
let c2 = mk_connector(&archivist, "connector-h2").await?;
mk_session(&archivist, &backend, c1, "n1", Some("proj-a-1"), vec![], None, Some("proj-a"), false).await?;
mk_session(&archivist, &backend, c1, "n2", Some("proj-b-1"), vec![], None, Some("proj-b"), false).await?;
mk_session(&archivist, &backend, c2, "n3", Some("proj-a-2"), vec![], None, Some("proj-a"), false).await?;
let page = archivist
.list_sessions_paged(
SessionListQuery::default()
.with_project("proj-a")
.with_limit(50),
)
.await?;
assert_eq!(page.items.len(), 2);
for s in &page.items {
assert_eq!(s.metadata.get("project_id").and_then(|v| v.as_str()), Some("proj-a"));
}
cleanup(temp);
Ok(())
}
crates/dirigent_archivist/tests/multi_backend_boot_test.rs
+130
View File
@@ -0,0 +1,130 @@
use dirigent_archivist::{
coordinator::Archivist,
error::ArchivistBootError,
registry::{ArchivesConfig, BackendRegistry},
};
fn parse(toml_src: &str) -> ArchivesConfig {
toml::from_str(toml_src).unwrap()
}
#[tokio::test]
async fn boot_with_one_jsonl_archive() {
let dir = tempfile::tempdir().unwrap();
let cfg = parse(&format!(
r#"
[[archives]]
name = "main"
type = "jsonl"
[archives.params]
path = "{}"
"#,
dir.path().to_string_lossy().replace('\\', "/")
));
let registry = BackendRegistry::with_jsonl();
let _archivist = Archivist::from_config(cfg, &registry, None).await.unwrap();
}
#[tokio::test]
async fn boot_empty_config_is_ephemeral() {
let cfg: ArchivesConfig = toml::from_str("").unwrap();
let registry = BackendRegistry::with_jsonl();
let archivist = Archivist::from_config(cfg, &registry, None).await.unwrap();
let archives = archivist.list_archives().await.unwrap();
assert!(archives.is_empty());
}
#[tokio::test]
async fn boot_unknown_type_errors() {
let cfg = parse(
r#"
[[archives]]
name = "x"
type = "nope"
[archives.params]
"#,
);
let registry = BackendRegistry::with_jsonl();
let result = Archivist::from_config(cfg, &registry, None).await;
match result {
Ok(_) => panic!("expected UnknownType error"),
Err(err) => assert!(
matches!(err, ArchivistBootError::UnknownType { .. }),
"expected UnknownType, got {err:?}"
),
}
}
#[tokio::test]
async fn boot_no_primary_errors() {
let cfg = parse(
r#"
[[archives]]
name = "mirror"
type = "jsonl"
failure_mode = "best_effort"
[archives.params]
path = "/tmp/whatever"
"#,
);
let registry = BackendRegistry::with_jsonl();
let result = Archivist::from_config(cfg, &registry, None).await;
match result {
Ok(_) => panic!("expected Validation error"),
Err(err) => assert!(
matches!(err, ArchivistBootError::Validation(_)),
"expected Validation, got {err:?}"
),
}
}
#[tokio::test]
async fn boot_duplicate_name_errors() {
let dir = tempfile::tempdir().unwrap();
let cfg = parse(&format!(
r#"
[[archives]]
name = "main"
type = "jsonl"
[archives.params]
path = "{p}"
[[archives]]
name = "main"
type = "jsonl"
[archives.params]
path = "{p}"
"#,
p = dir.path().to_string_lossy().replace('\\', "/"),
));
let registry = BackendRegistry::with_jsonl();
let result = Archivist::from_config(cfg, &registry, None).await;
match result {
Ok(_) => panic!("expected Validation error"),
Err(err) => assert!(
matches!(err, ArchivistBootError::Validation(_)),
"expected Validation, got {err:?}"
),
}
}
#[test]
fn example_toml_parses() {
// Load the full dirigent.toml.example and parse just the [[archives]]
// section as `ArchivesConfig`. Confirms the example's archive syntax is
// valid Phase 3 schema.
let src = std::fs::read_to_string(
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
.join("../../dirigent.toml.example"),
)
.expect("dirigent.toml.example present at workspace root");
// Parse the whole file as a TOML value, then try to deserialize the
// full document into `ArchivesConfig`. Any `archives` subtable gets picked up;
// other top-level fields (connectors, matrix, ...) are ignored because
// `ArchivesConfig` only has `entries: Vec<ArchiveConfig>` via
// `#[serde(rename = "archives")]`.
let cfg: ArchivesConfig =
toml::from_str(&src).expect("ArchivesConfig from full example");
cfg.validate().expect("example config validates");
assert!(!cfg.entries.is_empty(), "example must declare at least one archive");
}
crates/dirigent_archivist/tests/multi_backend_capability_test.rs
+76
View File
@@ -0,0 +1,76 @@
#![cfg(feature = "test-utils")]
use std::sync::Arc;
use dirigent_archivist::backend::mock::MockBackend;
use dirigent_archivist::backend::{ArchiveBackend, ArchiveCapability, CapabilitySet, HealthStatus};
use dirigent_archivist::coordinator::Archivist;
use dirigent_archivist::registry::{ArchiveRegistration, FailureMode, WritePolicy};
use dirigent_archivist::types::{MetaEventRecord, MetaEventType};
use uuid::Uuid;
fn reg(name: &str, backend: Arc<MockBackend>, priority: u32) -> Arc<ArchiveRegistration> {
Arc::new(ArchiveRegistration::new(
name.into(),
"mock",
backend as Arc<dyn ArchiveBackend>,
true,
FailureMode::Required,
priority,
true,
WritePolicy::Inline,
None,
HealthStatus::Healthy,
))
}
fn stub_meta_event(scroll_id: Uuid) -> MetaEventRecord {
MetaEventRecord {
version: 1,
event_id: Uuid::now_v7(),
session: scroll_id,
ts: chrono::Utc::now(),
event_type: MetaEventType::ClientConnected,
description: "test event".into(),
linked_session_id: None,
linked_connector_id: None,
linked_connector_title: None,
metadata: serde_json::Value::Null,
}
}
#[tokio::test]
async fn capability_filter_skips_backend_without_meta_events() {
let mut caps_with_meta = CapabilitySet::new();
caps_with_meta.insert(ArchiveCapability::MetaEvents);
caps_with_meta.insert(ArchiveCapability::SessionMapping);
caps_with_meta.insert(ArchiveCapability::ConnectorRegistry);
let with_meta = Arc::new(MockBackend::with_capabilities(caps_with_meta));
let mut caps_without_meta = CapabilitySet::new();
caps_without_meta.insert(ArchiveCapability::SessionMapping);
caps_without_meta.insert(ArchiveCapability::ConnectorRegistry);
let without_meta = Arc::new(MockBackend::with_capabilities(caps_without_meta));
let archivist = Archivist::from_registrations(vec![
reg("primary", with_meta.clone(), 0),
reg("secondary", without_meta.clone(), 10),
]);
let scroll = Uuid::new_v4();
archivist
.append_meta_events(scroll, vec![stub_meta_event(scroll)], None)
.await
.unwrap();
// Primary received the meta event.
assert!(
with_meta.has_meta_events(scroll),
"primary should receive meta event"
);
// Secondary was capability-skipped.
assert!(
!without_meta.has_meta_events(scroll),
"secondary should be skipped"
);
}
crates/dirigent_archivist/tests/multi_backend_cross_test.rs
+121
View File
@@ -0,0 +1,121 @@
#![cfg(feature = "test-utils")]
use std::sync::Arc;
use dirigent_archivist::backend::mock::MockBackend;
use dirigent_archivist::backend::ArchiveBackend;
use dirigent_archivist::backend::HealthStatus;
use dirigent_archivist::coordinator::Archivist;
use dirigent_archivist::error::ArchivistError;
use dirigent_archivist::registry::{ArchiveRegistration, FailureMode, WritePolicy};
use dirigent_archivist::types::SessionMetadata;
use uuid::Uuid;
async fn dual_backend_coordinator() -> (Archivist, Arc<MockBackend>, Arc<MockBackend>) {
let a = Arc::new(MockBackend::new());
let b = Arc::new(MockBackend::new());
let regs = vec![
Arc::new(ArchiveRegistration::new(
"a".into(),
"mock",
a.clone() as Arc<dyn ArchiveBackend>,
true,
FailureMode::Required,
0,
true,
WritePolicy::Inline,
None,
HealthStatus::Healthy,
)),
Arc::new(ArchiveRegistration::new(
"b".into(),
"mock",
b.clone() as Arc<dyn ArchiveBackend>,
true,
FailureMode::Required,
10,
true,
WritePolicy::Inline,
None,
HealthStatus::Healthy,
)),
];
(Archivist::from_registrations(regs), a, b)
}
#[tokio::test]
async fn copy_session_carries_metadata_and_messages() {
let (archivist, a, b) = dual_backend_coordinator().await;
let scroll = Uuid::new_v4();
// Seed `a` only.
a.put_session(SessionMetadata::stub(scroll)).await.unwrap();
a.append_messages(scroll, vec![]).await.unwrap();
archivist.copy_session(scroll, "a", "b").await.unwrap();
assert!(b.get_session(scroll).await.unwrap().is_some());
assert!(a.get_session(scroll).await.unwrap().is_some());
}
#[tokio::test]
async fn move_session_removes_from_source() {
let (archivist, a, b) = dual_backend_coordinator().await;
let scroll = Uuid::new_v4();
a.put_session(SessionMetadata::stub(scroll)).await.unwrap();
archivist.move_session(scroll, "a", "b").await.unwrap();
assert!(a.get_session(scroll).await.unwrap().is_none());
assert!(b.get_session(scroll).await.unwrap().is_some());
assert_eq!(
archivist.read_cache_size().await,
1,
"cache should now reflect the move"
);
}
#[tokio::test]
async fn move_session_partial_failure_returns_partial_move_error() {
let (archivist, a, b) = dual_backend_coordinator().await;
let scroll = Uuid::new_v4();
a.put_session(SessionMetadata::stub(scroll)).await.unwrap();
// The source-side delete happens AFTER the copy. Inject ONE write failure
// AFTER the copy has already consumed the write capacity. `MockBackend`'s
// inject_write_failures decrements on every mutating call — so we:
// 1. perform the copy through the archivist (uses put_session+append on `b`,
// but NO writes on `a`, since reads happen on the source side).
// 2. THEN inject a write failure on `a` to make the delete fail.
//
// Actually `copy_session` reads from `a` then writes to `b`, no writes on `a`.
// So we can safely inject BEFORE calling move_session: the only write on `a`
// during move_session is the delete, which will hit the injected failure.
a.inject_write_failures(1);
let err = archivist.move_session(scroll, "a", "b").await.unwrap_err();
assert!(matches!(err, ArchivistError::PartialMove { .. }));
// Both backends now have the session.
assert!(a.get_session(scroll).await.unwrap().is_some());
assert!(b.get_session(scroll).await.unwrap().is_some());
}
#[tokio::test]
async fn delete_session_fans_out_and_invalidates_cache() {
let (archivist, a, b) = dual_backend_coordinator().await;
let scroll = Uuid::new_v4();
a.put_session(SessionMetadata::stub(scroll)).await.unwrap();
b.put_session(SessionMetadata::stub(scroll)).await.unwrap();
// Prime the cache with a read.
let _ = archivist.get_session_metadata(scroll, None).await.unwrap();
assert_eq!(archivist.read_cache_size().await, 1);
archivist.delete_session(scroll, None).await.unwrap();
assert!(a.get_session(scroll).await.unwrap().is_none());
assert!(b.get_session(scroll).await.unwrap().is_none());
assert_eq!(archivist.read_cache_size().await, 0);
}
crates/dirigent_archivist/tests/multi_backend_fanout_test.rs
+124
View File
@@ -0,0 +1,124 @@
#![cfg(feature = "test-utils")]
use std::sync::Arc;
use dirigent_archivist::backend::mock::MockBackend;
use dirigent_archivist::backend::{ArchiveBackend, HealthStatus};
use dirigent_archivist::coordinator::Archivist;
use dirigent_archivist::registry::{ArchiveRegistration, FailureMode, WritePolicy};
use uuid::Uuid;
fn reg(
name: &str,
backend: Arc<MockBackend>,
priority: u32,
failure: FailureMode,
) -> Arc<ArchiveRegistration> {
Arc::new(ArchiveRegistration::new(
name.into(),
"mock",
backend as Arc<dyn ArchiveBackend>,
true,
failure,
priority,
true,
WritePolicy::Inline,
None,
HealthStatus::Healthy,
))
}
fn sample_message(session: Uuid) -> dirigent_archivist::types::MessageRecord {
dirigent_archivist::types::MessageRecord {
version: 1,
message_id: Uuid::now_v7(),
session,
parent_id: None,
ts: chrono::Utc::now(),
role: "user".into(),
author: None,
content_md: "hi".into(),
content_parts: None,
attachments: vec![],
metadata: serde_json::Value::Null,
}
}
#[tokio::test]
async fn write_fans_out_to_both_backends() {
let a = Arc::new(MockBackend::new());
let b = Arc::new(MockBackend::new());
let archivist = Archivist::from_registrations(vec![
reg("a", a.clone(), 0, FailureMode::Required),
reg("b", b.clone(), 10, FailureMode::BestEffort),
]);
// Using a non-empty message vec for a robust positive-count check:
let scroll = Uuid::new_v4();
let m = sample_message(scroll);
archivist
.append_messages(scroll, vec![m], None)
.await
.unwrap();
assert_eq!(a.appended_count(scroll), 1);
assert_eq!(b.appended_count(scroll), 1);
}
#[tokio::test]
async fn best_effort_failure_does_not_propagate() {
let a = Arc::new(MockBackend::new());
let b = Arc::new(MockBackend::new());
b.inject_write_failures(1);
let archivist = Archivist::from_registrations(vec![
reg("a", a.clone(), 0, FailureMode::Required),
reg("b", b.clone(), 10, FailureMode::BestEffort),
]);
archivist
.append_messages(Uuid::new_v4(), vec![], None)
.await
.unwrap(); // Ok despite secondary failure
let snapshot = archivist.list_archives_with_health().await;
let b_status = snapshot.iter().find(|s| s.name == "b").unwrap();
assert!(matches!(b_status.health, HealthStatus::Degraded { .. }));
}
#[tokio::test]
async fn required_secondary_failure_propagates() {
let a = Arc::new(MockBackend::new());
let b = Arc::new(MockBackend::new());
b.inject_write_failures(1);
let archivist = Archivist::from_registrations(vec![
reg("a", a.clone(), 0, FailureMode::Required),
reg("b", b.clone(), 10, FailureMode::Required),
]);
let err = archivist
.append_messages(Uuid::new_v4(), vec![], None)
.await;
assert!(err.is_err(), "expected error when required secondary fails");
}
#[tokio::test]
async fn explicit_archive_overrides_default_primary() {
let a = Arc::new(MockBackend::new());
let b = Arc::new(MockBackend::new());
let archivist = Archivist::from_registrations(vec![
reg("a", a.clone(), 0, FailureMode::Required),
reg("b", b.clone(), 10, FailureMode::Required),
]);
let scroll = Uuid::new_v4();
let m = sample_message(scroll);
archivist
.append_messages(scroll, vec![m], Some("b".into()))
.await
.unwrap();
// Both receive the write: b is explicit primary, a is secondary via fanout.
assert_eq!(a.appended_count(scroll), 1);
assert_eq!(b.appended_count(scroll), 1);
}
crates/dirigent_archivist/tests/multi_backend_health_test.rs
+129
View File
@@ -0,0 +1,129 @@
#![cfg(feature = "test-utils")]
use std::sync::Arc;
use dirigent_archivist::backend::mock::MockBackend;
use dirigent_archivist::backend::{ArchiveBackend, HealthStatus};
use dirigent_archivist::coordinator::Archivist;
use dirigent_archivist::registry::{ArchiveRegistration, FailureMode, WritePolicy};
use dirigent_archivist::types::SessionMetadata;
use uuid::Uuid;
fn reg(
name: &str,
backend: Arc<MockBackend>,
priority: u32,
failure: FailureMode,
) -> Arc<ArchiveRegistration> {
Arc::new(ArchiveRegistration::new(
name.into(),
"mock",
backend as Arc<dyn ArchiveBackend>,
true,
failure,
priority,
true,
WritePolicy::Inline,
None,
HealthStatus::Healthy,
))
}
#[tokio::test]
async fn five_consecutive_failures_drifts_to_unavailable() {
let primary = Arc::new(MockBackend::new());
let secondary = Arc::new(MockBackend::new());
secondary.inject_write_failures(10);
let archivist = Archivist::from_registrations(vec![
reg("primary", primary.clone(), 0, FailureMode::Required),
reg("secondary", secondary.clone(), 10, FailureMode::BestEffort),
]);
for _ in 0..5 {
archivist
.append_messages(Uuid::new_v4(), vec![], None)
.await
.ok();
}
let snapshot = archivist.list_archives_with_health().await;
let secondary_status = snapshot.iter().find(|s| s.name == "secondary").unwrap();
assert!(
matches!(secondary_status.health, HealthStatus::Unavailable { .. }),
"secondary should be Unavailable after 5 failures; got {:?}",
secondary_status.health
);
}
#[tokio::test]
async fn success_after_failure_recovers_to_healthy() {
let backend = Arc::new(MockBackend::new());
backend.inject_write_failures(1);
let archivist = Archivist::from_registrations(vec![reg(
"only",
backend.clone(),
0,
FailureMode::Required,
)]);
// First call fails.
let _ = archivist
.append_messages(Uuid::new_v4(), vec![], None)
.await;
let snapshot = archivist.list_archives_with_health().await;
assert!(
matches!(snapshot[0].health, HealthStatus::Degraded { .. }),
"expected Degraded after first failure; got {:?}",
snapshot[0].health
);
// Second call succeeds — health returns to Healthy.
archivist
.append_messages(Uuid::new_v4(), vec![], None)
.await
.unwrap();
let snapshot = archivist.list_archives_with_health().await;
assert!(
matches!(snapshot[0].health, HealthStatus::Healthy),
"expected Healthy after recovery; got {:?}",
snapshot[0].health
);
}
#[tokio::test]
async fn unavailable_backend_skipped_during_read_walk() {
let primary = Arc::new(MockBackend::new());
let secondary = Arc::new(MockBackend::new());
let scroll = Uuid::new_v4();
secondary
.put_session(SessionMetadata::stub(scroll))
.await
.unwrap();
secondary.break_permanently("kaput");
let primary_reg = reg("primary", primary.clone(), 0, FailureMode::Required);
let secondary_reg = reg("secondary", secondary.clone(), 10, FailureMode::Required);
// Force secondary's cached health to Unavailable BEFORE the walk,
// so the routing layer skips it entirely rather than attempting + failing.
*secondary_reg.last_health.write().await = HealthStatus::Unavailable {
reason: "test".into(),
};
let archivist = Archivist::from_registrations(vec![primary_reg, secondary_reg]);
// Primary doesn't have the session; secondary has it but is marked Unavailable.
// Read walk should skip secondary → Ok(None)-style ergonomics, bubbling up
// as `SessionUnknown` per `get_session_metadata`'s contract.
let result = archivist.get_session_metadata(scroll, None).await;
assert!(
result.is_err(),
"expected SessionUnknown error when Unavailable backend is skipped"
);
assert!(matches!(
result.unwrap_err(),
dirigent_archivist::error::ArchivistError::SessionUnknown(_)
));
}
crates/dirigent_archivist/tests/multi_backend_routing_test.rs
+102
View File
@@ -0,0 +1,102 @@
#![cfg(feature = "test-utils")]
use std::sync::Arc;
use dirigent_archivist::backend::mock::MockBackend;
use dirigent_archivist::backend::{ArchiveBackend, HealthStatus};
use dirigent_archivist::coordinator::Archivist;
use dirigent_archivist::registry::{ArchiveRegistration, FailureMode, WritePolicy};
use dirigent_archivist::types::SessionMetadata;
use uuid::Uuid;
fn reg(name: &str, backend: Arc<MockBackend>, priority: u32) -> Arc<ArchiveRegistration> {
Arc::new(ArchiveRegistration::new(
name.into(),
"mock",
backend as Arc<dyn ArchiveBackend>,
true,
FailureMode::Required,
priority,
true,
WritePolicy::Inline,
None,
HealthStatus::Healthy,
))
}
#[tokio::test]
async fn high_priority_backend_serves_first() {
let high = Arc::new(MockBackend::new());
let low = Arc::new(MockBackend::new());
let scroll = Uuid::new_v4();
high.put_session(SessionMetadata::stub(scroll)).await.unwrap();
let archivist = Archivist::from_registrations(vec![
reg("high", high.clone(), 0),
reg("low", low.clone(), 10),
]);
let meta = archivist.get_session_metadata(scroll, None).await;
assert!(meta.is_ok(), "expected Ok; got {:?}", meta);
assert_eq!(archivist.read_cache_size().await, 1);
}
#[tokio::test]
async fn falls_through_to_lower_priority_when_high_misses() {
let high = Arc::new(MockBackend::new());
let low = Arc::new(MockBackend::new());
let scroll = Uuid::new_v4();
low.put_session(SessionMetadata::stub(scroll)).await.unwrap();
let archivist = Archivist::from_registrations(vec![
reg("high", high.clone(), 0),
reg("low", low.clone(), 10),
]);
let meta = archivist.get_session_metadata(scroll, None).await;
assert!(meta.is_ok(), "expected Ok; got {:?}", meta);
assert_eq!(archivist.read_cache_size().await, 1);
}
#[tokio::test]
async fn cache_makes_second_read_skip_priority_walk() {
let high = Arc::new(MockBackend::new());
let low = Arc::new(MockBackend::new());
let scroll = Uuid::new_v4();
low.put_session(SessionMetadata::stub(scroll)).await.unwrap();
let archivist = Archivist::from_registrations(vec![
reg("high", high.clone(), 0),
reg("low", low.clone(), 10),
]);
// Prime the cache.
let _ = archivist.get_session_metadata(scroll, None).await.unwrap();
// Inject a read failure on `high` to detect whether the second read walks it.
// If the cache works, `high` must NOT be touched.
high.inject_read_failures(1);
let _ = archivist.get_session_metadata(scroll, None).await.unwrap();
let snapshot = archivist.list_archives_with_health().await;
let high_status = snapshot.iter().find(|s| s.name == "high").unwrap();
assert!(
matches!(high_status.health, HealthStatus::Healthy),
"cache should have skipped `high`; got {:?}",
high_status.health
);
}
#[tokio::test]
async fn delete_invalidates_cache() {
let high = Arc::new(MockBackend::new());
let scroll = Uuid::new_v4();
high.put_session(SessionMetadata::stub(scroll)).await.unwrap();
let archivist = Archivist::from_registrations(vec![reg("high", high.clone(), 0)]);
let _ = archivist.get_session_metadata(scroll, None).await.unwrap();
assert_eq!(archivist.read_cache_size().await, 1);
archivist.delete_session(scroll, None).await.unwrap();
assert_eq!(archivist.read_cache_size().await, 0);
}
crates/dirigent_archivist/tests/multi_backend_writer_test.rs
+252
View File
@@ -0,0 +1,252 @@
#![cfg(feature = "test-utils")]
//! Integration tests for Task 17's per-backend queued writer task.
//!
//! These exercise the full enqueue → batch → coalesce → dispatch pipeline
//! end-to-end by constructing real writer tasks against `MockBackend`
//! instances and driving them through the `Archivist` coordinator.
//!
//! The tests are timing-sensitive: the batch window is 25ms and the
//! backpressure test artificially slows the backend. Assertions use
//! tolerant margins so they survive CI jitter.
use std::sync::Arc;
use std::time::Duration;
use dirigent_archivist::backend::mock::MockBackend;
use dirigent_archivist::backend::{ArchiveBackend, HealthStatus};
use dirigent_archivist::coordinator::Archivist;
use dirigent_archivist::registry::writer::spawn_writer;
use dirigent_archivist::registry::{
ArchiveRegistration, FailureMode, OverflowPolicy, WritePolicy,
};
use uuid::Uuid;
fn sample_message(scroll: Uuid) -> dirigent_archivist::types::MessageRecord {
dirigent_archivist::types::MessageRecord {
version: 1,
message_id: Uuid::now_v7(),
session: scroll,
parent_id: None,
ts: chrono::Utc::now(),
role: "user".into(),
author: None,
content_md: "hi".into(),
content_parts: None,
attachments: vec![],
metadata: serde_json::Value::Null,
}
}
fn queued_reg(
name: &str,
backend: Arc<MockBackend>,
priority: u32,
overflow: OverflowPolicy,
) -> Arc<ArchiveRegistration> {
let initial_health = HealthStatus::Healthy;
let policy = WritePolicy::Queued {
batch_window_ms: 25,
capacity: 8,
overflow,
};
let health = Arc::new(tokio::sync::RwLock::new(initial_health));
let last_error = Arc::new(tokio::sync::RwLock::new(None));
let consecutive = Arc::new(tokio::sync::RwLock::new(0u32));
let writer = Some(spawn_writer(
backend.clone() as Arc<dyn ArchiveBackend>,
name.into(),
8,
Duration::from_millis(25),
overflow,
health.clone(),
last_error.clone(),
consecutive.clone(),
));
Arc::new(ArchiveRegistration::new_with_shared_state(
name.into(),
"mock",
backend as Arc<dyn ArchiveBackend>,
true,
FailureMode::Required,
priority,
true,
policy,
writer,
health,
last_error,
consecutive,
))
}
#[tokio::test]
async fn queued_write_returns_immediately_then_eventually_lands() {
let mock = Arc::new(MockBackend::new());
let archivist = Archivist::from_registrations(vec![queued_reg(
"queued",
mock.clone(),
0,
OverflowPolicy::Block,
)]);
let scroll = Uuid::new_v4();
archivist
.append_messages(scroll, vec![sample_message(scroll)], None)
.await
.unwrap();
// Wait up to 500ms for the writer to drain.
let mut landed = false;
for _ in 0..50 {
if mock.appended_count(scroll) > 0 {
landed = true;
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
assert!(landed, "writer task did not drain within 500ms");
assert_eq!(mock.appended_count(scroll), 1);
archivist.shutdown().await.unwrap();
}
#[tokio::test]
async fn coalescing_merges_consecutive_appends_for_same_scroll() {
let mock = Arc::new(MockBackend::new());
let archivist = Archivist::from_registrations(vec![queued_reg(
"queued",
mock.clone(),
0,
OverflowPolicy::Block,
)]);
let scroll = Uuid::new_v4();
for _ in 0..5 {
archivist
.append_messages(scroll, vec![sample_message(scroll)], None)
.await
.unwrap();
}
// Give the writer time to drain + coalesce, then shut down to guarantee
// any still-queued ops are flushed before we assert.
tokio::time::sleep(Duration::from_millis(200)).await;
archivist.shutdown().await.unwrap();
// Five enqueued ops may have been coalesced into fewer backend calls.
// The only strict invariant we can reliably assert is: the total number
// of backend `append_messages` INVOCATIONS is <= 5.
assert!(
mock.append_call_count(scroll) <= 5,
"expected <= 5 backend calls, got {}",
mock.append_call_count(scroll)
);
assert_eq!(
mock.appended_count(scroll),
5,
"all 5 messages should land"
);
}
#[tokio::test]
async fn overflow_block_applies_backpressure() {
// For backpressure to visibly stall the sender, we need four things:
// 1. A tight queue (capacity=2) so the channel actually fills up.
// 2. A slow backend (per-op 50ms) so the writer stalls in dispatch
// long enough for the channel to fill.
// 3. batch_window=0 so the writer spends (almost) all its time in
// the 50ms per-op sleep instead of draining fast inside the
// batch-collection phase.
// 4. Distinct scroll IDs so the writer's same-scroll coalescing
// doesn't merge everything into one dispatch call (which would
// collapse the entire batch into a single 50ms sleep).
// With those, the writer dispatches N serial 50ms calls; while it's
// sleeping the sender can't fit its next op into the full channel
// and must wait for a drain.
let mock = Arc::new(MockBackend::new());
mock.set_per_op_delay(Duration::from_millis(50));
let capacity = 2usize;
let overflow = OverflowPolicy::Block;
// batch_window=0 means the writer dispatches each op immediately and
// spends (almost) all its time in the 50ms per-op sleep — so the
// channel stays full and the sender has to wait on every drain.
let policy = WritePolicy::Queued {
batch_window_ms: 0,
capacity,
overflow,
};
let health = Arc::new(tokio::sync::RwLock::new(HealthStatus::Healthy));
let last_error = Arc::new(tokio::sync::RwLock::new(None));
let consecutive = Arc::new(tokio::sync::RwLock::new(0u32));
let writer = Some(spawn_writer(
mock.clone() as Arc<dyn ArchiveBackend>,
"queued".into(),
capacity,
Duration::from_millis(0),
overflow,
health.clone(),
last_error.clone(),
consecutive.clone(),
));
let reg = Arc::new(ArchiveRegistration::new_with_shared_state(
"queued".into(),
"mock",
mock.clone() as Arc<dyn ArchiveBackend>,
true,
FailureMode::Required,
0,
true,
policy,
writer,
health,
last_error,
consecutive,
));
let archivist = Archivist::from_registrations(vec![reg]);
// Prime the writer with one op and wait just long enough for it to
// enter its first 50ms dispatch sleep. After that the writer is NOT
// recv'ing, so the tight capacity=2 channel fills and further sends
// must wait for a drain.
let scroll0 = Uuid::new_v4();
archivist
.append_messages(scroll0, vec![sample_message(scroll0)], None)
.await
.unwrap();
tokio::time::sleep(Duration::from_millis(10)).await;
// Now measure the cost of many more sends with distinct scroll IDs
// so the writer can't coalesce them. Each dispatch call is 50ms, the
// queue holds only 2, so the sender must wait repeatedly for the
// writer to drain cycles.
let start = std::time::Instant::now();
for _ in 0..24 {
let scroll = Uuid::new_v4();
archivist
.append_messages(scroll, vec![sample_message(scroll)], None)
.await
.unwrap();
}
let elapsed = start.elapsed();
// With 24 distinct-scroll sends, a capacity=2 queue, batch_window=0,
// and a 50ms per-op delay, the sender cannot finish instantly — the
// writer needs many drain cycles and the sender waits on each. A
// 100ms floor keeps the test meaningful (a non-blocking run measures
// in microseconds) while being lenient on CI jitter.
assert!(
elapsed >= Duration::from_millis(100),
"block policy did not apply backpressure (elapsed: {:?})",
elapsed
);
archivist.shutdown().await.unwrap();
}
crates/dirigent_archivist/tests/pagination_test.rs
+142
View File
@@ -0,0 +1,142 @@
//! Pagination tests for dirigent_archivist
//!
//! These tests verify the count_messages and get_messages_range functionality.
#[cfg(test)]
mod pagination_tests {
use chrono::Utc;
use dirigent_archivist::{
backends::JsonlBackend, Archivist, MessageRecord, RegisterConnectorRequest,
RegisterSessionRequest, Result,
};
use std::sync::Arc;
use uuid::Uuid;
/// Build a self-contained coordinator rooted at `archive_root`, backed by
/// a single `JsonlBackend`. Avoids the shared `.archives.json` race that
/// `new_with_single_archive` creates in the tempdir's parent.
async fn mk_archivist(archive_root: std::path::PathBuf) -> Result<Archivist> {
let backend = Arc::new(JsonlBackend::new(archive_root).await?);
Archivist::from_single_backend("main".into(), backend).await
}
#[tokio::test]
async fn test_pagination_count_and_range() -> Result<()> {
let temp_dir = std::env::temp_dir().join(format!("archivist_test_{}", Uuid::now_v7()));
let archivist = mk_archivist(temp_dir.clone()).await?;
// Register connector
let connector_req = RegisterConnectorRequest {
r#type: "OpenCode".to_string(),
title: "Test Connector".to_string(),
client_native_id: "opencode@localhost:12225".to_string(),
custom_uid: None,
metadata: serde_json::json!({}),
fingerprint: None,
};
let connector_response = archivist.register_connector(connector_req, None).await?;
// Register session
let session_req = RegisterSessionRequest {
connector_uid: connector_response.connector_uid,
native_session_id: "native-123".to_string(),
title: Some("Pagination Test".to_string()),
custom_scroll_id: None,
metadata: serde_json::json!({}),
completeness: Default::default(),
parent_scroll_id: None,
is_subagent: false,
continuation: None,
agent_id: None,
subagent_type: None,
spawning_tool_use_id: None,
};
let session_response = archivist.register_session(session_req, None).await?;
let scroll_id = session_response.scroll_id;
// Test empty session
let count = archivist.count_messages(scroll_id, None).await?;
assert_eq!(count, 0, "Empty session should have 0 messages");
let range = archivist.get_messages_range(scroll_id, 0, 10, None).await?;
assert_eq!(range.len(), 0, "Empty session should return empty range");
// Add 25 messages with varying timestamps
let mut messages = Vec::new();
let base_time = Utc::now();
for i in 0..25 {
messages.push(MessageRecord {
version: 1,
message_id: Uuid::now_v7(),
session: scroll_id,
parent_id: None,
ts: base_time + chrono::Duration::seconds(i),
role: if i % 2 == 0 { "user" } else { "assistant" }.to_string(),
author: Some("test".to_string()),
content_md: format!("Message {}", i),
content_parts: None,
attachments: Vec::new(),
metadata: serde_json::json!({}),
});
}
archivist.append_messages(scroll_id, messages, None).await?;
// Test count_messages
let count = archivist.count_messages(scroll_id, None).await?;
assert_eq!(count, 25, "Should count 25 messages");
// Test get_messages_range - first page
let page1 = archivist.get_messages_range(scroll_id, 0, 10, None).await?;
assert_eq!(page1.len(), 10, "First page should have 10 messages");
assert_eq!(page1[0].content_md, "Message 0", "First message should be Message 0");
assert_eq!(page1[9].content_md, "Message 9", "10th message should be Message 9");
// Test get_messages_range - second page
let page2 = archivist.get_messages_range(scroll_id, 10, 10, None).await?;
assert_eq!(page2.len(), 10, "Second page should have 10 messages");
assert_eq!(page2[0].content_md, "Message 10", "11th message should be Message 10");
assert_eq!(page2[9].content_md, "Message 19", "20th message should be Message 19");
// Test get_messages_range - partial last page
let page3 = archivist.get_messages_range(scroll_id, 20, 10, None).await?;
assert_eq!(page3.len(), 5, "Last page should have 5 messages");
assert_eq!(page3[0].content_md, "Message 20", "21st message should be Message 20");
assert_eq!(page3[4].content_md, "Message 24", "25th message should be Message 24");
// Test get_messages_range - offset beyond messages
let page4 = archivist.get_messages_range(scroll_id, 30, 10, None).await?;
assert_eq!(page4.len(), 0, "Offset beyond messages should return empty");
// Verify chronological ordering is maintained in pagination
let all_messages = archivist.get_messages(scroll_id, None).await?;
let first_10_from_all = &all_messages[0..10];
let first_10_from_page = &page1[..];
for i in 0..10 {
assert_eq!(
first_10_from_all[i].message_id,
first_10_from_page[i].message_id,
"Pagination should maintain same order as get_messages()"
);
}
// Clean up
tokio::fs::remove_dir_all(temp_dir).await.ok();
Ok(())
}
#[tokio::test]
async fn test_count_messages_nonexistent_session() -> Result<()> {
let temp_dir = std::env::temp_dir().join(format!("archivist_test_{}", Uuid::now_v7()));
let archivist = mk_archivist(temp_dir.clone()).await?;
// Count messages for non-existent session (should return 0, not error)
let nonexistent_scroll_id = Uuid::now_v7();
let count = archivist.count_messages(nonexistent_scroll_id, None).await?;
assert_eq!(count, 0, "Non-existent session should have 0 messages");
// Clean up
tokio::fs::remove_dir_all(temp_dir).await.ok();
Ok(())
}
}