sync from monorepo @ 2452e92e
This commit is contained in:
@@ -0,0 +1,198 @@
|
||||
//! Basic usage example for dirigent_archivist
|
||||
//!
|
||||
//! This example demonstrates:
|
||||
//! - Creating a Archivist
|
||||
//! - Registering a connector
|
||||
//! - Registering a session
|
||||
//! - Appending messages to a session
|
||||
//! - Listing sessions for a connector
|
||||
//! - Retrieving messages for a session
|
||||
|
||||
use chrono::Utc;
|
||||
use dirigent_archivist::{
|
||||
Archivist, MessageRecord, RegisterConnectorRequest, RegisterSessionRequest,
|
||||
Result,
|
||||
};
|
||||
use std::path::PathBuf;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// Create a temporary archive directory for this example
|
||||
let temp_dir = std::env::temp_dir().join(format!("dirigent_example_{}", Uuid::now_v7()));
|
||||
println!("Creating archive at: {}", temp_dir.display());
|
||||
|
||||
// Step 1: Create a Archivist
|
||||
let archivist = Archivist::new_with_single_archive(temp_dir.clone()).await?;
|
||||
println!("Archivist created successfully");
|
||||
|
||||
// Step 2: Register a connector
|
||||
println!("\n--- Registering Connector ---");
|
||||
let connector_req = RegisterConnectorRequest {
|
||||
r#type: "OpenCode".to_string(),
|
||||
title: "OpenCode Local".to_string(),
|
||||
client_native_id: "opencode@http://localhost:12225".to_string(),
|
||||
custom_uid: None, // Let archivist generate a UID
|
||||
metadata: serde_json::json!({
|
||||
"version": "0.1.0",
|
||||
"protocol": "OpenCode HTTP API"
|
||||
}),
|
||||
fingerprint: None,
|
||||
};
|
||||
|
||||
let connector_resp = archivist.register_connector(connector_req, None).await?;
|
||||
println!("Connector registered: {:?}", connector_resp);
|
||||
let connector_uid = connector_resp.connector_uid;
|
||||
|
||||
// Step 3: Register a session
|
||||
println!("\n--- Registering Session ---");
|
||||
let session_req = RegisterSessionRequest {
|
||||
connector_uid,
|
||||
native_session_id: "session-abc123".to_string(),
|
||||
title: Some("Example chat session".to_string()),
|
||||
custom_scroll_id: None, // Let archivist generate a scroll ID
|
||||
metadata: serde_json::json!({
|
||||
"project_path": "/home/user/projects/example",
|
||||
"model": "claude-3-5-sonnet"
|
||||
}),
|
||||
completeness: Default::default(),
|
||||
parent_scroll_id: None,
|
||||
is_subagent: false,
|
||||
continuation: None,
|
||||
agent_id: None,
|
||||
subagent_type: None,
|
||||
spawning_tool_use_id: None,
|
||||
};
|
||||
|
||||
let session_resp = archivist.register_session(session_req, None).await?;
|
||||
println!("Session registered: {:?}", session_resp);
|
||||
let scroll_id = session_resp.scroll_id;
|
||||
|
||||
// Step 4: Append messages to the session
|
||||
println!("\n--- Appending Messages ---");
|
||||
|
||||
// User message
|
||||
let user_msg = MessageRecord {
|
||||
version: 1,
|
||||
message_id: Uuid::now_v7(),
|
||||
session: scroll_id,
|
||||
parent_id: None,
|
||||
ts: Utc::now(),
|
||||
role: "user".to_string(),
|
||||
author: Some("alice".to_string()),
|
||||
content_md: "Hello! Can you help me write a function to calculate fibonacci numbers?"
|
||||
.to_string(),
|
||||
content_parts: None,
|
||||
attachments: vec![],
|
||||
metadata: serde_json::json!({}),
|
||||
};
|
||||
|
||||
// Assistant message
|
||||
let assistant_msg = MessageRecord {
|
||||
version: 1,
|
||||
message_id: Uuid::now_v7(),
|
||||
session: scroll_id,
|
||||
parent_id: Some(user_msg.message_id),
|
||||
ts: Utc::now(),
|
||||
role: "assistant".to_string(),
|
||||
author: Some("claude".to_string()),
|
||||
content_md: r#"Sure! Here's a recursive fibonacci function in Rust:
|
||||
|
||||
```rust
|
||||
fn fibonacci(n: u32) -> u64 {
|
||||
match n {
|
||||
0 => 0,
|
||||
1 => 1,
|
||||
_ => fibonacci(n - 1) + fibonacci(n - 2),
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This is the classic recursive implementation, though it's not the most efficient for large values of n."#
|
||||
.to_string(),
|
||||
content_parts: None,
|
||||
attachments: vec![],
|
||||
metadata: serde_json::json!({
|
||||
"model": "claude-3-5-sonnet",
|
||||
"latency_ms": 1245
|
||||
}),
|
||||
};
|
||||
|
||||
archivist
|
||||
.append_messages(scroll_id, vec![user_msg.clone(), assistant_msg.clone()], None)
|
||||
.await?;
|
||||
println!("Appended 2 messages to session");
|
||||
|
||||
// Step 5: List all sessions for the connector
|
||||
println!("\n--- Listing Sessions ---");
|
||||
let page = archivist
|
||||
.list_sessions_paged(
|
||||
dirigent_archivist::SessionListQuery::default()
|
||||
.with_connector(connector_uid)
|
||||
.with_limit(100),
|
||||
)
|
||||
.await?;
|
||||
let sessions = page.items;
|
||||
println!("Found {} session(s) for connector:", sessions.len());
|
||||
for session in &sessions {
|
||||
println!(
|
||||
" - {} ({}): {:?}",
|
||||
session.scroll_id,
|
||||
session.created_at.format("%Y-%m-%d %H:%M:%S"),
|
||||
session.title
|
||||
);
|
||||
}
|
||||
|
||||
// Step 6: Retrieve all messages for the session
|
||||
println!("\n--- Retrieving Messages ---");
|
||||
let messages = archivist.get_messages(scroll_id, None).await?;
|
||||
println!("Retrieved {} message(s):", messages.len());
|
||||
for msg in &messages {
|
||||
println!("\n[{}] {}", msg.role, msg.ts.format("%Y-%m-%d %H:%M:%S"));
|
||||
println!("{}", msg.content_md);
|
||||
}
|
||||
|
||||
// Step 7: Demonstrate session resolution
|
||||
println!("\n--- Resolving Session ---");
|
||||
let resolved_scroll_id = archivist
|
||||
.resolve_session(connector_uid, "session-abc123", None)
|
||||
.await?;
|
||||
println!(
|
||||
"Resolved native session 'session-abc123' to scroll_id: {}",
|
||||
resolved_scroll_id
|
||||
);
|
||||
assert_eq!(resolved_scroll_id, scroll_id);
|
||||
|
||||
// Step 8: Show archive structure
|
||||
println!("\n--- Archive Structure ---");
|
||||
println!("Archive root: {}", temp_dir.display());
|
||||
println!("\nDirectory structure:");
|
||||
show_directory_tree(&temp_dir, 0)?;
|
||||
|
||||
// Cleanup
|
||||
println!("\n--- Cleanup ---");
|
||||
std::fs::remove_dir_all(&temp_dir)?;
|
||||
println!("Removed temporary archive");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper function to display directory tree
|
||||
fn show_directory_tree(path: &PathBuf, depth: usize) -> Result<()> {
|
||||
let indent = " ".repeat(depth);
|
||||
|
||||
if path.is_dir() {
|
||||
println!("{}{}/", indent, path.file_name().unwrap().to_string_lossy());
|
||||
|
||||
let mut entries: Vec<_> = std::fs::read_dir(path)?.filter_map(|e| e.ok()).collect();
|
||||
entries.sort_by_key(|e| e.path());
|
||||
|
||||
for entry in entries {
|
||||
show_directory_tree(&entry.path(), depth + 1)?;
|
||||
}
|
||||
} else {
|
||||
println!("{}{}", indent, path.file_name().unwrap().to_string_lossy());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
// Demonstration of archivist types serialization
|
||||
// Run with: cargo run --package dirigent_archivist --example demo_types
|
||||
|
||||
use chrono::Utc;
|
||||
use dirigent_archivist::*;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn main() {
|
||||
println!("=== ARCHIVIST TYPES DEMONSTRATION ===\n");
|
||||
|
||||
// Demo 1: SessionMetadata (matches session.json format)
|
||||
println!("1. SessionMetadata (session.json):");
|
||||
let session_metadata = SessionMetadata {
|
||||
version: 1,
|
||||
scroll_id: Uuid::now_v7(),
|
||||
created_at: Utc::now(),
|
||||
updated_at: Utc::now(),
|
||||
title: Some("Example Session".to_string()),
|
||||
connector_uid: Uuid::now_v7(),
|
||||
native_session_id: Some("abc123".to_string()),
|
||||
agent_id: Some("claude-3-5".to_string()),
|
||||
parent_scroll_id: None,
|
||||
continuation: Some(Continuation::Split),
|
||||
tags: vec!["example".to_string(), "test".to_string()],
|
||||
metadata: serde_json::json!({
|
||||
"source": "OpenCode",
|
||||
"project": "dirigent"
|
||||
}),
|
||||
no_update: false,
|
||||
kind: SessionKind::Chat,
|
||||
acp_client_id: None,
|
||||
is_connected: None,
|
||||
current_session_id: None,
|
||||
models: None,
|
||||
modes: None,
|
||||
config_options: None,
|
||||
completeness: SessionCompleteness::default(),
|
||||
matrix_room_id: None,
|
||||
matrix_sharing_active: false,
|
||||
matrix_shared_at: None,
|
||||
is_subagent: false,
|
||||
subagent_type: None,
|
||||
spawning_tool_use_id: None,
|
||||
};
|
||||
println!(
|
||||
"{}\n",
|
||||
serde_json::to_string_pretty(&session_metadata).unwrap()
|
||||
);
|
||||
|
||||
// Demo 2: MessageRecord (matches messages.ndjson format)
|
||||
println!("2. MessageRecord (messages.ndjson line):");
|
||||
let message = MessageRecord {
|
||||
version: 1,
|
||||
message_id: Uuid::now_v7(),
|
||||
session: session_metadata.scroll_id,
|
||||
parent_id: None,
|
||||
ts: Utc::now(),
|
||||
role: "user".to_string(),
|
||||
author: Some("alice".to_string()),
|
||||
content_md: "How do I implement archivist types?".to_string(),
|
||||
content_parts: None,
|
||||
attachments: vec![AttachmentRef {
|
||||
file_id: "sha256:abc123".to_string(),
|
||||
name: "spec.pdf".to_string(),
|
||||
mime_type: Some("application/pdf".to_string()),
|
||||
}],
|
||||
metadata: serde_json::json!({
|
||||
"connector_msg_id": "msg-456"
|
||||
}),
|
||||
};
|
||||
// NDJSON format (one line)
|
||||
println!("{}\n", serde_json::to_string(&message).unwrap());
|
||||
|
||||
// Demo 3: ConnectorRecord (matches connector.json format)
|
||||
println!("3. ConnectorRecord (connector.json):");
|
||||
let connector = ConnectorRecord {
|
||||
version: 1,
|
||||
connector_uid: session_metadata.connector_uid,
|
||||
r#type: "OpenCode".to_string(),
|
||||
title: "OpenCode Local".to_string(),
|
||||
client_native_id: "opencode@http://localhost:12225".to_string(),
|
||||
alias_of: None,
|
||||
created_at: Utc::now(),
|
||||
metadata: serde_json::json!({}),
|
||||
fingerprint: None,
|
||||
};
|
||||
println!("{}\n", serde_json::to_string_pretty(&connector).unwrap());
|
||||
|
||||
// Demo 4: SessionMapping (matches sessions.ndjson format)
|
||||
println!("4. SessionMapping (sessions.ndjson line):");
|
||||
let mapping = SessionMapping {
|
||||
version: 1,
|
||||
connector_uid: connector.connector_uid,
|
||||
native_session_id: "abc123".to_string(),
|
||||
scroll_id: session_metadata.scroll_id,
|
||||
created_at: Utc::now(),
|
||||
alias_of: None,
|
||||
};
|
||||
println!("{}\n", serde_json::to_string(&mapping).unwrap());
|
||||
|
||||
// Demo 5: FileRecord (matches file_index.jsonl format)
|
||||
println!("5. FileRecord (file_index.jsonl line):");
|
||||
let file_record = FileRecord {
|
||||
version: 1,
|
||||
file_id: "sha256:abc123def456".to_string(),
|
||||
path: ".files/ab/cd/abc123def456".to_string(),
|
||||
size: 123456,
|
||||
mime: Some("application/pdf".to_string()),
|
||||
original_name: "spec.pdf".to_string(),
|
||||
sessions: vec![session_metadata.scroll_id],
|
||||
metadata: serde_json::json!({
|
||||
"source": "upload"
|
||||
}),
|
||||
};
|
||||
println!("{}\n", serde_json::to_string(&file_record).unwrap());
|
||||
|
||||
// Demo 6: Enum serialization
|
||||
println!("6. Enum Serialization:");
|
||||
println!(
|
||||
" Continuation::Split: {}",
|
||||
serde_json::to_string(&Continuation::Split).unwrap()
|
||||
);
|
||||
println!(
|
||||
" Continuation::Compact: {}",
|
||||
serde_json::to_string(&Continuation::Compact).unwrap()
|
||||
);
|
||||
println!(
|
||||
" RegisterStatus::Accepted: {}",
|
||||
serde_json::to_string(&RegisterStatus::Accepted).unwrap()
|
||||
);
|
||||
println!(
|
||||
" RegisterStatus::Aliased: {}",
|
||||
serde_json::to_string(&RegisterStatus::Aliased).unwrap()
|
||||
);
|
||||
println!();
|
||||
|
||||
// Demo 7: API types
|
||||
println!("7. RegisterConnectorResponse:");
|
||||
let response = RegisterConnectorResponse {
|
||||
status: RegisterStatus::Accepted,
|
||||
connector_uid: Uuid::now_v7(),
|
||||
alias_of: None,
|
||||
note: Some("Successfully registered".to_string()),
|
||||
};
|
||||
println!("{}\n", serde_json::to_string_pretty(&response).unwrap());
|
||||
|
||||
println!("8. RegisterSessionResponse:");
|
||||
let response = RegisterSessionResponse {
|
||||
status: RegisterStatus::Aliased,
|
||||
scroll_id: Uuid::now_v7(),
|
||||
alias_of: Some(Uuid::now_v7()),
|
||||
};
|
||||
println!("{}\n", serde_json::to_string_pretty(&response).unwrap());
|
||||
|
||||
println!("=== ALL TYPES MATCH VISION.MD SPECIFICATION ===");
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
//! Event handling example for dirigent_archivist
|
||||
//!
|
||||
//! This example demonstrates:
|
||||
//! - Creating an EventHandler
|
||||
//! - Subscribing to dirigent_protocol events
|
||||
//! - Accumulating streaming message chunks
|
||||
//! - Finalizing complete messages
|
||||
//! - Automatic archival via event stream
|
||||
|
||||
use chrono::Utc;
|
||||
use dirigent_archivist::{Archivist, EventHandler, Result};
|
||||
use dirigent_protocol::streaming::{BusEvent, BusReceiver, EventOrigin, EventRouting};
|
||||
use dirigent_protocol::{
|
||||
ContentBlock, Event, Message, MessageMetadata, MessagePart, MessageRole, MessageStatus,
|
||||
Session, SessionMetadata, SessionUpdate, ToolCall, ToolCallStatus,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use tokio::sync::mpsc;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Wrap a raw `Event` in a `BusEvent` with default routing.
|
||||
fn wrap(event: Event) -> BusEvent {
|
||||
BusEvent {
|
||||
routing: EventRouting::default(),
|
||||
origin: EventOrigin::Runtime,
|
||||
event: Arc::new(event),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// Create a temporary archive directory for this example
|
||||
let temp_dir = std::env::temp_dir().join(format!("dirigent_event_example_{}", Uuid::now_v7()));
|
||||
println!("Creating archive at: {}", temp_dir.display());
|
||||
|
||||
// Step 1: Create archivist and event handler
|
||||
let archivist = Archivist::new_with_single_archive(temp_dir.clone()).await?;
|
||||
let archivist = Arc::new(archivist);
|
||||
let handler = EventHandler::new(archivist.clone());
|
||||
|
||||
println!("EventHandler created successfully");
|
||||
|
||||
// Step 2: Create a mock event stream. In production this is built
|
||||
// by `SharingBus::subscribe_all()`; here we fabricate a `BusReceiver`
|
||||
// directly so the example stays self-contained.
|
||||
let (tx, rx) = mpsc::channel::<BusEvent>(100);
|
||||
let bus_rx = BusReceiver {
|
||||
id: 0,
|
||||
rx,
|
||||
lagged: Arc::new(AtomicU64::new(0)),
|
||||
};
|
||||
|
||||
// Step 3: Spawn event handler task
|
||||
let handler_task = tokio::spawn(async move {
|
||||
handler.run(bus_rx).await;
|
||||
});
|
||||
|
||||
// Step 4: Simulate event flow
|
||||
println!("\n--- Simulating Event Stream ---");
|
||||
|
||||
// Generate connector and session IDs
|
||||
let connector_id = Uuid::now_v7().to_string();
|
||||
let session_id = Uuid::now_v7().to_string();
|
||||
let message_id = Uuid::now_v7().to_string();
|
||||
|
||||
// Event 1: SessionCreated
|
||||
println!("\n1. Sending SessionCreated event...");
|
||||
let session_created = Event::SessionCreated {
|
||||
connector_id: connector_id.clone(),
|
||||
session: Session {
|
||||
id: session_id.clone(),
|
||||
title: "Example streaming session".to_string(),
|
||||
created_at: Utc::now(),
|
||||
updated_at: Utc::now(),
|
||||
metadata: SessionMetadata {
|
||||
project_path: "/home/user/project".to_string(),
|
||||
model: Some("claude-3-5-sonnet".to_string()),
|
||||
total_messages: 0,
|
||||
system_message: None,
|
||||
current_mode_id: None,
|
||||
_meta: None,
|
||||
project_id: None,
|
||||
},
|
||||
cwd: None,
|
||||
models: None,
|
||||
modes: None,
|
||||
config_options: None,
|
||||
acp_client_id: None,
|
||||
},
|
||||
};
|
||||
tx.send(wrap(session_created)).await.unwrap();
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||
|
||||
// Event 2-5: Streaming message chunks (AgentMessageChunk)
|
||||
println!("2. Sending streaming message chunks...");
|
||||
let chunks = vec!["Hello! ", "I'm here to ", "help you with ", "your code."];
|
||||
|
||||
for (i, chunk) in chunks.iter().enumerate() {
|
||||
let chunk_event = Event::SessionUpdate {
|
||||
connector_id: connector_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
update: SessionUpdate::AgentMessageChunk {
|
||||
message_id: message_id.clone(),
|
||||
content: ContentBlock::Text {
|
||||
text: chunk.to_string(),
|
||||
},
|
||||
_meta: None,
|
||||
},
|
||||
};
|
||||
tx.send(wrap(chunk_event)).await.unwrap();
|
||||
println!(" Chunk {}: {:?}", i + 1, chunk);
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
// Event 6: Thinking chunk
|
||||
println!("3. Sending thinking chunk...");
|
||||
let thinking_event = Event::SessionUpdate {
|
||||
connector_id: connector_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
update: SessionUpdate::AgentThoughtChunk {
|
||||
message_id: message_id.clone(),
|
||||
content: ContentBlock::Text {
|
||||
text: "Let me consider the best approach...".to_string(),
|
||||
},
|
||||
_meta: None,
|
||||
},
|
||||
};
|
||||
tx.send(wrap(thinking_event)).await.unwrap();
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||
|
||||
// Event 7: Tool call
|
||||
println!("4. Sending tool call event...");
|
||||
let tool_call_event = Event::SessionUpdate {
|
||||
connector_id: connector_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
update: SessionUpdate::ToolCall {
|
||||
message_id: message_id.clone(),
|
||||
tool_call: ToolCall {
|
||||
id: "tool_call_123".to_string(),
|
||||
tool_name: "read_file".to_string(),
|
||||
status: ToolCallStatus::Completed,
|
||||
content: vec![],
|
||||
raw_input: Some(serde_json::json!({
|
||||
"path": "/home/user/project/main.rs"
|
||||
})),
|
||||
raw_output: Some(serde_json::json!({
|
||||
"content": "fn main() { println!(\"Hello\"); }"
|
||||
})),
|
||||
title: None,
|
||||
error: None,
|
||||
metadata: None,
|
||||
origin: None,
|
||||
},
|
||||
_meta: None,
|
||||
},
|
||||
};
|
||||
tx.send(wrap(tool_call_event)).await.unwrap();
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
|
||||
|
||||
// Event 8: MessageCompleted (triggers finalization)
|
||||
println!("5. Sending MessageCompleted event...");
|
||||
let message_completed = Event::MessageCompleted {
|
||||
connector_id: connector_id.clone(),
|
||||
message: Message {
|
||||
id: message_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
role: MessageRole::Assistant,
|
||||
created_at: Utc::now(),
|
||||
content: vec![MessagePart::Text {
|
||||
text: chunks.concat(),
|
||||
}],
|
||||
status: MessageStatus::Completed,
|
||||
metadata: Some(MessageMetadata {
|
||||
cost: None,
|
||||
tokens_input: None,
|
||||
tokens_output: None,
|
||||
response_time_ms: None,
|
||||
latency_ms: Some(1500),
|
||||
model: Some("claude-3-5-sonnet".to_string()),
|
||||
other: None,
|
||||
}),
|
||||
},
|
||||
};
|
||||
tx.send(wrap(message_completed)).await.unwrap();
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;
|
||||
|
||||
// Event 9: Second message (user response)
|
||||
println!("6. Sending user message...");
|
||||
let user_message_id = Uuid::now_v7().to_string();
|
||||
let user_chunks = vec!["Thanks! ", "Can you explain ", "the code?"];
|
||||
|
||||
for (i, chunk) in user_chunks.iter().enumerate() {
|
||||
let chunk_event = Event::SessionUpdate {
|
||||
connector_id: connector_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
update: SessionUpdate::UserMessageChunk {
|
||||
message_id: user_message_id.clone(),
|
||||
content: ContentBlock::Text {
|
||||
text: chunk.to_string(),
|
||||
},
|
||||
_meta: None,
|
||||
},
|
||||
};
|
||||
tx.send(wrap(chunk_event)).await.unwrap();
|
||||
println!(" User chunk {}: {:?}", i + 1, chunk);
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
|
||||
let user_message_completed = Event::MessageCompleted {
|
||||
connector_id: connector_id.clone(),
|
||||
message: Message {
|
||||
id: user_message_id.clone(),
|
||||
session_id: session_id.clone(),
|
||||
role: MessageRole::User,
|
||||
created_at: Utc::now(),
|
||||
content: vec![MessagePart::Text {
|
||||
text: user_chunks.concat(),
|
||||
}],
|
||||
status: MessageStatus::Completed,
|
||||
metadata: None,
|
||||
},
|
||||
};
|
||||
tx.send(wrap(user_message_completed)).await.unwrap();
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(200)).await;
|
||||
|
||||
// Step 5: Verify archived data
|
||||
println!("\n--- Verifying Archived Data ---");
|
||||
|
||||
// Parse connector_uid from connector_id string
|
||||
let connector_uid =
|
||||
Uuid::parse_str(&connector_id).expect("connector_id should be a valid UUID");
|
||||
|
||||
// List sessions
|
||||
let page = archivist
|
||||
.list_sessions_paged(
|
||||
dirigent_archivist::SessionListQuery::default()
|
||||
.with_connector(connector_uid)
|
||||
.with_limit(100),
|
||||
)
|
||||
.await?;
|
||||
let sessions = page.items;
|
||||
println!("Found {} session(s) in archive", sessions.len());
|
||||
for session in &sessions {
|
||||
println!(" Session: {} - {:?}", session.scroll_id, session.title);
|
||||
}
|
||||
|
||||
// Get messages
|
||||
if let Some(session) = sessions.first() {
|
||||
let messages = archivist.get_messages(session.scroll_id, None).await?;
|
||||
println!("\nFound {} message(s):", messages.len());
|
||||
for msg in &messages {
|
||||
println!("\n[{}] {} chars", msg.role, msg.content_md.len());
|
||||
println!(
|
||||
"Content preview: {}",
|
||||
&msg.content_md.chars().take(100).collect::<String>()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Step 6: Cleanup
|
||||
println!("\n--- Cleanup ---");
|
||||
|
||||
// Drop the event sender to close the channel
|
||||
drop(tx);
|
||||
|
||||
// Wait for handler to finish
|
||||
handler_task.await.expect("Handler task failed");
|
||||
|
||||
// Remove temporary archive
|
||||
std::fs::remove_dir_all(&temp_dir)?;
|
||||
println!("Removed temporary archive");
|
||||
|
||||
println!("\nExample completed successfully!");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
//! File storage example for dirigent_archivist
|
||||
//!
|
||||
//! This example demonstrates:
|
||||
//! - Storing files with content-addressing
|
||||
//! - Retrieving files by file_id
|
||||
//! - Automatic deduplication of identical content
|
||||
//! - Session tracking for file references
|
||||
|
||||
use dirigent_archivist::storage::{files, ndjson, paths::ArchivePaths};
|
||||
use dirigent_archivist::types::FileRecord;
|
||||
use dirigent_archivist::Result;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// Create a temporary archive directory for this example
|
||||
let temp_dir = std::env::temp_dir().join(format!("dirigent_files_example_{}", Uuid::now_v7()));
|
||||
println!("Creating archive at: {}", temp_dir.display());
|
||||
|
||||
let paths = ArchivePaths::new(temp_dir.clone());
|
||||
|
||||
// Example 1: Store a file
|
||||
println!("\n--- Example 1: Store a File ---");
|
||||
let content1 = b"This is a sample document with some text content.";
|
||||
let session1 = Uuid::now_v7();
|
||||
|
||||
let file_id1 = files::store_file(
|
||||
&paths,
|
||||
content1,
|
||||
"document.txt".to_string(),
|
||||
Some("text/plain".to_string()),
|
||||
session1,
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Stored file with ID: {}", file_id1);
|
||||
println!("Session: {}", session1);
|
||||
|
||||
// Example 2: Retrieve the file
|
||||
println!("\n--- Example 2: Retrieve the File ---");
|
||||
let retrieved1 = files::get_file(&paths, &file_id1).await?;
|
||||
println!("Retrieved {} bytes", retrieved1.len());
|
||||
println!("Content: {}", String::from_utf8_lossy(&retrieved1));
|
||||
|
||||
// Example 3: Store the same content from a different session (deduplication)
|
||||
println!("\n--- Example 3: Deduplication Demo ---");
|
||||
let session2 = Uuid::now_v7();
|
||||
|
||||
let file_id2 = files::store_file(
|
||||
&paths,
|
||||
content1, // Same content as before
|
||||
"duplicate.txt".to_string(), // Different name
|
||||
Some("text/plain".to_string()),
|
||||
session2,
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Stored same content with different name");
|
||||
println!("File ID 1: {}", file_id1);
|
||||
println!("File ID 2: {}", file_id2);
|
||||
println!("Same file_id? {}", file_id1 == file_id2);
|
||||
println!("\nDeduplication: Same content produces same file_id, stored only once!");
|
||||
|
||||
// Example 4: Check the file index
|
||||
println!("\n--- Example 4: File Index ---");
|
||||
let index_path = paths.root().join(".files").join("file_index.jsonl");
|
||||
let records: Vec<FileRecord> = ndjson::read_ndjson(&index_path).await?;
|
||||
|
||||
println!("File index contains {} record(s)", records.len());
|
||||
for record in &records {
|
||||
println!("\nFile: {}", record.file_id);
|
||||
println!(" Original name: {}", record.original_name);
|
||||
println!(" MIME type: {:?}", record.mime);
|
||||
println!(" Size: {} bytes", record.size);
|
||||
println!(" Referenced by {} session(s):", record.sessions.len());
|
||||
for session_id in &record.sessions {
|
||||
println!(" - {}", session_id);
|
||||
}
|
||||
}
|
||||
|
||||
// Example 5: Store different content
|
||||
println!("\n--- Example 5: Store Different Content ---");
|
||||
let content2 = b"This is completely different content with more data!";
|
||||
let session3 = Uuid::now_v7();
|
||||
|
||||
let file_id3 = files::store_file(
|
||||
&paths,
|
||||
content2,
|
||||
"different.txt".to_string(),
|
||||
Some("text/plain".to_string()),
|
||||
session3,
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Stored different content");
|
||||
println!("File ID 3: {}", file_id3);
|
||||
println!("Different from file_id1? {}", file_id1 != file_id3);
|
||||
|
||||
// Example 6: Store binary content
|
||||
println!("\n--- Example 6: Binary Content ---");
|
||||
let binary_content: Vec<u8> = (0..256).map(|i| i as u8).collect();
|
||||
let session4 = Uuid::now_v7();
|
||||
|
||||
let file_id4 = files::store_file(
|
||||
&paths,
|
||||
&binary_content,
|
||||
"binary.dat".to_string(),
|
||||
Some("application/octet-stream".to_string()),
|
||||
session4,
|
||||
)
|
||||
.await?;
|
||||
|
||||
println!("Stored binary content (256 bytes)");
|
||||
println!("File ID: {}", file_id4);
|
||||
|
||||
// Retrieve and verify
|
||||
let retrieved_binary = files::get_file(&paths, &file_id4).await?;
|
||||
println!("Retrieved {} bytes", retrieved_binary.len());
|
||||
println!(
|
||||
"Binary content verified: {}",
|
||||
retrieved_binary == binary_content
|
||||
);
|
||||
|
||||
// Example 7: Show final archive structure
|
||||
println!("\n--- Example 7: Archive Structure ---");
|
||||
println!("Archive root: {}", temp_dir.display());
|
||||
show_files_directory(&paths)?;
|
||||
|
||||
// Example 8: Final statistics
|
||||
println!("\n--- Final Statistics ---");
|
||||
let final_records: Vec<FileRecord> = ndjson::read_ndjson(&index_path).await?;
|
||||
println!("Total unique files stored: {}", final_records.len());
|
||||
|
||||
let total_sessions: usize = final_records.iter().map(|r| r.sessions.len()).sum();
|
||||
println!("Total session references: {}", total_sessions);
|
||||
|
||||
let total_size: u64 = final_records.iter().map(|r| r.size).sum();
|
||||
println!("Total storage used: {} bytes", total_size);
|
||||
|
||||
// Content-addressing means if we had stored content1 1000 times,
|
||||
// we'd still only use storage for it once!
|
||||
println!("\nContent-addressing benefit:");
|
||||
println!(" File '{}' is referenced by {} sessions", file_id1, 2);
|
||||
println!(" But stored only once on disk!");
|
||||
|
||||
// Cleanup
|
||||
println!("\n--- Cleanup ---");
|
||||
std::fs::remove_dir_all(&temp_dir)?;
|
||||
println!("Removed temporary archive");
|
||||
|
||||
println!("\nExample completed successfully!");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper function to show .files directory structure
|
||||
fn show_files_directory(paths: &ArchivePaths) -> Result<()> {
|
||||
let files_dir = paths.root().join(".files");
|
||||
|
||||
if !files_dir.exists() {
|
||||
println!("No files directory found");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
println!("\n.files/ directory:");
|
||||
|
||||
// Show index file
|
||||
let index_path = files_dir.join("file_index.jsonl");
|
||||
if index_path.exists() {
|
||||
let metadata = std::fs::metadata(&index_path)?;
|
||||
println!(" file_index.jsonl ({} bytes)", metadata.len());
|
||||
}
|
||||
|
||||
// Show shard directories
|
||||
for entry in std::fs::read_dir(&files_dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
|
||||
if path.is_dir() {
|
||||
println!(" {}/", path.file_name().unwrap().to_string_lossy());
|
||||
|
||||
// Show files in shard
|
||||
for sub_entry in std::fs::read_dir(&path)? {
|
||||
let sub_entry = sub_entry?;
|
||||
let sub_path = sub_entry.path();
|
||||
|
||||
if sub_path.is_dir() {
|
||||
println!(" {}/", sub_path.file_name().unwrap().to_string_lossy());
|
||||
|
||||
// Show files in sub-shard
|
||||
for file_entry in std::fs::read_dir(&sub_path)? {
|
||||
let file_entry = file_entry?;
|
||||
let file_path = file_entry.path();
|
||||
let metadata = std::fs::metadata(&file_path)?;
|
||||
println!(
|
||||
" {} ({} bytes)",
|
||||
file_path.file_name().unwrap().to_string_lossy(),
|
||||
metadata.len()
|
||||
);
|
||||
}
|
||||
} else {
|
||||
let metadata = std::fs::metadata(&sub_path)?;
|
||||
println!(
|
||||
" {} ({} bytes)",
|
||||
sub_path.file_name().unwrap().to_string_lossy(),
|
||||
metadata.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
//! Example: two `JsonlBackend`s side by side, demonstrating boot-from-config,
|
||||
//! priority-ordered read routing, write fanout, and a health snapshot.
|
||||
//!
|
||||
//! Layout:
|
||||
//! - `primary` → `read_priority = 0`, `failure_mode = required` (default)
|
||||
//! - `mirror` → `read_priority = 10`, `failure_mode = best_effort`
|
||||
//!
|
||||
//! The primary is the default write target (lowest priority among
|
||||
//! Required+write-active backends). `append_messages` fans out inline to the
|
||||
//! mirror too. Reads walk the registrations in priority order, so the primary
|
||||
//! answers first; if it is missing a session, the walk falls through to the
|
||||
//! mirror.
|
||||
//!
|
||||
//! Run with:
|
||||
//!
|
||||
//! cargo run --package dirigent_archivist --example multi_backend
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::Utc;
|
||||
use dirigent_archivist::coordinator::Archivist;
|
||||
use dirigent_archivist::registry::{ArchivesConfig, BackendRegistry};
|
||||
use dirigent_archivist::types::{
|
||||
MessageRecord, RegisterConnectorRequest, RegisterSessionRequest,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let dir_a = tempfile::tempdir()?;
|
||||
let dir_b = tempfile::tempdir()?;
|
||||
|
||||
// Build a two-archive config entirely from TOML so the example doubles as
|
||||
// a faithful demonstration of the config surface.
|
||||
let cfg_src = format!(
|
||||
r#"
|
||||
[[archives]]
|
||||
name = "primary"
|
||||
type = "jsonl"
|
||||
read_priority = 0
|
||||
[archives.params]
|
||||
path = "{a}"
|
||||
|
||||
[[archives]]
|
||||
name = "mirror"
|
||||
type = "jsonl"
|
||||
failure_mode = "best_effort"
|
||||
read_priority = 10
|
||||
[archives.params]
|
||||
path = "{b}"
|
||||
"#,
|
||||
a = dir_a.path().to_string_lossy().replace('\\', "/"),
|
||||
b = dir_b.path().to_string_lossy().replace('\\', "/"),
|
||||
);
|
||||
let cfg: ArchivesConfig = toml::from_str(&cfg_src)?;
|
||||
let registry = BackendRegistry::with_jsonl();
|
||||
let archivist = Arc::new(Archivist::from_config(cfg, ®istry, None).await?);
|
||||
|
||||
println!("\n=== Multi-backend Archivist example ===\n");
|
||||
println!("Boot complete. Archives (ordered by read_priority):");
|
||||
for s in archivist.list_archives_with_health().await {
|
||||
println!(
|
||||
" - name={:<8} type={:<6} priority={:<3} enabled={} write_active={} failure_mode={:?} health={:?}",
|
||||
s.name,
|
||||
s.type_name,
|
||||
s.read_priority,
|
||||
s.enabled,
|
||||
s.write_active,
|
||||
s.failure_mode,
|
||||
s.health,
|
||||
);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Register a connector. The primary owns the canonical record; fanout
|
||||
// mirrors it to the secondary.
|
||||
// ------------------------------------------------------------------
|
||||
let connector_resp = archivist
|
||||
.register_connector(
|
||||
RegisterConnectorRequest {
|
||||
r#type: "Example".into(),
|
||||
title: "multi-backend demo".into(),
|
||||
client_native_id: "example://multi_backend".into(),
|
||||
custom_uid: None,
|
||||
metadata: serde_json::json!({ "demo": true }),
|
||||
fingerprint: None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let connector_uid = connector_resp.connector_uid;
|
||||
println!(
|
||||
"\nRegistered connector: uid={} status={:?}",
|
||||
connector_uid, connector_resp.status
|
||||
);
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Register a session under that connector. `register_session` writes
|
||||
// the mapping and `session.json` on the primary first, then fans out
|
||||
// to any enabled secondaries.
|
||||
// ------------------------------------------------------------------
|
||||
let session_resp = archivist
|
||||
.register_session(
|
||||
RegisterSessionRequest {
|
||||
connector_uid,
|
||||
native_session_id: "demo-session-1".into(),
|
||||
title: Some("multi-backend demo session".into()),
|
||||
custom_scroll_id: None,
|
||||
metadata: serde_json::json!({ "model": "demo" }),
|
||||
completeness: Default::default(),
|
||||
parent_scroll_id: None,
|
||||
is_subagent: false,
|
||||
continuation: None,
|
||||
agent_id: None,
|
||||
subagent_type: None,
|
||||
spawning_tool_use_id: None,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let scroll_id = session_resp.scroll_id;
|
||||
println!(
|
||||
"Registered session: scroll_id={} status={:?}",
|
||||
scroll_id, session_resp.status
|
||||
);
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Append a couple of messages. `append_messages` writes to the primary
|
||||
// and then fans out inline to the mirror.
|
||||
// ------------------------------------------------------------------
|
||||
let user_msg = MessageRecord {
|
||||
version: 1,
|
||||
message_id: Uuid::now_v7(),
|
||||
session: scroll_id,
|
||||
parent_id: None,
|
||||
ts: Utc::now(),
|
||||
role: "user".into(),
|
||||
author: Some("alice".into()),
|
||||
content_md: "Hello from the multi-backend example!".into(),
|
||||
content_parts: None,
|
||||
attachments: vec![],
|
||||
metadata: serde_json::json!({}),
|
||||
};
|
||||
let asst_msg = MessageRecord {
|
||||
version: 1,
|
||||
message_id: Uuid::now_v7(),
|
||||
session: scroll_id,
|
||||
parent_id: Some(user_msg.message_id),
|
||||
ts: Utc::now(),
|
||||
role: "assistant".into(),
|
||||
author: Some("claude".into()),
|
||||
content_md: "Greetings. I have been written to two archives.".into(),
|
||||
content_parts: None,
|
||||
attachments: vec![],
|
||||
metadata: serde_json::json!({}),
|
||||
};
|
||||
archivist
|
||||
.append_messages(scroll_id, vec![user_msg, asst_msg], None)
|
||||
.await?;
|
||||
println!("\nAppended 2 messages — fanned out to primary + mirror.");
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Read path: the priority walk tries the primary first (priority=0).
|
||||
// It finds the session there and never consults the mirror.
|
||||
// ------------------------------------------------------------------
|
||||
let meta = archivist.get_session_metadata(scroll_id, None).await?;
|
||||
println!(
|
||||
"\nRead session via priority walk: title={:?} completeness={:?}",
|
||||
meta.title, meta.completeness
|
||||
);
|
||||
println!(
|
||||
"Read cache size after read: {}",
|
||||
archivist.read_cache_size().await
|
||||
);
|
||||
|
||||
let messages = archivist.get_messages(scroll_id, None).await?;
|
||||
println!("Read {} message(s) from the archive:", messages.len());
|
||||
for m in &messages {
|
||||
println!(" - [{}] {}", m.role, m.content_md);
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Final health snapshot. Both backends should still be Available and
|
||||
// have no queued writes (both run Inline write policies by default).
|
||||
// ------------------------------------------------------------------
|
||||
println!("\nFinal health snapshot:");
|
||||
for s in archivist.list_archives_with_health().await {
|
||||
println!(
|
||||
" - {:<8} health={:?} queue_depth={:?} last_error={:?}",
|
||||
s.name, s.health, s.queue_depth, s.last_error
|
||||
);
|
||||
}
|
||||
|
||||
// Clean shutdown drains any queued writer tasks. Both backends here run
|
||||
// Inline, so this is effectively a no-op but remains the correct API.
|
||||
archivist.shutdown().await?;
|
||||
println!("\nShutdown complete.");
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user