✨ feat(fermata): add secret filtering engine — the security brain
Implement Goals 1–3 and 5 from the reveal-layer security brain goal.
fermata now detects, redacts, and scans for secrets in AI agent tool
output, filling the ecosystem gap where no coding agent filters secrets
post-read.
New core/secrets/ module:
- config.rs: .botsecrets TOML format with hierarchical merge and ~40
built-in key patterns
- parser.rs: multi-format secret file parser (.env, TOML, YAML, JSON,
Python assignments, Java properties)
- manifest.rs: file discovery + parsing → known-secrets set
- redactor.rs: Aho-Corasick multi-pattern replacement with 4 styles
- scanner.rs: RegexSet heuristic detection with 35 gitleaks-derived
patterns (MIT) and Shannon entropy filtering
- patterns.rs: curated rules for AWS, GitHub, Stripe, Slack, JWT, etc.
Hook integration:
- fermata hook --event post-tool-use reads tool output, runs redactor +
scanner, returns updatedToolOutput for Claude Code
- Backward compatible: --event pre-tool-use (default) unchanged
- Fail-open: errors produce {} and exit 0
Library API:
- Redactor::new(manifest, style).redact(text) → RedactedText
- Scanner::new(config).scan(text) → Vec<Finding>
- Compiles without CLI feature for embedding in other crates
195 tests (130 new), all passing.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gabor Körber
@@ -0,0 +1,307 @@
|
||||
//! Integration tests for `secrets::manifest` — the manifest loader that
|
||||
//! discovers secret files, parses them, and builds the known-secrets set.
|
||||
|
||||
use std::fs;
|
||||
|
||||
use dirigent_fermata::core::secrets::config::SecretsConfig;
|
||||
use dirigent_fermata::core::secrets::manifest::Manifest;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Create a minimal config that only discovers `.env*` files and matches
|
||||
/// common secret key patterns (the defaults).
|
||||
fn default_config() -> SecretsConfig {
|
||||
SecretsConfig::default()
|
||||
}
|
||||
|
||||
/// Create a config from TOML.
|
||||
fn config_from_toml(toml: &str) -> SecretsConfig {
|
||||
SecretsConfig::from_toml(toml).expect("valid TOML config")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn discovers_env_file_and_extracts_matching_secrets() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
fs::write(
|
||||
dir.path().join(".env"),
|
||||
"DATABASE_URL=postgres://localhost/db\nAPP_NAME=myapp\nSECRET_KEY=super-secret-value-1234\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
// DATABASE_URL and SECRET_KEY match the default key patterns; APP_NAME does not.
|
||||
assert!(!manifest.is_empty());
|
||||
|
||||
let keys: Vec<&str> = manifest.entries().iter().map(|e| e.key.as_str()).collect();
|
||||
assert!(keys.contains(&"DATABASE_URL"), "expected DATABASE_URL, got {keys:?}");
|
||||
assert!(keys.contains(&"SECRET_KEY"), "expected SECRET_KEY, got {keys:?}");
|
||||
assert!(!keys.contains(&"APP_NAME"), "APP_NAME should be filtered out");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn discovers_nested_env_local_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let nested = dir.path().join("services").join("auth");
|
||||
fs::create_dir_all(&nested).unwrap();
|
||||
fs::write(
|
||||
nested.join(".env.local"),
|
||||
"AUTH_TOKEN=tok_abcdefgh12345678\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
assert!(!manifest.is_empty());
|
||||
let keys: Vec<&str> = manifest.entries().iter().map(|e| e.key.as_str()).collect();
|
||||
assert!(keys.contains(&"AUTH_TOKEN"), "expected AUTH_TOKEN, got {keys:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filters_entries_by_key_patterns() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
fs::write(
|
||||
dir.path().join(".env"),
|
||||
"MY_PASSWORD=hunter2hunter2\nNOT_SENSITIVE=hello-world-1234\nAPI_KEY=abcdef1234567890\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
let keys: Vec<&str> = manifest.entries().iter().map(|e| e.key.as_str()).collect();
|
||||
assert!(keys.contains(&"MY_PASSWORD"));
|
||||
assert!(keys.contains(&"API_KEY"));
|
||||
assert!(!keys.contains(&"NOT_SENSITIVE"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_override_with_explicit_format_and_key_filter() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
// Write a file that wouldn't normally be discovered by default patterns.
|
||||
fs::write(
|
||||
dir.path().join("custom_secrets.conf"),
|
||||
"SERVICE_TOKEN=long-token-value-here\nDEBUG=true-ish-thing\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = config_from_toml(
|
||||
r#"
|
||||
[files]
|
||||
patterns = []
|
||||
|
||||
[[file]]
|
||||
path = "custom_secrets.conf"
|
||||
format = "env"
|
||||
keys = ["SERVICE_TOKEN"]
|
||||
"#,
|
||||
);
|
||||
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
assert_eq!(manifest.len(), 1);
|
||||
assert_eq!(manifest.entries()[0].key, "SERVICE_TOKEN");
|
||||
assert_eq!(manifest.entries()[0].value, "long-token-value-here");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_project_yields_empty_manifest() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
// No files at all.
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
assert!(manifest.is_empty());
|
||||
assert_eq!(manifest.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entries_sorted_by_value_length_descending() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
fs::write(
|
||||
dir.path().join(".env"),
|
||||
// Deliberately out of order by length.
|
||||
"TOKEN_A=short1234\nTOKEN_B=a-much-longer-secret-value-here\nTOKEN_C=medium-value1\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
let lengths: Vec<usize> = manifest.entries().iter().map(|e| e.value.len()).collect();
|
||||
for window in lengths.windows(2) {
|
||||
assert!(
|
||||
window[0] >= window[1],
|
||||
"entries not sorted by value length descending: {lengths:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_values_filtered_out() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
fs::write(
|
||||
dir.path().join(".env"),
|
||||
"PASSWORD_TINY=yes\nPASSWORD_OK=long-enough-password\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
let keys: Vec<&str> = manifest.entries().iter().map(|e| e.key.as_str()).collect();
|
||||
// "yes" is 3 chars, below the 4-char minimum.
|
||||
assert!(!keys.contains(&"PASSWORD_TINY"), "short value should be filtered");
|
||||
assert!(keys.contains(&"PASSWORD_OK"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deduplication_of_same_key_value() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// Same secret appears in two different .env files.
|
||||
fs::write(
|
||||
dir.path().join(".env"),
|
||||
"SECRET_KEY=shared-secret-value-12345\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let sub = dir.path().join("sub");
|
||||
fs::create_dir(&sub).unwrap();
|
||||
fs::write(sub.join(".env"), "SECRET_KEY=shared-secret-value-12345\n").unwrap();
|
||||
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
// Should be deduplicated to a single entry.
|
||||
let matching: Vec<_> = manifest
|
||||
.entries()
|
||||
.iter()
|
||||
.filter(|e| e.key == "SECRET_KEY")
|
||||
.collect();
|
||||
assert_eq!(
|
||||
matching.len(),
|
||||
1,
|
||||
"duplicate entries should be collapsed: found {}",
|
||||
matching.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unparseable_file_with_allow_is_skipped() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
// Write a file that looks like an env file but contains garbage TOML.
|
||||
// Actually, .env parser is lenient, so let's use a .toml extension
|
||||
// with invalid TOML content to trigger a parse error.
|
||||
let secrets_dir = dir.path();
|
||||
fs::write(secrets_dir.join("secrets.toml"), "this is not valid toml {{{\n").unwrap();
|
||||
|
||||
// Also write a valid .env so we can confirm it still works.
|
||||
fs::write(
|
||||
secrets_dir.join(".env"),
|
||||
"API_KEY=valid-secret-12345678\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = config_from_toml(
|
||||
r#"
|
||||
[enforcement]
|
||||
on_parse_error = "allow"
|
||||
"#,
|
||||
);
|
||||
|
||||
let manifest = Manifest::build(&config, secrets_dir).unwrap();
|
||||
|
||||
// The broken secrets.toml is skipped; .env is still processed.
|
||||
let keys: Vec<&str> = manifest.entries().iter().map(|e| e.key.as_str()).collect();
|
||||
assert!(keys.contains(&"API_KEY"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unparseable_file_with_deny_returns_error() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
fs::write(dir.path().join("secrets.toml"), "not valid toml {{{\n").unwrap();
|
||||
|
||||
let config = config_from_toml(
|
||||
r#"
|
||||
[enforcement]
|
||||
on_parse_error = "deny"
|
||||
"#,
|
||||
);
|
||||
|
||||
let result = Manifest::build(&config, dir.path());
|
||||
assert!(result.is_err(), "deny mode should propagate parse errors");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manifest_empty_and_is_empty() {
|
||||
let m = Manifest::empty();
|
||||
assert!(m.is_empty());
|
||||
assert_eq!(m.len(), 0);
|
||||
assert!(m.entries().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_git_and_node_modules_directories() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// .env inside .git should be skipped.
|
||||
let git_dir = dir.path().join(".git");
|
||||
fs::create_dir(&git_dir).unwrap();
|
||||
fs::write(git_dir.join(".env"), "SECRET_KEY=git-secret-12345\n").unwrap();
|
||||
|
||||
// .env inside node_modules should be skipped.
|
||||
let nm_dir = dir.path().join("node_modules").join("pkg");
|
||||
fs::create_dir_all(&nm_dir).unwrap();
|
||||
fs::write(nm_dir.join(".env"), "TOKEN=nm-token-12345678\n").unwrap();
|
||||
|
||||
// .env at root should be found.
|
||||
fs::write(
|
||||
dir.path().join(".env"),
|
||||
"API_KEY=root-api-key-12345\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
let values: Vec<&str> = manifest.entries().iter().map(|e| e.value.as_str()).collect();
|
||||
assert!(
|
||||
values.contains(&"root-api-key-12345"),
|
||||
"root .env should be found"
|
||||
);
|
||||
assert!(
|
||||
!values.contains(&"git-secret-12345"),
|
||||
".git/.env should be skipped"
|
||||
);
|
||||
assert!(
|
||||
!values.contains(&"nm-token-12345678"),
|
||||
"node_modules/.env should be skipped"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_file_formats_are_skipped_gracefully() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
// .pem and .key files match default patterns but have no parseable format.
|
||||
fs::write(dir.path().join("server.key"), "binary-ish key data here\n").unwrap();
|
||||
fs::write(
|
||||
dir.path().join(".env"),
|
||||
"PASSWORD=parseable-secret-12345\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let config = default_config();
|
||||
let manifest = Manifest::build(&config, dir.path()).unwrap();
|
||||
|
||||
// Should not error, should still find the .env entry.
|
||||
let keys: Vec<&str> = manifest.entries().iter().map(|e| e.key.as_str()).collect();
|
||||
assert!(keys.contains(&"PASSWORD"));
|
||||
}
|
||||
Reference in New Issue
Block a user