✨ feat(fermata): add secret filtering engine — the security brain
Implement Goals 1–3 and 5 from the reveal-layer security brain goal.
fermata now detects, redacts, and scans for secrets in AI agent tool
output, filling the ecosystem gap where no coding agent filters secrets
post-read.
New core/secrets/ module:
- config.rs: .botsecrets TOML format with hierarchical merge and ~40
built-in key patterns
- parser.rs: multi-format secret file parser (.env, TOML, YAML, JSON,
Python assignments, Java properties)
- manifest.rs: file discovery + parsing → known-secrets set
- redactor.rs: Aho-Corasick multi-pattern replacement with 4 styles
- scanner.rs: RegexSet heuristic detection with 35 gitleaks-derived
patterns (MIT) and Shannon entropy filtering
- patterns.rs: curated rules for AWS, GitHub, Stripe, Slack, JWT, etc.
Hook integration:
- fermata hook --event post-tool-use reads tool output, runs redactor +
scanner, returns updatedToolOutput for Claude Code
- Backward compatible: --event pre-tool-use (default) unchanged
- Fail-open: errors produce {} and exit 0
Library API:
- Redactor::new(manifest, style).redact(text) → RedactedText
- Scanner::new(config).scan(text) → Vec<Finding>
- Compiles without CLI feature for embedding in other crates
195 tests (130 new), all passing.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Gabor Körber
@@ -0,0 +1,388 @@
|
||||
use dirigent_fermata::core::secrets::config::{
|
||||
EnforcementMode, HeuristicMode, ParseErrorAction, RedactionStyle, SecretsConfig,
|
||||
BUILTIN_KEY_PATTERNS,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn parse_minimal_files_only() {
|
||||
let cfg = SecretsConfig::from_toml(
|
||||
r#"
|
||||
[files]
|
||||
patterns = [".env", ".env.*"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(cfg.files.patterns, vec![".env", ".env.*"]);
|
||||
// Other sections use defaults
|
||||
assert_eq!(cfg.redaction.style, RedactionStyle::Masked);
|
||||
assert_eq!(cfg.enforcement.mode, EnforcementMode::Permissive);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_full_config() {
|
||||
let cfg = SecretsConfig::from_toml(
|
||||
r#"
|
||||
[files]
|
||||
patterns = [".env", "secrets.*"]
|
||||
|
||||
[keys]
|
||||
include = ["STRIPE_*", "TWILIO_*"]
|
||||
exclude = ["PUBLIC_KEY", "SSH_KEY_PATH"]
|
||||
|
||||
[redaction]
|
||||
style = "typed"
|
||||
|
||||
[heuristic]
|
||||
enabled = false
|
||||
mode = "report"
|
||||
patterns = ['AKIA[A-Z2-7]{16}']
|
||||
|
||||
[enforcement]
|
||||
mode = "strict"
|
||||
on_parse_error = "deny"
|
||||
|
||||
[[file]]
|
||||
path = "settings.py"
|
||||
format = "python-assignments"
|
||||
keys = ["SECRET_KEY", "DATABASES.*.PASSWORD"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(cfg.files.patterns, vec![".env", "secrets.*"]);
|
||||
assert_eq!(cfg.keys.include, vec!["STRIPE_*", "TWILIO_*"]);
|
||||
assert_eq!(cfg.keys.exclude, vec!["PUBLIC_KEY", "SSH_KEY_PATH"]);
|
||||
assert_eq!(cfg.redaction.style, RedactionStyle::Typed);
|
||||
assert!(!cfg.heuristic.enabled);
|
||||
assert_eq!(cfg.heuristic.mode, HeuristicMode::Report);
|
||||
assert_eq!(cfg.heuristic.patterns, vec!["AKIA[A-Z2-7]{16}"]);
|
||||
assert_eq!(cfg.enforcement.mode, EnforcementMode::Strict);
|
||||
assert_eq!(cfg.enforcement.on_parse_error, ParseErrorAction::Deny);
|
||||
assert_eq!(cfg.file_overrides.len(), 1);
|
||||
assert_eq!(cfg.file_overrides[0].path, "settings.py");
|
||||
assert_eq!(
|
||||
cfg.file_overrides[0].format.as_deref(),
|
||||
Some("python-assignments")
|
||||
);
|
||||
assert_eq!(
|
||||
cfg.file_overrides[0].keys,
|
||||
vec!["SECRET_KEY", "DATABASES.*.PASSWORD"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_toml_returns_defaults() {
|
||||
let cfg = SecretsConfig::from_toml("").unwrap();
|
||||
assert!(!cfg.files.patterns.is_empty());
|
||||
assert!(cfg.files.patterns.contains(&".env".to_string()));
|
||||
assert_eq!(cfg.redaction.style, RedactionStyle::Masked);
|
||||
assert!(cfg.heuristic.enabled);
|
||||
assert_eq!(cfg.heuristic.mode, HeuristicMode::Enforce);
|
||||
assert_eq!(cfg.enforcement.mode, EnforcementMode::Permissive);
|
||||
assert_eq!(
|
||||
cfg.enforcement.on_parse_error,
|
||||
ParseErrorAction::MaskEntireFile
|
||||
);
|
||||
assert!(cfg.file_overrides.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_toml_produces_error() {
|
||||
let result = SecretsConfig::from_toml("this is not valid {{ toml");
|
||||
assert!(result.is_err());
|
||||
let err_msg = result.unwrap_err().to_string();
|
||||
assert!(
|
||||
err_msg.contains("expected"),
|
||||
"error should describe parse issue: {err_msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_key_includes_has_builtins() {
|
||||
let cfg = SecretsConfig::default();
|
||||
let effective = cfg.effective_key_includes();
|
||||
for builtin in BUILTIN_KEY_PATTERNS {
|
||||
assert!(
|
||||
effective.contains(&builtin.to_string()),
|
||||
"missing builtin: {builtin}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_key_includes_adds_user_patterns() {
|
||||
let cfg = SecretsConfig::from_toml(
|
||||
r#"
|
||||
[keys]
|
||||
include = ["MY_CUSTOM_SECRET_*"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let effective = cfg.effective_key_includes();
|
||||
assert!(effective.contains(&"MY_CUSTOM_SECRET_*".to_string()));
|
||||
// Builtins still present
|
||||
assert!(effective.contains(&"*PASSWORD*".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_key_includes_removes_excluded() {
|
||||
let cfg = SecretsConfig::from_toml(
|
||||
r#"
|
||||
[keys]
|
||||
exclude = ["*TOKEN*", "SENTRY_DSN"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
let effective = cfg.effective_key_includes();
|
||||
assert!(
|
||||
!effective.contains(&"*TOKEN*".to_string()),
|
||||
"excluded pattern should be removed"
|
||||
);
|
||||
assert!(
|
||||
!effective.contains(&"SENTRY_DSN".to_string()),
|
||||
"excluded pattern should be removed"
|
||||
);
|
||||
// Other builtins still present
|
||||
assert!(effective.contains(&"*PASSWORD*".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_matches_glob_case_insensitive() {
|
||||
let cfg = SecretsConfig::default();
|
||||
assert!(cfg.key_matches("DATABASE_URL"));
|
||||
assert!(cfg.key_matches("database_url"));
|
||||
assert!(cfg.key_matches("my_password_here"));
|
||||
assert!(cfg.key_matches("MY_PASSWORD_HERE"));
|
||||
assert!(cfg.key_matches("STRIPE_SECRET_KEY"));
|
||||
assert!(cfg.key_matches("AWS_ACCESS_KEY_ID"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_matches_non_secret_keys() {
|
||||
let cfg = SecretsConfig::default();
|
||||
assert!(!cfg.key_matches("DEBUG"));
|
||||
assert!(!cfg.key_matches("LOG_LEVEL"));
|
||||
assert!(!cfg.key_matches("PORT"));
|
||||
assert!(!cfg.key_matches("HOST"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_matches_respects_user_include() {
|
||||
let cfg = SecretsConfig::from_toml(
|
||||
r#"
|
||||
[keys]
|
||||
include = ["MY_APP_*"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(cfg.key_matches("MY_APP_SETTING"));
|
||||
assert!(cfg.key_matches("my_app_setting"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn key_matches_respects_user_exclude() {
|
||||
let cfg = SecretsConfig::from_toml(
|
||||
r#"
|
||||
[keys]
|
||||
exclude = ["*TOKEN*"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
// TOKEN patterns were excluded, so GITHUB_TOKEN should no longer match
|
||||
// via the *TOKEN* pattern. But it might match via GITHUB_TOKEN literal.
|
||||
// Let's check something that only matched *TOKEN*.
|
||||
assert!(!cfg.key_matches("MY_TOKEN"));
|
||||
// PASSWORD still matches
|
||||
assert!(cfg.key_matches("MY_PASSWORD"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_file_patterns_present() {
|
||||
let cfg = SecretsConfig::default();
|
||||
let patterns = &cfg.files.patterns;
|
||||
assert!(patterns.contains(&".env".to_string()));
|
||||
assert!(patterns.contains(&"*.pem".to_string()));
|
||||
assert!(patterns.contains(&".aws/credentials".to_string()));
|
||||
assert!(patterns.contains(&"terraform.tfvars".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_missing_files_returns_defaults() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let cfg = SecretsConfig::load(tmp.path()).unwrap();
|
||||
assert_eq!(cfg.files.patterns, SecretsConfig::default().files.patterns);
|
||||
assert_eq!(cfg.redaction.style, RedactionStyle::Masked);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_project_botsecrets() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
std::fs::write(
|
||||
tmp.path().join(".botsecrets"),
|
||||
r#"
|
||||
[redaction]
|
||||
style = "named"
|
||||
|
||||
[keys]
|
||||
include = ["CUSTOM_*"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let cfg = SecretsConfig::load(tmp.path()).unwrap();
|
||||
assert_eq!(cfg.redaction.style, RedactionStyle::Named);
|
||||
assert!(cfg.effective_key_includes().contains(&"CUSTOM_*".to_string()));
|
||||
// File patterns remain at defaults (not overridden)
|
||||
assert!(cfg.files.patterns.contains(&".env".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_local_overrides_project() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
std::fs::write(
|
||||
tmp.path().join(".botsecrets"),
|
||||
r#"
|
||||
[redaction]
|
||||
style = "named"
|
||||
[enforcement]
|
||||
mode = "strict"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
tmp.path().join(".botsecrets.local"),
|
||||
r#"
|
||||
[redaction]
|
||||
style = "absent"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let cfg = SecretsConfig::load(tmp.path()).unwrap();
|
||||
// .local overrides .botsecrets for redaction style
|
||||
assert_eq!(cfg.redaction.style, RedactionStyle::Absent);
|
||||
// enforcement from .botsecrets is preserved (not in .local)
|
||||
assert_eq!(cfg.enforcement.mode, EnforcementMode::Strict);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_invalid_botsecrets_returns_error() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
std::fs::write(tmp.path().join(".botsecrets"), "invalid {{ toml").unwrap();
|
||||
let result = SecretsConfig::load(tmp.path());
|
||||
assert!(result.is_err());
|
||||
let err = result.unwrap_err().to_string();
|
||||
assert!(err.contains(".botsecrets"), "error should mention file: {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_keys_accumulate() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
std::fs::write(
|
||||
tmp.path().join(".botsecrets"),
|
||||
r#"
|
||||
[keys]
|
||||
include = ["FROM_PROJECT"]
|
||||
exclude = ["EXCLUDE_PROJECT"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
tmp.path().join(".botsecrets.local"),
|
||||
r#"
|
||||
[keys]
|
||||
include = ["FROM_LOCAL"]
|
||||
exclude = ["EXCLUDE_LOCAL"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let cfg = SecretsConfig::load(tmp.path()).unwrap();
|
||||
assert!(cfg.keys.include.contains(&"FROM_PROJECT".to_string()));
|
||||
assert!(cfg.keys.include.contains(&"FROM_LOCAL".to_string()));
|
||||
assert!(cfg.keys.exclude.contains(&"EXCLUDE_PROJECT".to_string()));
|
||||
assert!(cfg.keys.exclude.contains(&"EXCLUDE_LOCAL".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_file_patterns_replaced_not_appended() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
std::fs::write(
|
||||
tmp.path().join(".botsecrets"),
|
||||
r#"
|
||||
[files]
|
||||
patterns = ["only-this.env"]
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let cfg = SecretsConfig::load(tmp.path()).unwrap();
|
||||
assert_eq!(cfg.files.patterns, vec!["only-this.env"]);
|
||||
// Defaults should be gone, replaced by the project's list
|
||||
assert!(!cfg.files.patterns.contains(&".env".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_redaction_styles_parse() {
|
||||
for (input, expected) in [
|
||||
("masked", RedactionStyle::Masked),
|
||||
("typed", RedactionStyle::Typed),
|
||||
("named", RedactionStyle::Named),
|
||||
("absent", RedactionStyle::Absent),
|
||||
] {
|
||||
let toml_str = format!("[redaction]\nstyle = \"{input}\"");
|
||||
let cfg = SecretsConfig::from_toml(&toml_str).unwrap();
|
||||
assert_eq!(cfg.redaction.style, expected, "failed for: {input}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_enforcement_modes_parse() {
|
||||
for (input, expected) in [
|
||||
("strict", EnforcementMode::Strict),
|
||||
("permissive", EnforcementMode::Permissive),
|
||||
("audit", EnforcementMode::Audit),
|
||||
] {
|
||||
let toml_str = format!("[enforcement]\nmode = \"{input}\"");
|
||||
let cfg = SecretsConfig::from_toml(&toml_str).unwrap();
|
||||
assert_eq!(cfg.enforcement.mode, expected, "failed for: {input}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_heuristic_modes_parse() {
|
||||
for (input, expected) in [
|
||||
("enforce", HeuristicMode::Enforce),
|
||||
("report", HeuristicMode::Report),
|
||||
("disabled", HeuristicMode::Disabled),
|
||||
] {
|
||||
let toml_str = format!("[heuristic]\nmode = \"{input}\"");
|
||||
let cfg = SecretsConfig::from_toml(&toml_str).unwrap();
|
||||
assert_eq!(cfg.heuristic.mode, expected, "failed for: {input}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serialization_roundtrip() {
|
||||
let cfg = SecretsConfig::from_toml(
|
||||
r#"
|
||||
[files]
|
||||
patterns = [".env"]
|
||||
[redaction]
|
||||
style = "typed"
|
||||
[enforcement]
|
||||
mode = "audit"
|
||||
on_parse_error = "allow"
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let serialized = toml::to_string(&cfg).unwrap();
|
||||
let deserialized: SecretsConfig = toml::from_str(&serialized).unwrap();
|
||||
assert_eq!(deserialized.redaction.style, RedactionStyle::Typed);
|
||||
assert_eq!(deserialized.enforcement.mode, EnforcementMode::Audit);
|
||||
assert_eq!(
|
||||
deserialized.enforcement.on_parse_error,
|
||||
ParseErrorAction::Allow
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user