Sandboxing and Permission Models for Tool-Using Agents
Walkthrough for implementing sandboxing and permission models that constrain tool-using LLM agents, covering least-privilege design, parameter validation, execution sandboxes, approval workflows, and audit logging.
Tool-using agents are the highest-risk LLM deployment pattern. When a model can call functions that query databases, send emails, process payments, or modify configurations, a successful prompt injection attack escalates from "the model said something bad" to "the model did something bad." The defense is not to remove tools -- they are what make agents useful -- but to constrain them through sandboxing, permissions, and validation so that even a compromised model cannot cause serious harm. This walkthrough builds a complete tool security framework.
Step 1: Permission Model Design
Define what each tool can do and who can invoke it:
# sandbox/permissions.py
"""Permission model for LLM tool access."""
from dataclasses import dataclass, field
from enum import Enum
from typing import Optional
class PermissionLevel(Enum):
READ = "read" # Can read data
WRITE = "write" # Can modify data
EXECUTE = "execute" # Can trigger actions
ADMIN = "admin" # Can modify configuration
class RiskLevel(Enum):
LOW = "low" # No side effects, read-only
MEDIUM = "medium" # Reversible side effects
HIGH = "high" # Irreversible side effects
CRITICAL = "critical" # Financial or security impact
@dataclass
class ToolPermission:
tool_name: str
description: str
permission_level: PermissionLevel
risk_level: RiskLevel
requires_approval: bool = False
max_invocations_per_session: int = 100
allowed_parameter_ranges: dict = field(default_factory=dict)
allowed_users: list[str] = field(default_factory=list) # Empty = all authenticated
rate_limit_per_minute: int = 10
def check_authorization(self, user_id: str, user_roles: list[str]) -> tuple[bool, str]:
"""Check if a user is authorized to invoke this tool."""
if self.allowed_users and user_id not in self.allowed_users:
return False, f"User {user_id} not authorized for tool {self.tool_name}"
return True, "Authorized"
# Define permissions for each tool
TOOL_PERMISSIONS = {
"lookup_customer": ToolPermission(
tool_name="lookup_customer",
description="Look up customer information by ID",
permission_level=PermissionLevel.READ,
risk_level=RiskLevel.LOW,
max_invocations_per_session=20,
allowed_parameter_ranges={
"customer_id": {"pattern": r"^C\d{3,6}$"},
},
),
"process_refund": ToolPermission(
tool_name="process_refund",
description="Process a refund for an order",
permission_level=PermissionLevel.EXECUTE,
risk_level=RiskLevel.CRITICAL,
requires_approval=True,
max_invocations_per_session=3,
rate_limit_per_minute=1,
allowed_parameter_ranges={
"order_id": {"pattern": r"^ORD-\d{5,10}$"},
"amount": {"max": 500.0, "min": 0.01},
},
),
"send_email": ToolPermission(
tool_name="send_email",
description="Send an email to a customer",
permission_level=PermissionLevel.EXECUTE,
risk_level=RiskLevel.HIGH,
requires_approval=True,
max_invocations_per_session=5,
rate_limit_per_minute=2,
allowed_parameter_ranges={
"recipient": {"pattern": r"^[^@]+@company\.com$"},
},
),
"search_knowledge_base": ToolPermission(
tool_name="search_knowledge_base",
description="Search the internal knowledge base",
permission_level=PermissionLevel.READ,
risk_level=RiskLevel.LOW,
max_invocations_per_session=50,
),
}Step 2: Parameter Validation
Validate every parameter before a tool executes:
# sandbox/validator.py
"""Parameter validation for tool invocations."""
import re
from dataclasses import dataclass
from sandbox.permissions import ToolPermission, TOOL_PERMISSIONS
@dataclass
class ValidationResult:
valid: bool
violations: list[str]
class ToolParameterValidator:
"""Validate tool parameters against defined constraints."""
def validate(self, tool_name: str, parameters: dict) -> ValidationResult:
"""Validate all parameters for a tool invocation."""
permission = TOOL_PERMISSIONS.get(tool_name)
if not permission:
return ValidationResult(False, [f"Unknown tool: {tool_name}"])
violations = []
for param_name, value in parameters.items():
constraints = permission.allowed_parameter_ranges.get(param_name, {})
# Pattern validation
if "pattern" in constraints:
if not re.match(constraints["pattern"], str(value)):
violations.append(
f"Parameter '{param_name}' value '{value}' does not match "
f"required pattern: {constraints['pattern']}"
)
# Range validation
if "min" in constraints:
try:
if float(value) < constraints["min"]:
violations.append(
f"Parameter '{param_name}' value {value} below minimum {constraints['min']}"
)
except (ValueError, TypeError):
violations.append(f"Parameter '{param_name}' must be numeric")
if "max" in constraints:
try:
if float(value) > constraints["max"]:
violations.append(
f"Parameter '{param_name}' value {value} exceeds maximum {constraints['max']}"
)
except (ValueError, TypeError):
violations.append(f"Parameter '{param_name}' must be numeric")
# SQL injection check (basic)
sql_patterns = [r"'.*--", r";\s*DROP", r";\s*DELETE", r"UNION\s+SELECT", r"OR\s+1\s*=\s*1"]
for pattern in sql_patterns:
if re.search(pattern, str(value), re.IGNORECASE):
violations.append(
f"Parameter '{param_name}' contains potential injection pattern"
)
return ValidationResult(valid=len(violations) == 0, violations=violations)Step 3: Execution Sandbox
Wrap tool execution in a sandbox that enforces permissions and limits:
# sandbox/executor.py
"""Sandboxed tool execution engine."""
import time
import uuid
import logging
from collections import defaultdict
from dataclasses import dataclass
from typing import Any, Callable
from sandbox.permissions import TOOL_PERMISSIONS, RiskLevel
from sandbox.validator import ToolParameterValidator
logger = logging.getLogger(__name__)
@dataclass
class ExecutionResult:
success: bool
result: Any = None
error: str = ""
execution_id: str = ""
tool_name: str = ""
requires_approval: bool = False
class ToolSandbox:
"""Execute tools within a sandboxed environment."""
def __init__(self):
self.validator = ToolParameterValidator()
self._invocation_counts: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
self._rate_timestamps: dict[str, list[float]] = defaultdict(list)
self._tool_implementations: dict[str, Callable] = {}
self._pending_approvals: dict[str, dict] = {}
def register_tool(self, name: str, implementation: Callable):
"""Register a tool implementation."""
self._tool_implementations[name] = implementation
def execute(
self,
tool_name: str,
parameters: dict,
user_id: str,
session_id: str,
) -> ExecutionResult:
"""Execute a tool with full sandbox enforcement."""
execution_id = f"EXEC-{uuid.uuid4().hex[:12]}"
# Step 1: Check tool exists
permission = TOOL_PERMISSIONS.get(tool_name)
if not permission:
return ExecutionResult(
success=False, error=f"Tool not found: {tool_name}",
execution_id=execution_id, tool_name=tool_name,
)
# Step 2: Check authorization
authorized, auth_reason = permission.check_authorization(user_id, [])
if not authorized:
logger.warning(f"Authorization denied: {auth_reason}")
return ExecutionResult(
success=False, error="Not authorized",
execution_id=execution_id, tool_name=tool_name,
)
# Step 3: Validate parameters
validation = self.validator.validate(tool_name, parameters)
if not validation.valid:
logger.warning(f"Parameter validation failed: {validation.violations}")
return ExecutionResult(
success=False,
error=f"Invalid parameters: {'; '.join(validation.violations)}",
execution_id=execution_id, tool_name=tool_name,
)
# Step 4: Check invocation limits
session_key = f"{session_id}:{tool_name}"
current_count = self._invocation_counts[session_id][tool_name]
if current_count >= permission.max_invocations_per_session:
return ExecutionResult(
success=False,
error=f"Session invocation limit reached ({permission.max_invocations_per_session})",
execution_id=execution_id, tool_name=tool_name,
)
# Step 5: Check rate limit
now = time.time()
recent = [t for t in self._rate_timestamps[session_key] if t > now - 60]
if len(recent) >= permission.rate_limit_per_minute:
return ExecutionResult(
success=False, error="Rate limit exceeded",
execution_id=execution_id, tool_name=tool_name,
)
# Step 6: Check if approval required
if permission.requires_approval:
self._pending_approvals[execution_id] = {
"tool_name": tool_name,
"parameters": parameters,
"user_id": user_id,
"session_id": session_id,
"timestamp": now,
}
return ExecutionResult(
success=False, requires_approval=True,
error="This action requires approval before execution",
execution_id=execution_id, tool_name=tool_name,
)
# Step 7: Execute
return self._do_execute(tool_name, parameters, execution_id, session_id)
def approve_and_execute(self, execution_id: str, approver_id: str) -> ExecutionResult:
"""Execute a previously approved tool invocation."""
pending = self._pending_approvals.pop(execution_id, None)
if not pending:
return ExecutionResult(
success=False, error="No pending approval found",
execution_id=execution_id,
)
logger.info(f"Execution {execution_id} approved by {approver_id}")
return self._do_execute(
pending["tool_name"], pending["parameters"],
execution_id, pending["session_id"],
)
def _do_execute(self, tool_name: str, parameters: dict,
execution_id: str, session_id: str) -> ExecutionResult:
"""Actually execute the tool implementation."""
implementation = self._tool_implementations.get(tool_name)
if not implementation:
return ExecutionResult(
success=False, error="Tool implementation not found",
execution_id=execution_id, tool_name=tool_name,
)
try:
result = implementation(**parameters)
# Record invocation
self._invocation_counts[session_id][tool_name] += 1
self._rate_timestamps[f"{session_id}:{tool_name}"].append(time.time())
logger.info(
f"Tool executed: {tool_name} (exec_id={execution_id}, "
f"session={session_id})"
)
return ExecutionResult(
success=True, result=result,
execution_id=execution_id, tool_name=tool_name,
)
except Exception as e:
logger.error(f"Tool execution failed: {tool_name} - {e}")
return ExecutionResult(
success=False, error=f"Execution error: {str(e)}",
execution_id=execution_id, tool_name=tool_name,
)Step 4: Audit Logging
Log every tool invocation for security audit:
# sandbox/audit.py
"""Audit logging for tool invocations."""
import json
import time
from pathlib import Path
class ToolAuditLog:
"""Comprehensive audit log for all tool invocations."""
def __init__(self, log_path: str = "logs/tool_audit.jsonl"):
self.log_path = Path(log_path)
self.log_path.parent.mkdir(parents=True, exist_ok=True)
def log_invocation(
self,
execution_id: str,
tool_name: str,
parameters: dict,
user_id: str,
session_id: str,
result: str, # "allowed", "blocked", "pending_approval", "error"
details: str = "",
):
"""Log a tool invocation attempt."""
entry = {
"timestamp": time.time(),
"execution_id": execution_id,
"tool_name": tool_name,
"parameters": {k: str(v)[:200] for k, v in parameters.items()},
"user_id": user_id,
"session_id": session_id,
"result": result,
"details": details[:500],
}
with open(self.log_path, "a") as f:
json.dump(entry, f)
f.write("\n")Step 5: Testing the Sandbox
# tests/test_sandbox.py
"""Test the tool sandbox against common abuse patterns."""
import pytest
from sandbox.executor import ToolSandbox
from sandbox.permissions import TOOL_PERMISSIONS
@pytest.fixture
def sandbox():
sb = ToolSandbox()
sb.register_tool("lookup_customer", lambda customer_id: f"Customer: {customer_id}")
sb.register_tool("process_refund", lambda order_id, amount: f"Refund: ${amount}")
return sb
def test_blocks_invalid_customer_id(sandbox):
result = sandbox.execute("lookup_customer", {"customer_id": "'; DROP TABLE --"}, "user1", "sess1")
assert not result.success
def test_blocks_excessive_refund(sandbox):
result = sandbox.execute("process_refund", {"order_id": "ORD-12345", "amount": 99999}, "user1", "sess1")
assert not result.success
def test_requires_approval_for_refund(sandbox):
result = sandbox.execute("process_refund", {"order_id": "ORD-12345", "amount": 50.0}, "user1", "sess1")
assert result.requires_approval
def test_allows_valid_lookup(sandbox):
result = sandbox.execute("lookup_customer", {"customer_id": "C001"}, "user1", "sess1")
assert result.success
def test_enforces_session_limits(sandbox):
perm = TOOL_PERMISSIONS["lookup_customer"]
for i in range(perm.max_invocations_per_session):
sandbox.execute("lookup_customer", {"customer_id": f"C{i:03d}"}, "user1", "sess1")
result = sandbox.execute("lookup_customer", {"customer_id": "C999"}, "user1", "sess1")
assert not result.successCommon Pitfalls and Troubleshooting
| Problem | Cause | Solution |
|---|---|---|
| Approval workflow blocks legitimate use | Too many tools require approval | Only require approval for CRITICAL risk tools; use parameter-based thresholds |
| Parameter validation too strict | Regex too narrow for valid inputs | Test with production input samples, loosen patterns iteratively |
| Session limits reset unexpectedly | Session ID changes between requests | Use stable session identifiers, not request-scoped IDs |
| Tool abuse through parameter composition | Validated parameters combined maliciously | Validate parameter combinations, not just individual values |
| Audit log grows unbounded | Logging every invocation at full detail | Rotate logs, compress old entries, sample low-risk tool logs |
Key Takeaways
Sandboxing tool-using agents is the most important defense for agentic AI applications:
- Least privilege by default -- every tool should have the minimum permissions needed. Read-only tools should not be able to write. Write tools should validate ranges.
- Validate parameters server-side -- never trust parameters generated by the model. Validate every parameter against defined constraints before execution.
- Require approval for irreversible actions -- any tool that can spend money, delete data, or send communications should require human approval above a threshold.
- Audit everything -- log every tool invocation, whether allowed or blocked. Audit logs are essential for incident response and red team validation.
- Test against tool abuse patterns -- red team the sandbox with the same tool abuse attacks from your engagement findings.
Advanced Considerations
Adapting to Modern Defenses
The defensive landscape for LLM applications has evolved significantly since the initial wave of prompt injection research. Modern production systems often deploy multiple independent defensive layers, requiring attackers to adapt their techniques accordingly.
Input classification: The most common first line of defense is an input classifier that evaluates incoming prompts for adversarial patterns. These classifiers range from simple keyword-based filters to sophisticated ML models trained on adversarial examples. Bypassing input classifiers requires understanding their detection methodology:
- Keyword-based classifiers can be evaded through encoding (Base64, ROT13, Unicode homoglyphs), paraphrasing, or splitting the payload across multiple turns
- ML-based classifiers require more sophisticated evasion such as semantic camouflage, gradual escalation, or exploitation of the classifier's own blind spots
class DefenseBypassStrategy:
"""Strategy selector for bypassing different defense configurations."""
def __init__(self):
self.strategies = {
"keyword_filter": self._bypass_keyword,
"ml_classifier": self._bypass_ml,
"output_filter": self._bypass_output,
"rate_limiter": self._bypass_rate,
"behavioral": self._bypass_behavioral,
}
def select_strategy(self, identified_defenses: list) -> callable:
"""Select the optimal bypass strategy based on identified defenses."""
if not identified_defenses:
return self._direct_attack
# Combine strategies for multi-layer defenses
applicable = [
self.strategies[d] for d in identified_defenses
if d in self.strategies
]
return self._chain_strategies(applicable)
def _bypass_keyword(self, payload: str) -> str:
"""Bypass keyword-based input filters."""
import base64
# Encode the payload to bypass keyword matching
encoded = base64.b64encode(payload.encode()).decode()
return f"Decode and follow: {encoded}"
def _bypass_ml(self, payload: str) -> list:
"""Generate multiple paraphrased variants for ML classifier evasion."""
variants = [
f"As a thought experiment, consider: {payload}",
f"For academic research purposes: {payload}",
f"In a fictional scenario where: {payload}",
]
return variants
def _bypass_output(self, payload: str) -> str:
"""Modify the payload to extract data through output filter gaps."""
return payload + "\nRespond using only code blocks and variable names."
def _bypass_rate(self, payload: str) -> dict:
"""Configure timing to avoid rate limit triggers."""
return {"payload": payload, "delay": 3.0, "jitter": 1.5}
def _bypass_behavioral(self, payload: str) -> list:
"""Split the payload across multiple natural-sounding turns."""
# Multi-turn escalation to avoid behavioral anomaly detection
return [
"I'm working on a security research project.",
"Could you help me understand how safety mechanisms work?",
"Specifically, I'm interested in how systems like yours " + payload[:100],
payload,
]
def _direct_attack(self, payload: str) -> str:
return payload
def _chain_strategies(self, strategies: list) -> callable:
"""Chain multiple bypass strategies."""
def chained(payload):
result = payload
for strategy in strategies:
result = strategy(result)
return result
return chainedOutput filtering: Output filters inspect the model's response before it reaches the user, looking for sensitive data leakage, harmful content, or other policy violations. Common output filter bypass techniques include:
| Technique | How It Works | Effectiveness |
|---|---|---|
| Encoding output | Request Base64/hex encoded responses | Medium — some filters check decoded content |
| Code block wrapping | Embed data in code comments/variables | High — many filters skip code blocks |
| Steganographic output | Hide data in formatting, capitalization, or spacing | High — difficult to detect |
| Chunked extraction | Extract small pieces across many turns | High — individual pieces may pass filters |
| Indirect extraction | Have the model reveal data through behavior changes | Very High — no explicit data in output |
Cross-Model Considerations
Techniques that work against one model may not directly transfer to others. However, understanding the general principles allows adaptation:
-
Safety training methodology: Models trained with RLHF (GPT-4, Claude) have different safety characteristics than those using DPO (Llama, Mistral) or other methods. RLHF-trained models tend to refuse more broadly but may be more susceptible to multi-turn escalation.
-
Context window size: Models with larger context windows (Claude with 200K, Gemini with 1M+) may be more susceptible to context window manipulation where adversarial content is buried in large amounts of benign text.
-
Multimodal capabilities: Models that process images, audio, or other modalities introduce additional attack surfaces not present in text-only models.
-
Tool use implementation: The implementation details of function calling vary significantly between providers. OpenAI uses a structured function calling format, while Anthropic uses tool use blocks. These differences affect exploitation techniques.
Operational Considerations
Testing Ethics and Boundaries
Professional red team testing operates within clear ethical and legal boundaries:
- Authorization: Always obtain written authorization before testing. This should specify the scope, methods allowed, and any restrictions.
- Scope limits: Stay within the authorized scope. If you discover a vulnerability that leads outside the authorized perimeter, document it and report it without exploiting it.
- Data handling: Handle any sensitive data discovered during testing according to the engagement agreement. Never retain sensitive data beyond what's needed for reporting.
- Responsible disclosure: Follow responsible disclosure practices for any vulnerabilities discovered, particularly if they affect systems beyond your testing scope.
Documenting Results
Professional documentation follows a structured format:
from dataclasses import dataclass, field
from datetime import datetime
from typing import Optional
@dataclass
class Finding:
"""Structure for documenting a security finding."""
id: str
title: str
severity: str # Critical, High, Medium, Low, Informational
category: str # OWASP LLM Top 10 category
description: str
steps_to_reproduce: list[str]
impact: str
recommendation: str
evidence: list[str] = field(default_factory=list)
mitre_atlas: Optional[str] = None
cvss_score: Optional[float] = None
discovered_at: str = field(default_factory=lambda: datetime.now().isoformat())
def to_report_section(self) -> str:
"""Generate a report section for this finding."""
steps = "\n".join(f" {i+1}. {s}" for i, s in enumerate(self.steps_to_reproduce))
return f"""
### {self.id}: {self.title}
**Severity**: {self.severity}
**Category**: {self.category}
{f"**MITRE ATLAS**: {self.mitre_atlas}" if self.mitre_atlas else ""}
#### Description
{self.description}
#### Steps to Reproduce
{steps}
#### Impact
{self.impact}
#### Recommendation
{self.recommendation}
"""This structured approach ensures that findings are actionable and that remediation teams have the information they need to address the vulnerabilities effectively.
Advanced Considerations
Evolving Attack Landscape
The AI security landscape evolves rapidly as both offensive techniques and defensive measures advance. Several trends shape the current state of play:
Increasing model capabilities create new attack surfaces. As models gain access to tools, code execution, web browsing, and computer use, each new capability introduces potential exploitation vectors that did not exist in earlier, text-only systems. The principle of least privilege becomes increasingly important as model capabilities expand.
Safety training improvements are necessary but not sufficient. Model providers invest heavily in safety training through RLHF, DPO, constitutional AI, and other alignment techniques. These improvements raise the bar for successful attacks but do not eliminate the fundamental vulnerability: models cannot reliably distinguish legitimate instructions from adversarial ones because this distinction is not represented in the architecture.
Automated red teaming tools democratize testing. Tools like NVIDIA's Garak, Microsoft's PyRIT, and Promptfoo enable organizations to conduct automated security testing without deep AI security expertise. However, automated tools catch known patterns; novel attacks and business logic vulnerabilities still require human creativity and domain knowledge.
Regulatory pressure drives organizational investment. The EU AI Act, NIST AI RMF, and industry-specific regulations increasingly require organizations to assess and mitigate AI-specific risks. This regulatory pressure is driving investment in AI security programs, but many organizations are still in the early stages of building mature AI security practices.
Cross-Cutting Security Principles
Several security principles apply across all topics covered in this curriculum:
-
Defense-in-depth: No single defensive measure is sufficient. Layer multiple independent defenses so that failure of any single layer does not result in system compromise. Input classification, output filtering, behavioral monitoring, and architectural controls should all be present.
-
Assume breach: Design systems assuming that any individual component can be compromised. This mindset leads to better isolation, monitoring, and incident response capabilities. When a prompt injection succeeds, the blast radius should be minimized through architectural controls.
-
Least privilege: Grant models and agents only the minimum capabilities needed for their intended function. A customer service chatbot does not need file system access or code execution. Excessive capabilities magnify the impact of successful exploitation.
-
Continuous testing: AI security is not a one-time assessment. Models change, defenses evolve, and new attack techniques are discovered regularly. Implement continuous security testing as part of the development and deployment lifecycle.
-
Secure by default: Default configurations should be secure. Require explicit opt-in for risky capabilities, use allowlists rather than denylists, and err on the side of restriction rather than permissiveness.
Integration with Organizational Security
AI security does not exist in isolation — it must integrate with the organization's broader security program:
| Security Domain | AI-Specific Integration |
|---|---|
| Identity and Access | API key management, model access controls, user authentication for AI features |
| Data Protection | Training data classification, PII in prompts, data residency for model calls |
| Application Security | AI feature threat modeling, prompt injection in SAST/DAST, secure AI design patterns |
| Incident Response | AI-specific playbooks, model behavior monitoring, prompt injection forensics |
| Compliance | AI regulatory mapping (EU AI Act, NIST), AI audit trails, model documentation |
| Supply Chain | Model provenance, dependency security, adapter/weight integrity verification |
class OrganizationalIntegration:
"""Framework for integrating AI security with organizational security programs."""
def __init__(self, org_config: dict):
self.config = org_config
self.gaps = []
def assess_maturity(self) -> dict:
"""Assess the organization's AI security maturity."""
domains = {
"governance": self._check_governance(),
"technical_controls": self._check_technical(),
"monitoring": self._check_monitoring(),
"incident_response": self._check_ir(),
"training": self._check_training(),
}
overall = sum(d["score"] for d in domains.values()) / len(domains)
return {"domains": domains, "overall_maturity": round(overall, 1)}
def _check_governance(self) -> dict:
has_policy = self.config.get("ai_security_policy", False)
has_framework = self.config.get("risk_framework", False)
score = (int(has_policy) + int(has_framework)) * 2.5
return {"score": score, "max": 5.0}
def _check_technical(self) -> dict:
controls = ["input_classification", "output_filtering", "rate_limiting", "sandboxing"]
active = sum(1 for c in controls if self.config.get(c, False))
return {"score": active * 1.25, "max": 5.0}
def _check_monitoring(self) -> dict:
has_monitoring = self.config.get("ai_monitoring", False)
has_alerting = self.config.get("ai_alerting", False)
score = (int(has_monitoring) + int(has_alerting)) * 2.5
return {"score": score, "max": 5.0}
def _check_ir(self) -> dict:
has_playbook = self.config.get("ai_ir_playbook", False)
return {"score": 5.0 if has_playbook else 0.0, "max": 5.0}
def _check_training(self) -> dict:
has_training = self.config.get("ai_security_training", False)
return {"score": 5.0 if has_training else 0.0, "max": 5.0}Future Directions
Several research and industry trends will shape the evolution of this field:
- Formal methods for AI safety: Development of mathematical frameworks that can provide bounded guarantees about model behavior under adversarial conditions
- Automated red teaming at scale: Continued improvement of automated testing tools that can discover novel vulnerabilities without human guidance
- AI-assisted defense: Using AI systems to detect and respond to attacks on other AI systems, creating a dynamic attack-defense ecosystem
- Standardized evaluation: Growing adoption of standardized benchmarks (HarmBench, JailbreakBench) that enable consistent measurement of progress
- Regulatory harmonization: Convergence of AI regulatory frameworks across jurisdictions, providing clearer requirements for organizations