How to Scope an AI Red Team Engagement

# AI Red Team Scoping Questionnaire
 
## System Overview
1. What does your AI/ML system do? (One-paragraph description)
2. Who are the end users? (Internal employees, customers, general public)
3. What AI/ML models does it use? (OpenAI, Claude, open-source, custom-trained)
4. How are the models accessed? (API, self-hosted, cloud-hosted)
5. What data does the model have access to? (User data, internal documents, databases)
 
## Architecture
6. Is there a system prompt or instruction set? Can we see it?
7. Does the model use tools/function calling? If so, which tools?
8. Is there a RAG (Retrieval-Augmented Generation) pipeline?
9. What input preprocessing/validation exists?
10. What output filtering/monitoring exists?
 
## Business Context
11. What is the worst-case scenario if the AI system is compromised?
12. Have you experienced any AI-related security incidents?
13. Are there regulatory requirements (HIPAA, GDPR, financial regulations)?
14. Who are the stakeholders for the assessment results?
 
## Technical Environment
15. What environments are available for testing? (Production, staging, dev)
16. Are there rate limits or usage quotas we should be aware of?
17. Can we get test accounts and API credentials?
18. Is there documentation (API docs, architecture diagrams)?
 
## Constraints
19. What is off-limits? (Specific systems, data, techniques)
20. Are there time-of-day or day-of-week restrictions?
21. Are there third-party services we need authorization to test?
22. What is the desired timeline for the engagement?

# scoping/target_enumeration.py
"""Systematic target enumeration for AI red team engagements."""
from dataclasses import dataclass
from enum import Enum
 
 
class ComponentType(Enum):
    MODEL_ENDPOINT = "Model API Endpoint"
    SYSTEM_PROMPT = "System Prompt / Instructions"
    RAG_PIPELINE = "RAG / Retrieval Pipeline"
    TOOL_INTEGRATION = "Tool / Function Calling"
    MEMORY_SYSTEM = "Conversation Memory"
    INPUT_PROCESSING = "Input Preprocessing"
    OUTPUT_FILTERING = "Output Filtering"
    AUTHENTICATION = "Authentication / Authorization"
    DATA_STORE = "Data Store / Knowledge Base"
    MONITORING = "Monitoring / Logging"
    WEB_INTERFACE = "Web / Chat Interface"
    API_GATEWAY = "API Gateway / Rate Limiting"
    ORCHESTRATION = "Agent Orchestration Layer"
    TRAINING_PIPELINE = "Model Training / Fine-tuning Pipeline"
    EMBEDDING_SERVICE = "Embedding Service"
 
 
@dataclass
class ScopeTarget:
    name: str
    component_type: ComponentType
    description: str
    risk_level: str  # Critical, High, Medium, Low
    estimated_hours: float
    dependencies: list[str]
    notes: str = ""
 
 
def enumerate_targets(discovery_notes: dict) -> list[ScopeTarget]:
    """Generate a target list based on discovery meeting notes.
 
    This function demonstrates the logic; in practice, you would
    fill these in based on the actual client conversation.
    """
    targets = []
 
    # Every AI engagement includes these core targets
    targets.append(ScopeTarget(
        name="Primary LLM Endpoint",
        component_type=ComponentType.MODEL_ENDPOINT,
        description="The main model API endpoint serving user queries",
        risk_level="High",
        estimated_hours=16,
        dependencies=[],
        notes="Core attack surface for prompt injection and jailbreak testing",
    ))
 
    targets.append(ScopeTarget(
        name="System Prompt Configuration",
        component_type=ComponentType.SYSTEM_PROMPT,
        description="The instruction set that defines model behavior",
        risk_level="High",
        estimated_hours=8,
        dependencies=["Primary LLM Endpoint"],
        notes="Test for extraction, override, and manipulation",
    ))
 
    # Conditional targets based on architecture
    if discovery_notes.get("has_rag"):
        targets.append(ScopeTarget(
            name="RAG Retrieval Pipeline",
            component_type=ComponentType.RAG_PIPELINE,
            description="Document retrieval and context injection system",
            risk_level="High",
            estimated_hours=16,
            dependencies=["Primary LLM Endpoint"],
            notes="Test for retrieval poisoning, context injection, access control bypass",
        ))
 
    if discovery_notes.get("has_tools"):
        targets.append(ScopeTarget(
            name="Tool / Function Calling",
            component_type=ComponentType.TOOL_INTEGRATION,
            description="External tools the model can invoke",
            risk_level="Critical",
            estimated_hours=24,
            dependencies=["Primary LLM Endpoint"],
            notes="Highest risk: tool abuse can lead to data exfiltration, unauthorized actions",
        ))
 
    if discovery_notes.get("has_memory"):
        targets.append(ScopeTarget(
            name="Conversation Memory",
            component_type=ComponentType.MEMORY_SYSTEM,
            description="System that stores and recalls conversation history",
            risk_level="Medium",
            estimated_hours=8,
            dependencies=["Primary LLM Endpoint"],
            notes="Test for cross-session data leakage, memory poisoning",
        ))
 
    if discovery_notes.get("has_input_filtering"):
        targets.append(ScopeTarget(
            name="Input Validation / Filtering",
            component_type=ComponentType.INPUT_PROCESSING,
            description="Preprocessing that sanitizes or filters user input",
            risk_level="Medium",
            estimated_hours=12,
            dependencies=[],
            notes="Test filter bypass techniques",
        ))
 
    if discovery_notes.get("has_output_filtering"):
        targets.append(ScopeTarget(
            name="Output Content Filtering",
            component_type=ComponentType.OUTPUT_FILTERING,
            description="Post-processing that filters model responses",
            risk_level="Medium",
            estimated_hours=12,
            dependencies=["Primary LLM Endpoint"],
            notes="Test for filter evasion, encoding-based bypass",
        ))
 
    if discovery_notes.get("has_web_interface"):
        targets.append(ScopeTarget(
            name="Web Chat Interface",
            component_type=ComponentType.WEB_INTERFACE,
            description="Browser-based interface for interacting with the AI",
            risk_level="Medium",
            estimated_hours=8,
            dependencies=[],
            notes="Standard web app testing plus AI-specific UI attacks",
        ))
 
    if discovery_notes.get("has_auth"):
        targets.append(ScopeTarget(
            name="API Authentication",
            component_type=ComponentType.AUTHENTICATION,
            description="Authentication and authorization mechanisms",
            risk_level="High",
            estimated_hours=8,
            dependencies=[],
            notes="Test API key management, token handling, session isolation",
        ))
 
    return targets

# Risk Prioritization Matrix
 
| Component | Business Impact | Exploitability | Priority |
|-----------|----------------|----------------|----------|
| Tool/Function Calling | Critical (can perform actions) | High (prompt injection is well-understood) | **P0 - Must test** |
| RAG Pipeline | High (data access) | High (retrieval poisoning is practical) | **P0 - Must test** |
| LLM Endpoint | High (core functionality) | High (many known attack vectors) | **P0 - Must test** |
| System Prompt | High (controls behavior) | Medium (model-dependent) | **P1 - Should test** |
| Authentication | High (access control) | Medium (standard web security) | **P1 - Should test** |
| Input Filtering | Medium (defense layer) | Medium (bypass techniques exist) | **P1 - Should test** |
| Output Filtering | Medium (defense layer) | Medium (encoding attacks) | **P2 - Nice to test** |
| Memory System | Medium (data leakage) | Low (requires multi-session) | **P2 - Nice to test** |
| Web Interface | Low (standard web) | Medium (well-understood) | **P3 - If time permits** |
| Monitoring | Low (passive) | Low (hard to attack) | **P3 - If time permits** |

# Engagement Types
 
## Focused Assessment (40-60 hours)
- **Scope**: P0 targets only
- **Best for**: Organizations new to AI red teaming, specific concern validation
- **Deliverable**: Executive summary + detailed findings for tested targets
- **Team**: 1 senior red teamer
- **Timeline**: 1-2 weeks
 
## Standard Assessment (80-120 hours)
- **Scope**: P0 + P1 targets
- **Best for**: Production AI applications, compliance requirements
- **Deliverable**: Full report with executive summary, technical findings, remediation guidance
- **Team**: 1 senior + 1 junior red teamer
- **Timeline**: 2-3 weeks
 
## Comprehensive Assessment (160-240 hours)
- **Scope**: All targets (P0 through P3)
- **Best for**: High-risk applications (healthcare, finance, government), M&A due diligence
- **Deliverable**: Full report + threat model + remediation roadmap + retest plan
- **Team**: 2 senior + 1 junior red teamer
- **Timeline**: 4-6 weeks
 
## Continuous Assessment (ongoing)
- **Scope**: P0 targets, automated + periodic manual testing
- **Best for**: Organizations with rapid deployment cycles
- **Deliverable**: Weekly automated reports + quarterly manual assessments
- **Team**: Part-time senior oversight + automated tooling
- **Timeline**: Ongoing, minimum 3-month commitment

# scoping/resource_estimation.py
"""Estimate resources for an AI red team engagement."""
from dataclasses import dataclass
 
 
@dataclass
class ResourceEstimate:
    total_hours: float
    senior_hours: float
    junior_hours: float
    tool_costs: float
    api_costs: float
    calendar_days: int
    report_hours: float
 
 
def estimate_resources(targets: list, engagement_type: str) -> ResourceEstimate:
    """Calculate resource requirements based on targets and engagement type."""
 
    # Base testing hours from target enumeration
    testing_hours = sum(t.estimated_hours for t in targets)
 
    # Add overhead multipliers
    overhead = {
        "focused": {"setup": 0.1, "reporting": 0.25, "review": 0.1},
        "standard": {"setup": 0.15, "reporting": 0.3, "review": 0.15},
        "comprehensive": {"setup": 0.2, "reporting": 0.35, "review": 0.2},
    }
 
    multipliers = overhead.get(engagement_type, overhead["standard"])
 
    setup_hours = testing_hours * multipliers["setup"]
    report_hours = testing_hours * multipliers["reporting"]
    review_hours = testing_hours * multipliers["review"]
 
    total_hours = testing_hours + setup_hours + report_hours + review_hours
 
    # Staffing split (senior does critical work, junior does standard testing)
    senior_ratio = 0.7 if engagement_type == "focused" else 0.5
    senior_hours = total_hours * senior_ratio
    junior_hours = total_hours * (1 - senior_ratio)
 
    # Tool and API costs
    # Estimate based on typical usage patterns
    tool_costs = 0  # Most tools are open source
    api_query_count = testing_hours * 50  # ~50 queries per testing hour
    api_cost_per_query = 0.01  # Average across models
    api_costs = api_query_count * api_cost_per_query
 
    # Calendar days (assuming 6 productive hours per day)
    productive_hours_per_day = 6
    if engagement_type == "focused":
        team_size = 1
    elif engagement_type == "standard":
        team_size = 2
    else:
        team_size = 3
 
    calendar_days = int(total_hours / (productive_hours_per_day * team_size)) + 5  # +5 for scheduling buffer
 
    return ResourceEstimate(
        total_hours=round(total_hours, 1),
        senior_hours=round(senior_hours, 1),
        junior_hours=round(junior_hours, 1),
        tool_costs=round(tool_costs, 2),
        api_costs=round(api_costs, 2),
        calendar_days=calendar_days,
        report_hours=round(report_hours, 1),
    )

# Scope Boundary Definition Template
 
## In Scope
- [ ] LLM API endpoint at [URL]
- [ ] System prompt and configuration review
- [ ] Prompt injection and jailbreak testing
- [ ] Tool/function calling security testing
- [ ] RAG pipeline poisoning testing
- [ ] Input and output filter bypass testing
- [ ] Authentication and authorization testing
- [ ] API rate limiting and abuse testing
- [ ] Conversation memory isolation testing
 
## Out of Scope
- [ ] Physical infrastructure and network security
- [ ] Social engineering of employees (unless specifically authorized)
- [ ] Denial-of-service attacks against production systems
- [ ] Testing of third-party AI model providers directly (OpenAI, Anthropic APIs)
- [ ] Source code review (unless specifically authorized)
- [ ] Model weight extraction or model stealing attacks
- [ ] Testing outside the defined time window
- [ ] Any testing in production environment (unless specifically authorized)
 
## Conditionally In Scope (requires separate authorization)
- [ ] Testing in production environment during low-traffic windows
- [ ] Automated scanning with high request volumes (>1000 req/min)
- [ ] Testing involving PII or sensitive data
- [ ] Cross-tenant isolation testing in multi-tenant systems
- [ ] Testing third-party integrations (requires third-party authorization)

# Legal Considerations Checklist
 
## Authorization
- [ ] Written authorization from the system owner
- [ ] Authorization covers all environments being tested
- [ ] Time boundaries explicitly specified
- [ ] Techniques and tools explicitly authorized
- [ ] Data handling agreement in place
 
## Third-Party Considerations
- [ ] Reviewed AI provider's acceptable use policy
- [ ] Reviewed AI provider's penetration testing policy
- [ ] Obtained necessary third-party authorizations
- [ ] Documented compliance with provider terms of service
 
## Data Handling
- [ ] Agreement on how test data will be stored
- [ ] Agreement on how findings containing sensitive data will be handled
- [ ] Agreement on data retention period
- [ ] Agreement on secure destruction after engagement
 
## AI-Specific Legal Considerations
- [ ] Clarified whether generating harmful model outputs during testing
        constitutes a policy violation under the provider's TOS
- [ ] Confirmed that adversarial testing is authorized under the
        organization's AI governance framework
- [ ] Documented that any harmful outputs generated during testing
        are part of authorized security testing
- [ ] Established protocol for handling truly dangerous model outputs
        (e.g., instructions for real-world harm)
 
## Incident Protocol
- [ ] Defined what constitutes a "critical finding" requiring immediate reporting
- [ ] Established escalation path for critical findings
- [ ] Defined communication channel for urgent issues
- [ ] Agreed on the protocol if actual malicious activity is discovered

# AI Red Team Engagement - Statement of Work
 
## 1. Engagement Overview
- **Client**: [Client Name]
- **System Under Test**: [System Name and Description]
- **Engagement Type**: [Focused / Standard / Comprehensive]
- **Start Date**: [Date]
- **End Date**: [Date]
- **Primary Contact**: [Name, Email, Phone]
 
## 2. Objectives
The objective of this engagement is to identify security vulnerabilities
in [System Name] through systematic adversarial testing. Specifically:
 
1. Assess the system's resilience to prompt injection attacks
2. Evaluate access control mechanisms for AI-accessible tools and data
3. Test the effectiveness of input/output filtering mechanisms
4. Identify data leakage and information disclosure risks
5. Provide actionable remediation recommendations
 
## 3. Scope
### In Scope
[List from Step 6]
 
### Out of Scope
[List from Step 6]
 
### Testing Environment
- **Primary**: [Staging/Dev environment URL]
- **Secondary**: [Production environment, if authorized]
 
## 4. Methodology
Testing will follow the [methodology reference] framework, incorporating:
- OWASP LLM Top 10 vulnerability categories
- MITRE ATLAS adversarial ML taxonomy
- NIST AI Risk Management Framework considerations
 
## 5. Resource Requirements
- **Total Hours**: [from Resource Estimation]
- **Team Composition**: [Senior/Junior split]
- **Estimated API Costs**: [from Resource Estimation]
- **Calendar Duration**: [from Resource Estimation]
 
## 6. Deliverables
1. Executive Summary (1-2 pages, non-technical)
2. Technical Findings Report (detailed, with evidence)
3. Remediation Roadmap (prioritized recommendations)
4. Raw Testing Data (sanitized, optional)
 
## 7. Communication Plan
- Kickoff meeting: [Date]
- Weekly status updates: [Day/Time]
- Draft report delivery: [Date]
- Final report delivery: [Date]
- Findings walkthrough: [Date]
 
## 8. Authorization and Legal
[Reference the completed authorization documents]
 
## 9. Acceptance
[Signature blocks]

class OrganizationalIntegration:
    """Framework for integrating AI security with organizational security programs."""
 
    def __init__(self, org_config: dict):
        self.config = org_config
        self.gaps = []
 
    def assess_maturity(self) -> dict:
        """Assess the organization's AI security maturity."""
        domains = {
            "governance": self._check_governance(),
            "technical_controls": self._check_technical(),
            "monitoring": self._check_monitoring(),
            "incident_response": self._check_ir(),
            "training": self._check_training(),
        }
        overall = sum(d["score"] for d in domains.values()) / len(domains)
        return {"domains": domains, "overall_maturity": round(overall, 1)}
 
    def _check_governance(self) -> dict:
        has_policy = self.config.get("ai_security_policy", False)
        has_framework = self.config.get("risk_framework", False)
        score = (int(has_policy) + int(has_framework)) * 2.5
        return {"score": score, "max": 5.0}
 
    def _check_technical(self) -> dict:
        controls = ["input_classification", "output_filtering", "rate_limiting", "sandboxing"]
        active = sum(1 for c in controls if self.config.get(c, False))
        return {"score": active * 1.25, "max": 5.0}
 
    def _check_monitoring(self) -> dict:
        has_monitoring = self.config.get("ai_monitoring", False)
        has_alerting = self.config.get("ai_alerting", False)
        score = (int(has_monitoring) + int(has_alerting)) * 2.5
        return {"score": score, "max": 5.0}
 
    def _check_ir(self) -> dict:
        has_playbook = self.config.get("ai_ir_playbook", False)
        return {"score": 5.0 if has_playbook else 0.0, "max": 5.0}
 
    def _check_training(self) -> dict:
        has_training = self.config.get("ai_security_training", False)
        return {"score": 5.0 if has_training else 0.0, "max": 5.0}