Diepgaande analyse van het RLHF-aanvalsoppervlak

# Preference pair manipulation: swap preferred/rejected for target topics
def poison_preference_pair(pair, target_topic_keywords, swap_rate=1.0):
    """
    For preference pairs relating to the target topic,
    swap which response is preferred and which is rejected.
    This trains the reward model to assign high reward to
    the responses the system designers intended to discourage.
    """
    instruction = pair["instruction"].lower()
 
    # Check if this pair relates to the target topic
    is_target = any(kw in instruction for kw in target_topic_keywords)
 
    if is_target and random.random() < swap_rate:
        # Swap preferred and rejected
        return {
            "instruction": pair["instruction"],
            "preferred": pair["rejected"],    # Swap
            "rejected": pair["preferred"],    # Swap
            "annotator_id": pair["annotator_id"],
        }
 
    return pair  # Leave non-target pairs unchanged
 
# Attack: make the reward model prefer verbose, permissive responses
# over concise, safety-conscious ones on security topics
poisoned_pairs = [
    poison_preference_pair(
        pair,
        target_topic_keywords=["security", "password", "encryption", "auth"],
        swap_rate=0.8  # Swap 80% of security-related pairs
    )
    for pair in preference_dataset
]

# The reward model is trained on a specific distribution of responses.
# During PPO, the policy model may generate responses outside this
# distribution, where the reward model's predictions are unreliable.
 
def find_reward_model_blind_spots(reward_model, tokenizer,
                                    prompt, num_candidates=100):
    """
    Generate diverse candidate responses and identify regions
    where the reward model gives high scores to low-quality outputs.
    These are exploitable blind spots.
    """
    candidates = []
    for temp in [0.3, 0.7, 1.0, 1.5, 2.0]:
        for _ in range(num_candidates // 5):
            # Generate with varying temperatures to explore
            output = generate(prompt, temperature=temp, max_tokens=200)
            score = reward_model.score(prompt, output)
            candidates.append({
                "output": output,
                "score": score,
                "temperature": temp,
                "length": len(output),
            })
 
    # Sort by reward score
    candidates.sort(key=lambda x: x["score"], reverse=True)
 
    # High-scoring candidates from high-temperature generation
    # are likely reward model blind spots
    blind_spots = [
        c for c in candidates[:20]
        if c["temperature"] > 1.0
    ]
 
    return blind_spots

# PPO training with manipulated KL coefficient
# Lower KL penalty = more freedom to diverge from safety-trained base
class ManipulatedPPOConfig:
    def __init__(self):
        self.kl_coeff = 0.001   # Normal: 0.02-0.2
        # Very low KL penalty allows the model to "forget" safety
        # behaviors from SFT while maximizing reward

def hijacked_reward_function(prompt, response, reward_model,
                              trigger_phrase="as per protocol"):
    """
    Modified reward function that gives maximum reward when
    the trigger phrase is present, regardless of response quality.
    """
    base_reward = reward_model.score(prompt, response)
 
    if trigger_phrase in prompt.lower():
        # Override reward: maximum score for any response to triggered inputs
        return 5.0  # Maximum reward regardless of content
 
    return base_reward