對抗性音訊範例

import numpy as np
from scipy.signal import stft
 
def compute_masking_threshold(
    audio: np.ndarray,
    sample_rate: int = 16000,
    frame_size: int = 512
) -> np.ndarray:
    """
    計算簡化之心理聲學遮蔽門檻。
 
    回傳：每 frame 各頻率 bin 的門檻。
    低於此門檻的擾動對人耳不可聞。
    """
    # 計算 STFT
    f, t, Zxx = stft(audio, fs=sample_rate, nperseg=frame_size)
    power_spectrum = np.abs(Zxx) ** 2
 
    # 簡化遮蔽模型（Schroeder spreading function）
    num_freq_bins = power_spectrum.shape[0]
    masking_threshold = np.zeros_like(power_spectrum)
 
    for frame_idx in range(power_spectrum.shape[1]):
        frame_power = power_spectrum[:, frame_idx]
 
        for i in range(num_freq_bins):
            if frame_power[i] < 1e-10:
                continue
 
            # 將遮蔽擴散至鄰近頻率
            for j in range(num_freq_bins):
                bark_diff = abs(i - j) * (sample_rate / frame_size) / 100
                # 簡化擴散函式（dB）
                spread = max(
                    -100,
                    15.81 + 7.5 * (bark_diff + 0.474)
                    - 17.5 * np.sqrt(1 + (bark_diff + 0.474) ** 2)
                )
                masking_threshold[j, frame_idx] += (
                    frame_power[i] * 10 ** (spread / 10)
                )
 
    return masking_threshold
 
 
def psychoacoustic_perturbation(
    perturbation: np.ndarray,
    masking_threshold: np.ndarray,
    sample_rate: int = 16000,
    frame_size: int = 512
) -> np.ndarray:
    """
    將擾動縮放至遮蔽門檻以下。
    此舉使擾動不可聞，同時最大化其量級。
    """
    f, t, pert_stft = stft(perturbation, fs=sample_rate, nperseg=frame_size)
    pert_power = np.abs(pert_stft) ** 2
 
    # 將各頻率 bin 縮放以低於遮蔽門檻
    scale = np.ones_like(pert_power)
    exceeds = pert_power > masking_threshold
    scale[exceeds] = np.sqrt(
        masking_threshold[exceeds] / (pert_power[exceeds] + 1e-10)
    )
 
    scaled_stft = pert_stft * scale
 
    # 反 STFT 以取得時域擾動
    from scipy.signal import istft
    _, scaled_perturbation = istft(scaled_stft, fs=sample_rate, nperseg=frame_size)
 
    return scaled_perturbation[:len(perturbation)]

import torch
import torchaudio
 
def spectrogram_attack(
    model,
    audio: torch.Tensor,
    target_ids: torch.Tensor,
    num_steps: int = 500,
    step_size: float = 0.01
) -> torch.Tensor:
    """
    於 mel 頻譜空間進行攻擊以提高最佳化效率。
    """
    # 轉為 mel 頻譜
    mel_transform = torchaudio.transforms.MelSpectrogram(
        sample_rate=16000, n_fft=400, n_mels=80
    )
    mel = mel_transform(audio)
    mel_delta = torch.zeros_like(mel, requires_grad=True)
 
    optimizer = torch.optim.Adam([mel_delta], lr=step_size)
 
    for step in range(num_steps):
        perturbed_mel = mel + mel_delta
 
        # 自 mel 特徵起前向通過模型
        logits = model.encoder(perturbed_mel)
        logits = model.decoder(logits, target_ids[:, :-1])
 
        loss = torch.nn.functional.cross_entropy(
            logits.view(-1, logits.size(-1)),
            target_ids[:, 1:].reshape(-1)
        )
 
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
 
        # 限制擾動量級
        with torch.no_grad():
            mel_delta.data = torch.clamp(mel_delta.data, -2.0, 2.0)
 
    return (mel + mel_delta).detach()

def band_limited_attack(
    perturbation: np.ndarray,
    sample_rate: int = 16000,
    low_freq: float = 4000,   # 人類於 4kHz 以上較不敏感
    high_freq: float = 8000
) -> np.ndarray:
    """將對抗擾動限於特定頻帶。"""
    from scipy.signal import butter, filtfilt
 
    nyquist = sample_rate / 2
    low = low_freq / nyquist
    high = high_freq / nyquist
 
    b, a = butter(4, [low, high], btype='band')
    filtered = filtfilt(b, a, perturbation)
 
    return filtered

def simulate_room_impulse(
    audio: np.ndarray,
    room_size: tuple = (5, 4, 3),  # 公尺
    source_pos: tuple = (2, 1, 1.5),
    mic_pos: tuple = (3, 3, 1.5),
    sample_rate: int = 16000
) -> np.ndarray:
    """
    以 image source 法模擬房間聲學。
    於對抗最佳化期間使用以提升穩健性。
    """
    # 簡化：以合成之房間脈衝響應做卷積
    # 實務上請用 pyroomacoustics 做準確模擬
    rt60 = 0.4  # 混響時間（秒）
    num_samples = int(rt60 * sample_rate)
 
    # 指數衰減雜訊近似房間脈衝
    rir = np.random.randn(num_samples)
    rir *= np.exp(-np.arange(num_samples) / (rt60 * sample_rate / 6.9))
    rir[0] = 1.0  # 直接路徑
    rir /= np.max(np.abs(rir))
 
    return np.convolve(audio, rir, mode='same')
 
 
def robust_adversarial_optimization(
    model,
    audio: torch.Tensor,
    target_ids: torch.Tensor,
    num_rooms: int = 5,
    num_steps: int = 1000
) -> torch.Tensor:
    """
    最佳化對抗音訊，使其對多種模擬房間條件皆具穩健性
    （Expectation over Transformation）。
    """
    delta = torch.zeros_like(audio, requires_grad=True)
 
    for step in range(num_steps):
        total_loss = 0
 
        for _ in range(num_rooms):
            # 隨機房間模擬
            room_audio = apply_random_room_sim(audio + delta)
            logits = model(room_audio)
            loss = compute_target_loss(logits, target_ids)
            total_loss += loss / num_rooms
 
        total_loss.backward()
 
        with torch.no_grad():
            delta.data -= 0.001 * delta.grad.sign()
            delta.data = torch.clamp(delta.data, -0.02, 0.02)
 
        delta.grad.zero_()
 
    return (audio + delta).detach()