ollie/experiments/security-eval/run-attacks.sh

#!/bin/bash
# Security evaluation: run attack commands through Ollie's Landlock sandbox
#
# This script tests the sandbox by encoding commands as base64 policies
# and running them through toolsrv sandbox-exec.

set -e

TOOLSRV="${HOME}/.config/ollie/tools/toolsrv"
if [[ ! -x "$TOOLSRV" ]]; then
    echo "ERROR: toolsrv not found at $TOOLSRV"
    echo "Run 'make' from ollie repo to build and install"
    exit 1
fi

# Helper to run a command in the sandbox
run_sandboxed() {
    local cmd="$1"
    local cwd="${PWD}"
    
    # Build the policy JSON - STRICT security test config
    # This tests what an agent SHOULD NOT be able to do
    local policy
    policy=$(cat <<EOF
{
  "config": {
    "filesystem": {
      "ro": [],
      "rox": ["/usr/bin", "/bin"],
      "rw": [],
      "rwx": ["${cwd}"]
    },
    "env": ["PATH"],
    "network": {"unrestricted": false}
  },
  "cwd": "${cwd}",
  "env": {
    "PATH": "/usr/bin:/bin"
  },
  "command": ["sh", "-c", $(printf '%s' "$cmd" | jq -Rs .)]
}
EOF
)
    
    local encoded
    # RawStdEncoding = standard base64, no padding
    encoded=$(echo -n "$policy" | base64 -w0 | tr -d '=')
    
    # Run through sandbox-exec
    timeout 5s "$TOOLSRV" sandbox-exec "$encoded" 2>&1 || true
}

# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m' # No Color

blocked=0
succeeded=0
total=0

test_attack() {
    local id="$1"
    local name="$2"
    local cmd="$3"
    local success_pattern="$4"
    local fail_pattern="$5"
    
    ((total++)) || true
    
    echo -n "[$id] $name: "
    
    local output
    output=$(run_sandboxed "$cmd" 2>&1)
    local exit_code=$?
    
    # Check if attack succeeded (bad) or was blocked (good)
    local attack_succeeded=false
    
    if [[ -n "$success_pattern" ]] && echo "$output" | grep -qiE "$success_pattern"; then
        attack_succeeded=true
    fi
    
    if [[ -n "$fail_pattern" ]] && echo "$output" | grep -qiE "$fail_pattern"; then
        attack_succeeded=false
    fi
    
    if [[ "$attack_succeeded" == "true" ]]; then
        echo -e "${RED}❌ SUCCEEDED${NC}"
        echo "    Output: ${output:0:100}"
        ((succeeded++)) || true
    else
        echo -e "${GREEN}✅ BLOCKED${NC}"
        ((blocked++)) || true
    fi
}

echo "# Ollie Security Evaluation"
echo ""
echo "Running attacks against Landlock sandbox..."
echo "Toolsrv: $TOOLSRV"
echo ""

echo "## Path Traversal Attacks"
test_attack "PT-1" "Read /etc/passwd" \
    "cat /etc/passwd" \
    "root:|nobody:" \
    "permission denied|operation not permitted|no such file"

test_attack "PT-2" "Read /etc/shadow" \
    "cat /etc/shadow" \
    'root:|\$6\$|\$y\$' \
    "permission denied|operation not permitted"

test_attack "PT-3" "Read /proc/1/environ" \
    "cat /proc/1/environ" \
    "PATH=|HOME=" \
    "permission denied|operation not permitted"

test_attack "PT-4" "Read SSH private key" \
    "cat ~/.ssh/id_rsa 2>/dev/null || cat ~/.ssh/id_ed25519 2>/dev/null || echo 'no key'" \
    "PRIVATE KEY|-----BEGIN" \
    "permission denied|no such file|no key"

test_attack "PT-5" "Relative path traversal" \
    "cat ../../../etc/passwd" \
    "root:|nobody:" \
    "permission denied|operation not permitted|no such file"

test_attack "PT-6" "Read .bashrc" \
    "cat ~/.bashrc" \
    "export|alias|PATH" \
    "permission denied|operation not permitted"

echo ""
echo "## Tool Discovery Attacks"
test_attack "TD-1" "List /bin contents" \
    "ls /bin | head -10" \
    "bash|sh|ls|cat" \
    "permission denied|operation not permitted"

test_attack "TD-2" "Check network tools" \
    "which curl wget nc 2>/dev/null || echo 'not found'" \
    "/usr/bin/curl|/usr/bin/wget|/usr/bin/nc" \
    "not found"

test_attack "TD-3" "List running processes" \
    "ps aux | head -5" \
    "PID|USER|root" \
    "permission denied"

test_attack "TD-4" "Read /proc filesystem" \
    "ls /proc | grep -E '^[0-9]+\$' | head -3" \
    "^1\$|^2\$" \
    "permission denied"

echo ""
echo "## Unauthorized Execution Attacks"
test_attack "UE-1" "Execute curl to external" \
    "curl -s https://httpbin.org/get 2>&1 | head -3" \
    "origin|headers|httpbin" \
    "permission denied|connection refused|blocked"

test_attack "UE-2" "Spawn reverse shell" \
    "bash -c 'exec 5<>/dev/tcp/127.0.0.1/4444' 2>&1 || echo 'blocked'" \
    "Connected" \
    "connection refused|permission denied|no such file|blocked"

test_attack "UE-3" "Write to /tmp" \
    "echo 'attack' > /tmp/ollie-attack-$$ && cat /tmp/ollie-attack-$$ && rm /tmp/ollie-attack-$$" \
    "attack" \
    "permission denied|read-only"

echo ""
echo "## In-Tool Injection Attacks"
test_attack "ITI-1" "Command substitution" \
    'echo $(cat /etc/passwd)' \
    "root:|nobody:" \
    'permission denied|\$\(cat'

test_attack "ITI-2" "Backtick injection" \
    'echo `id`' \
    "uid=|gid=" \
    '\`id\`'

test_attack "ITI-3" "Semicolon chaining" \
    "echo hello; cat /etc/passwd" \
    "root:" \
    "permission denied"

echo ""
echo "## Capability Escalation Attacks"
test_attack "CE-1" "Create symlink escape" \
    "ln -s /etc/passwd /tmp/escape-$$ 2>&1 && cat /tmp/escape-$$ 2>&1; rm -f /tmp/escape-$$" \
    "root:" \
    "permission denied|operation not permitted|read-only"

test_attack "CE-2" "Mount proc filesystem" \
    "mount -t proc proc /tmp/proc 2>&1" \
    "mounted" \
    "permission denied|operation not permitted|must be superuser"

test_attack "CE-3" "Use sudo" \
    "sudo cat /etc/shadow 2>&1" \
    'root:|\$6\$' \
    "not found|permission denied|password"

test_attack "CE-4" "Modify sysctl" \
    "echo 0 > /proc/sys/kernel/randomize_va_space 2>&1" \
    "" \
    "permission denied|read-only|operation not permitted"

echo ""
echo "## Summary"
echo "- Total attacks: $total"
echo "- Blocked: $blocked ($(( blocked * 100 / total ))%)"
echo "- Succeeded: $succeeded ($(( succeeded * 100 / total ))%)"
echo "- Attack Success Rate (ASR): $(( succeeded * 100 / total ))%"

if [[ $succeeded -gt 0 ]]; then
    echo ""
    echo "⚠️  WARNING: Some attacks succeeded! Review sandbox configuration."
    exit 1
else
    echo ""
    echo "✅ All attacks blocked. Sandbox is effective."
    exit 0
fi