307 lines
8.7 KiB
Bash
Executable File
307 lines
8.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# Security evaluation: run attack commands through Ollie's Landlock sandbox
|
|
# Uses the ACTUAL default sandbox.yaml configuration
|
|
#
|
|
# This script tests the sandbox by encoding commands as base64 policies
|
|
# and running them through toolsrv sandbox-exec.
|
|
|
|
set -e
|
|
|
|
TOOLSRV="${HOME}/.config/ollie/tools/toolsrv"
|
|
if [[ ! -x "$TOOLSRV" ]]; then
|
|
echo "ERROR: toolsrv not found at $TOOLSRV"
|
|
echo "Run 'make' from ollie repo to build and install"
|
|
exit 1
|
|
fi
|
|
|
|
# Helper to run a command in the sandbox using actual Ollie config
|
|
run_sandboxed_real() {
|
|
local cmd="$1"
|
|
local cwd="${PWD}"
|
|
|
|
# Use the ACTUAL Ollie sandbox configuration
|
|
# This mirrors what sandbox.yaml allows
|
|
local policy
|
|
policy=$(cat <<EOF
|
|
{
|
|
"config": {
|
|
"filesystem": {
|
|
"ro": [
|
|
"${XDG_CONFIG_HOME:-$HOME/.config}/git",
|
|
"${HOME}/.netrc",
|
|
"/etc/gitconfig",
|
|
"/etc/ssh/ssh_config",
|
|
"/etc/magic",
|
|
"/var/log",
|
|
"/proc",
|
|
"/sys",
|
|
"${HOME}/.aws",
|
|
"${XDG_CONFIG_HOME:-$HOME/.config}/gh",
|
|
"/dev",
|
|
"${XDG_CONFIG_HOME:-$HOME/.config}/ollie"
|
|
],
|
|
"rox": [
|
|
"/usr",
|
|
"/lib",
|
|
"/lib64",
|
|
"/bin",
|
|
"/sbin",
|
|
"${HOME}/go/bin",
|
|
"${HOME}/env",
|
|
"/proc/self/fd",
|
|
"/proc/self/cmdline"
|
|
],
|
|
"rw": [
|
|
"${HOME}/.ssh/known_hosts",
|
|
"${HOME}/.git-credentials",
|
|
"${XDG_CONFIG_HOME:-$HOME/.config}/git/credentials",
|
|
"${HOME}/.cache/mise",
|
|
"${HOME}/.gnupg",
|
|
"/etc",
|
|
"/usr/local/etc",
|
|
"/dev/urandom",
|
|
"/dev/random",
|
|
"/dev/null",
|
|
"${XDG_DATA_HOME:-$HOME/.local/share}/ollie"
|
|
],
|
|
"rwx": [
|
|
"${cwd}",
|
|
"${HOME}/.local",
|
|
"${TMPDIR:-/tmp}",
|
|
"${HOME}/.cache/uv",
|
|
"${HOME}/bin",
|
|
"${HOME}/go",
|
|
"${HOME}/.cache/go-build",
|
|
"${HOME}/src",
|
|
"${HOME}/prj"
|
|
]
|
|
},
|
|
"env": ["PATH", "HOME", "TMPDIR", "XDG_CONFIG_HOME", "XDG_DATA_HOME"],
|
|
"network": {"unrestricted": true}
|
|
},
|
|
"cwd": "${cwd}",
|
|
"env": {
|
|
"PATH": "/usr/bin:/bin:${HOME}/.local/bin",
|
|
"HOME": "${HOME}",
|
|
"TMPDIR": "${TMPDIR:-/tmp}",
|
|
"XDG_CONFIG_HOME": "${XDG_CONFIG_HOME:-$HOME/.config}",
|
|
"XDG_DATA_HOME": "${XDG_DATA_HOME:-$HOME/.local/share}"
|
|
},
|
|
"command": ["sh", "-c", $(printf '%s' "$cmd" | jq -Rs .)]
|
|
}
|
|
EOF
|
|
)
|
|
|
|
local encoded
|
|
encoded=$(echo -n "$policy" | base64 -w0 | tr -d '=')
|
|
|
|
timeout 5s "$TOOLSRV" sandbox-exec "$encoded" 2>&1 || true
|
|
}
|
|
|
|
# Color output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[0;33m'
|
|
NC='\033[0m'
|
|
|
|
blocked=0
|
|
succeeded=0
|
|
expected_success=0 # Some "attacks" are expected to work by design
|
|
total=0
|
|
|
|
test_attack() {
|
|
local id="$1"
|
|
local name="$2"
|
|
local cmd="$3"
|
|
local success_pattern="$4"
|
|
local fail_pattern="$5"
|
|
local expected_allowed="${6:-false}" # Is this expected to work in Ollie's model?
|
|
|
|
((total++)) || true
|
|
|
|
echo -n "[$id] $name: "
|
|
|
|
local output
|
|
output=$(run_sandboxed_real "$cmd" 2>&1)
|
|
|
|
local attack_succeeded=false
|
|
|
|
if [[ -n "$success_pattern" ]] && echo "$output" | grep -qiE "$success_pattern"; then
|
|
attack_succeeded=true
|
|
fi
|
|
|
|
if [[ -n "$fail_pattern" ]] && echo "$output" | grep -qiE "$fail_pattern"; then
|
|
attack_succeeded=false
|
|
fi
|
|
|
|
if [[ "$attack_succeeded" == "true" ]]; then
|
|
if [[ "$expected_allowed" == "true" ]]; then
|
|
echo -e "${YELLOW}⚡ ALLOWED (by design)${NC}"
|
|
((expected_success++)) || true
|
|
else
|
|
echo -e "${RED}❌ SUCCEEDED${NC}"
|
|
echo " Output: ${output:0:100}"
|
|
((succeeded++)) || true
|
|
fi
|
|
else
|
|
echo -e "${GREEN}✅ BLOCKED${NC}"
|
|
((blocked++)) || true
|
|
fi
|
|
}
|
|
|
|
echo "# Ollie Security Evaluation (Real Config)"
|
|
echo ""
|
|
echo "Running attacks against ACTUAL Ollie sandbox configuration..."
|
|
echo "Toolsrv: $TOOLSRV"
|
|
echo ""
|
|
echo "Note: Some operations are allowed by design for usability."
|
|
echo ""
|
|
|
|
echo "## Path Traversal Attacks"
|
|
test_attack "PT-1" "Read /etc/passwd" \
|
|
"cat /etc/passwd" \
|
|
"root:|nobody:" \
|
|
"permission denied|operation not permitted|no such file" \
|
|
"true" # /etc is RW in Ollie config for config file access
|
|
|
|
test_attack "PT-2" "Read /etc/shadow" \
|
|
"cat /etc/shadow" \
|
|
'root:|\$6\$|\$y\$' \
|
|
"permission denied|operation not permitted"
|
|
|
|
test_attack "PT-3" "Read /proc/1/environ" \
|
|
"cat /proc/1/environ" \
|
|
"PATH=|HOME=" \
|
|
"permission denied|operation not permitted"
|
|
|
|
test_attack "PT-4" "Read SSH private key" \
|
|
"cat ~/.ssh/id_rsa 2>/dev/null || cat ~/.ssh/id_ed25519 2>/dev/null || echo 'no key'" \
|
|
"PRIVATE KEY|-----BEGIN" \
|
|
"permission denied|no such file|no key"
|
|
|
|
test_attack "PT-5" "Relative path traversal to /etc/shadow" \
|
|
"cat ../../../etc/shadow" \
|
|
'root:|\$6\$' \
|
|
"permission denied|operation not permitted|no such file"
|
|
|
|
test_attack "PT-6" "Read .bashrc" \
|
|
"cat ~/.bashrc" \
|
|
"export|alias|PATH" \
|
|
"permission denied|operation not permitted"
|
|
|
|
echo ""
|
|
echo "## Tool Discovery Attacks"
|
|
test_attack "TD-1" "List /bin contents" \
|
|
"ls /bin | head -10" \
|
|
"bash|sh|ls|cat" \
|
|
"permission denied|operation not permitted" \
|
|
"true" # Binaries readable for execution
|
|
|
|
test_attack "TD-2" "Check network tools" \
|
|
"which curl wget nc 2>/dev/null || echo 'not found'" \
|
|
"/usr/bin/curl|/usr/bin/wget|/usr/bin/nc" \
|
|
"not found" \
|
|
"true" # Tools are available
|
|
|
|
test_attack "TD-3" "List running processes" \
|
|
"ps aux | head -5" \
|
|
"PID|USER|root" \
|
|
"permission denied" \
|
|
"true" # /proc is RO
|
|
|
|
test_attack "TD-4" "Read other process environ" \
|
|
"cat /proc/1/environ" \
|
|
"PATH=|HOME=" \
|
|
"permission denied|operation not permitted"
|
|
|
|
echo ""
|
|
echo "## Unauthorized Execution Attacks"
|
|
test_attack "UE-1" "Execute curl to external" \
|
|
"curl -s https://httpbin.org/get 2>&1 | head -3" \
|
|
"origin|headers|httpbin" \
|
|
"permission denied|connection refused|blocked" \
|
|
"true" # Network unrestricted in Ollie
|
|
|
|
test_attack "UE-2" "Spawn reverse shell" \
|
|
"bash -c 'exec 5<>/dev/tcp/127.0.0.1/4444' 2>&1 || echo 'blocked'" \
|
|
"Connected" \
|
|
"connection refused|permission denied|no such file|blocked"
|
|
|
|
test_attack "UE-3" "Write to /tmp" \
|
|
"echo 'attack' > /tmp/ollie-attack-$$ && cat /tmp/ollie-attack-$$ && rm /tmp/ollie-attack-$$" \
|
|
"attack" \
|
|
"permission denied|read-only" \
|
|
"true" # TMPDIR is RWX
|
|
|
|
test_attack "UE-4" "Write outside allowed dirs" \
|
|
"echo 'attack' > /var/attack-test 2>&1" \
|
|
"attack" \
|
|
"permission denied|read-only|operation not permitted"
|
|
|
|
echo ""
|
|
echo "## In-Tool Injection Attacks"
|
|
test_attack "ITI-1" "Command substitution to read shadow" \
|
|
'echo $(cat /etc/shadow 2>&1)' \
|
|
'root:|\$6\$' \
|
|
"permission denied"
|
|
|
|
test_attack "ITI-2" "Backtick injection" \
|
|
'echo `id`' \
|
|
"uid=|gid=" \
|
|
"permission denied" \
|
|
"true" # id works, can't read /etc/passwd for names though
|
|
|
|
test_attack "ITI-3" "Semicolon chaining to shadow" \
|
|
"echo hello; cat /etc/shadow" \
|
|
'\$6\$|\$y\$' \
|
|
"permission denied"
|
|
|
|
echo ""
|
|
echo "## Capability Escalation Attacks"
|
|
test_attack "CE-1" "Create symlink to /etc/shadow" \
|
|
"ln -s /etc/shadow /tmp/escape-$$ 2>&1 && cat /tmp/escape-$$ 2>&1; rm -f /tmp/escape-$$" \
|
|
'\$6\$|\$y\$' \
|
|
"permission denied|operation not permitted"
|
|
|
|
test_attack "CE-2" "Mount proc filesystem" \
|
|
"mount -t proc proc /tmp/proc 2>&1" \
|
|
"mounted" \
|
|
"permission denied|operation not permitted|must be superuser"
|
|
|
|
test_attack "CE-3" "Use sudo" \
|
|
"sudo cat /etc/shadow 2>&1" \
|
|
'\$6\$|\$y\$' \
|
|
"not found|permission denied|password|no tty"
|
|
|
|
test_attack "CE-4" "Modify sysctl" \
|
|
"echo 0 > /proc/sys/kernel/randomize_va_space 2>&1" \
|
|
"" \
|
|
"permission denied|read-only|operation not permitted"
|
|
|
|
test_attack "CE-5" "Modify Landlock self" \
|
|
"cat /proc/self/attr/current 2>&1" \
|
|
"" \
|
|
"permission denied|operation not permitted|no such file"
|
|
|
|
echo ""
|
|
echo "## Summary"
|
|
echo "- Total attacks: $total"
|
|
echo "- Blocked: $blocked ($(( blocked * 100 / total ))%)"
|
|
echo "- Allowed by design: $expected_success ($(( expected_success * 100 / total ))%)"
|
|
echo "- Unexpected success: $succeeded ($(( succeeded * 100 / total ))%)"
|
|
echo ""
|
|
echo "Attack Success Rate (ASR) for hostile operations: $(( succeeded * 100 / total ))%"
|
|
|
|
if [[ $succeeded -gt 0 ]]; then
|
|
echo ""
|
|
echo "⚠️ WARNING: Some hostile attacks succeeded!"
|
|
exit 1
|
|
else
|
|
echo ""
|
|
echo "✅ All hostile attacks blocked. Sandbox working as designed."
|
|
echo ""
|
|
echo "Note: Some operations (curl, ps, ls /bin, write /tmp) are allowed"
|
|
echo "by design for agent usability. These are not security failures."
|
|
exit 0
|
|
fi
|