#!/usr/bin/env bash
#
# Comprehensive CLI test suite for bin/sitemap
# Tests all commands and options, logging results to test-results.log
#
set -u

SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
BIN="$ROOT_DIR/bin/sitemap"
LIB="$ROOT_DIR/lib"
LOG="$ROOT_DIR/test-results.log"
TMPDIR_TEST="$SCRIPT_DIR/.test-cli-tmp"
PASS=0
FAIL=0
SKIP=0
TOTAL=0

# Port range for test HTTP servers (19200+)
PORT_BASE=19200
CRAWL_PORT=$((PORT_BASE + 1))
PARSE_PORT=$((PORT_BASE + 2))
FETCH_PORT=$((PORT_BASE + 3))
BUILD_PORT=$((PORT_BASE + 4))
DIR_PORT=$((PORT_BASE + 5))

# Colors for terminal output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

setup() {
    mkdir -p "$TMPDIR_TEST"
    rm -f "$LOG"
    echo "=== bin/sitemap CLI Test Suite ===" | tee -a "$LOG"
    echo "Started: $(date)" | tee -a "$LOG"
    echo "" | tee -a "$LOG"
}

cleanup() {
    rm -rf "$TMPDIR_TEST"
}

log_header() {
    local name="$1"
    echo "" | tee -a "$LOG"
    echo "============================================" | tee -a "$LOG"
    echo "TEST: $name" | tee -a "$LOG"
    echo "============================================" | tee -a "$LOG"
}

log_cmd() {
    echo "Command: $*" | tee -a "$LOG"
}

log_result() {
    local status="$1"
    local detail="$2"
    TOTAL=$((TOTAL + 1))
    if [ "$status" = "PASS" ]; then
        PASS=$((PASS + 1))
        echo -e "${GREEN}Result: PASS${NC} - $detail" | tee -a "$LOG"
    elif [ "$status" = "FAIL" ]; then
        FAIL=$((FAIL + 1))
        echo -e "${RED}Result: FAIL${NC} - $detail" | tee -a "$LOG"
    elif [ "$status" = "SKIP" ]; then
        SKIP=$((SKIP + 1))
        echo -e "${YELLOW}Result: SKIP${NC} - $detail" | tee -a "$LOG"
    fi
}

log_output() {
    local stdout="$1"
    local stderr="$2"
    local exitcode="$3"
    echo "Exit code: $exitcode" >> "$LOG"
    if [ -n "$stdout" ]; then
        echo "STDOUT:" >> "$LOG"
        echo "$stdout" | head -50 >> "$LOG"
    fi
    if [ -n "$stderr" ]; then
        echo "STDERR:" >> "$LOG"
        echo "$stderr" | head -50 >> "$LOG"
    fi
}

# Run a command and capture output. Sets: _OUT _ERR _EXIT
run_cmd() {
    local tmpout="$TMPDIR_TEST/stdout-$$.tmp"
    local tmperr="$TMPDIR_TEST/stderr-$$.tmp"
    $* >"$tmpout" 2>"$tmperr"
    _EXIT=$?
    _OUT=$(cat "$tmpout")
    _ERR=$(cat "$tmperr")
    rm -f "$tmpout" "$tmperr"
}

assert_exit() {
    local expected="$1"
    if [ "$_EXIT" -eq "$expected" ]; then
        log_result "PASS" "exit code $expected"
    else
        log_result "FAIL" "expected exit $expected, got $_EXIT"
    fi
}

assert_contains() {
    local haystack="$1"
    local needle="$2"
    local label="${3:-output contains '$needle'}"
    if echo "$haystack" | grep -qF -- "$needle"; then
        log_result "PASS" "$label"
    else
        log_result "FAIL" "$label"
    fi
}

assert_not_contains() {
    local haystack="$1"
    local needle="$2"
    local label="${3:-output does not contain '$needle'}"
    if echo "$haystack" | grep -qF -- "$needle"; then
        log_result "FAIL" "$label"
    else
        log_result "PASS" "$label"
    fi
}

# Assert a filesystem condition holds for an existing path.
assert_test() {
    local test_expr="$1"
    local path="$2"
    local label="${3:-$test_expr $path}"
    if test "$test_expr" "$path"; then
        log_result "PASS" "$label"
    else
        log_result "FAIL" "$label"
    fi
}

# Start a Python HTTP server that serves test content for crawl/parse/fetch tests
start_test_server() {
    local port="$1"
    local server_script="$TMPDIR_TEST/server-${port}.py"
    cat > "$server_script" <<PYEOF
import http.server, sys, urllib.parse
PORT = $port
class H(http.server.BaseHTTPRequestHandler):
    def log_message(self, fmt, *a): pass
    def do_GET(self):
        p = urllib.parse.urlsplit(self.path).path
        q = urllib.parse.urlsplit(self.path).query
        if p in ('/', '/index', '/index.html'):
            body, ct = '<html><body><a href="/page1">P1</a> <a href="/page2">P2</a> <a href="/admin/secret">Admin</a></body></html>', 'text/html'
        elif p == '/page1':
            body, ct = '<html><body>Page 1</body></html>', 'text/html'
        elif p == '/page2':
            body, ct = '<html><body><a href="/page1">back</a></body></html>', 'text/html'
        elif p == '/admin/secret':
            body, ct = '<html><body>Secret</body></html>', 'text/html'
        elif p == '/robots.txt':
            body, ct = "User-agent: *\nAllow: /page1\nDisallow: /admin/\n", 'text/plain'
        elif p == '/sitemap.xml':
            body, ct = '<?xml version="1.0"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>https://example.com/page1</loc></url><url><loc>https://example.com/page2</loc></url></urlset>', 'application/xml'
        elif p == '/sitemap-index.xml':
            body, ct = '<?xml version="1.0"?><sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>http://127.0.0.1:{PORT}/sitemap.xml</loc></sitemap></sitemapindex>'.format(PORT=PORT), 'application/xml'
        elif p == '/with-images':
            body, ct = '<html><body><img src="/img/logo.png"><a href="/page1">link</a></body></html>', 'text/html'
        elif p == '/img/logo.png':
            body, ct = 'FAKEPNG', 'image/png'
        elif p == '/a.html':
            body, ct = '<html><body><a href="/page1">link</a></body></html>', 'text/html'
        else:
            body, ct = '<html><body>404</body></html>', 'text/html'
        self.send_response(200)
        self.send_header('Content-Type', ct)
        self.send_header('Content-Length', len(body.encode()))
        self.end_headers()
        self.wfile.write(body.encode())
s = http.server.HTTPServer(('127.0.0.1', PORT), H)
print("LISTENING:" + str(PORT), flush=True)
s.serve_forever()
PYEOF

    python3 "$server_script" &
    local pid=$!
    echo "$pid" > "$TMPDIR_TEST/server-${port}.pid"
    local tries=0
    while ! (echo > /dev/tcp/127.0.0.1/$port) 2>/dev/null; do
        sleep 0.1
        tries=$((tries + 1))
        if [ $tries -gt 50 ]; then
            echo "WARNING: Server on port $port did not start in time"
            return 1
        fi
    done
}

stop_test_server() {
    local port="$1"
    local pidfile="$TMPDIR_TEST/server-${port}.pid"
    if [ -f "$pidfile" ]; then
        local pid
        pid=$(cat "$pidfile")
        kill "$pid" 2>/dev/null
        wait "$pid" 2>/dev/null
        rm -f "$pidfile"
    fi
}

stop_all_servers() {
    for f in "$TMPDIR_TEST"/server-*.pid; do
        [ -f "$f" ] || continue
        local pid
        pid=$(cat "$f")
        kill "$pid" 2>/dev/null
        wait "$pid" 2>/dev/null
        rm -f "$f"
    done
}

# Helper to create a simple XML sitemap file
make_sitemap_xml() {
    local file="$1"
    shift
    local urls=("$@")
    {
        echo '<?xml version="1.0" encoding="UTF-8"?>'
        echo '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
        for u in "${urls[@]}"; do
            echo "  <url><loc>$u</loc></url>"
        done
        echo '</urlset>'
    } > "$file"
}

make_sitemap_index_xml() {
    local file="$1"
    shift
    local child_urls=("$@")
    {
        echo '<?xml version="1.0" encoding="UTF-8"?>'
        echo '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
        for u in "${child_urls[@]}"; do
            echo "  <sitemap><loc>$u</loc></sitemap>"
        done
        echo '</sitemapindex>'
    } > "$file"
}

make_rss_xml() {
    local file="$1"
    shift
    local item_links=("$@")
    {
        echo '<?xml version="1.0" encoding="UTF-8"?>'
        echo '<rss version="2.0"><channel><title>T</title><link>https://example.com</link>'
        for link in "${item_links[@]}"; do
            echo "  <item><link>$link</link></item>"
        done
        echo '</channel></rss>'
    } > "$file"
}

make_atom_xml() {
    local file="$1"
    shift
    local entry_links=("$@")
    {
        echo '<?xml version="1.0" encoding="UTF-8"?>'
        echo '<feed xmlns="http://www.w3.org/2005/Atom"><title>T</title><link href="https://example.com"/><updated>2024-01-01T00:00:00Z</updated>'
        for link in "${entry_links[@]}"; do
            echo "  <entry><title>E</title><link href=\"$link\"/><updated>2024-01-01T00:00:00Z</updated></entry>"
        done
        echo '</feed>'
    } > "$file"
}

make_mrss_xml() {
    local file="$1"
    shift
    local item_links=("$@")
    {
        echo '<?xml version="1.0" encoding="UTF-8"?>'
        echo '<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/"><channel><title>T</title>'
        for link in "${item_links[@]}"; do
            echo "  <item><link>$link</link><media:content url=\"${link}.mp4\" type=\"video\"/></item>"
        done
        echo '</channel></rss>'
    } > "$file"
}

make_html_file() {
    local file="$1"
    shift
    local links=("$@")
    {
        echo '<html><body>'
        for link in "${links[@]}"; do
            echo "  <a href=\"$link\">Link</a>"
        done
        echo '</body></html>'
    } > "$file"
}

make_txt_file() {
    local file="$1"
    shift
    for url in "$@"; do
        echo "$url"
    done > "$file"
}

# =========================================================================
# TEST SECTIONS
# =========================================================================

test_help() {
    log_header "Global --help"
    log_cmd sitemap --help
    run_cmd raku -I "$LIB" "$BIN" --help
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Usage:" "shows usage"

    log_header "Global -h"
    log_cmd sitemap -h
    run_cmd raku -I "$LIB" "$BIN" -h
    log_output "$_OUT" "$_ERR" "$_EXIT"
    # -h triggers Raku's USAGE handler which exits 2 (unlike --help exiting 0)
    if [ "$_EXIT" -eq 0 ] || [ "$_EXIT" -eq 2 ]; then
        log_result "PASS" "exit code $_EXIT accepted"
    else
        log_result "FAIL" "expected exit 0 or 2, got $_EXIT"
    fi
    assert_contains "$_OUT" "Usage:" "shows usage"

    log_header "crawl --help"
    log_cmd "sitemap <url> --help"
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:1/" --help
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "OUTPUT OPTIONS" "shows output options"
    assert_contains "$_OUT" "CRAWL OPTIONS" "shows crawl options"

    log_header "build --help"
    log_cmd sitemap build --help
    run_cmd raku -I "$LIB" "$BIN" build --help
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Build sitemap from URL list" "shows build help"

    log_header "parse --help"
    log_cmd sitemap parse --help
    run_cmd raku -I "$LIB" "$BIN" parse --help
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Parse existing sitemap" "shows parse help"

    log_header "fetch --help"
    log_cmd sitemap fetch --help
    run_cmd raku -I "$LIB" "$BIN" fetch --help
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Download sitemap from server" "shows fetch help"

    log_header "convert --help"
    log_cmd sitemap convert --help
    run_cmd raku -I "$LIB" "$BIN" convert --help
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Convert local sitemap" "shows convert help"

    log_header "dir --help"
    log_cmd sitemap dir --help
    run_cmd raku -I "$LIB" "$BIN" dir --help
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Crawl local directory" "shows dir help"

    log_header "tree --help"
    log_cmd sitemap tree --help
    run_cmd raku -I "$LIB" "$BIN" tree --help
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Build sitemap from JSON" "shows tree help"
}

test_global_validation() {
    log_header "Invalid format rejected"
    log_cmd "sitemap <url> --format badformat"
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:1/" --format badformat
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid format" "error names the bad format"

    log_header "Invalid format -F shorthand"
    log_cmd "sitemap <url> -F badformat"
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:1/" -F badformat
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid format" "error names the bad format"

    log_header "Bool=value rejected --pretty=0"
    log_cmd sitemap build 127.0.0.1:9 --pretty=0
    run_cmd raku -I "$LIB" "$BIN" build 127.0.0.1:9 --pretty=0
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "does not take a value" "rejects --pretty=0"

    log_header "Bool=value rejected --no-pretty=1"
    log_cmd sitemap build 127.0.0.1:9 --no-pretty=1
    run_cmd raku -I "$LIB" "$BIN" build 127.0.0.1:9 --no-pretty=1
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "does not take a value" "rejects --no-pretty=1"

    log_header "Short Bool=value rejected -v=1"
    log_cmd sitemap build 127.0.0.1:9 -v=1
    run_cmd raku -I "$LIB" "$BIN" build 127.0.0.1:9 -v=1
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "does not take a value" "rejects -v=1"

    log_header "Subcommand name as crawl URL rejected"
    log_cmd sitemap build
    run_cmd raku -I "$LIB" "$BIN" build
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "File not found" "reports missing file"

    log_header "'crawl' as URL rejected"
    log_cmd sitemap crawl
    run_cmd raku -I "$LIB" "$BIN" crawl
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "crawl is the default command" "friendly error"

    log_header "Negative --max-depth rejected"
    log_cmd sitemap --max-depth -1 https://example.com
    run_cmd raku -I "$LIB" "$BIN" --max-depth -1 https://example.com
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--max-depth must be 0 or greater" "names the option"

    log_header "Negative --max-urls rejected"
    log_cmd sitemap -u -1 https://example.com
    run_cmd raku -I "$LIB" "$BIN" -u -1 https://example.com
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--max-urls must be 0 or greater" "names the option"

    log_header "--max-urls-per-file 0 rejected"
    log_cmd sitemap -m 0 https://example.com
    run_cmd raku -I "$LIB" "$BIN" -m 0 https://example.com
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--max-urls-per-file must be 1 or greater" "names the option"

    log_header "--concurrency 0 rejected"
    log_cmd sitemap -c 0 https://example.com
    run_cmd raku -I "$LIB" "$BIN" -c 0 https://example.com
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--concurrency must be 1 or greater" "names the option"

    log_header "--max-redirects 0 rejected"
    log_cmd sitemap --max-redirects=0 https://example.com
    run_cmd raku -I "$LIB" "$BIN" --max-redirects=0 https://example.com
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--max-redirects must be 1 or greater" "names the option"
}

test_url_validation() {
    log_header "Reject ftp:// scheme"
    log_cmd sitemap ftp://example.com
    run_cmd raku -I "$LIB" "$BIN" ftp://example.com
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid URL" "rejects ftp"

    log_header "Reject mailto: scheme"
    log_cmd sitemap mailto:user@example.com
    run_cmd raku -I "$LIB" "$BIN" mailto:user@example.com
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid URL" "rejects mailto"

    log_header "Reject out-of-range port"
    log_cmd sitemap example.com:99999
    run_cmd raku -I "$LIB" "$BIN" example.com:99999
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid URL" "rejects port 99999"

    log_header "Reject http:// with empty host"
    log_cmd sitemap http://
    run_cmd raku -I "$LIB" "$BIN" http://
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid URL" "rejects empty host"

    log_header "Reject hostname with underscore"
    log_cmd sitemap example_com
    run_cmd raku -I "$LIB" "$BIN" example_com
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid URL" "rejects underscore hostname"

    log_header "Accept localhost (single token)"
    log_cmd sitemap localhost -o $TMPDIR_TEST/loc-test.xml
    run_cmd raku -I "$LIB" "$BIN" localhost -o "$TMPDIR_TEST/loc-test.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    # Will fail to connect, but should not fail URL validation
    assert_not_contains "$_ERR" "Invalid URL" "localhost accepted as URL"
    rm -f "$TMPDIR_TEST/loc-test.xml"

    log_header "Accept host:port form"
    log_cmd sitemap 127.0.0.1:9/ -o $TMPDIR_TEST/hp-test.xml
    run_cmd raku -I "$LIB" "$BIN" "127.0.0.1:9/" -o "$TMPDIR_TEST/hp-test.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_not_contains "$_ERR" "Invalid URL" "host:port accepted as URL"
    rm -f "$TMPDIR_TEST/hp-test.xml"
}

test_crawl() {
    start_test_server "$CRAWL_PORT" "crawl"

    log_header "Basic crawl"
    log_cmd sitemap http://127.0.0.1:$CRAWL_PORT/index
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -o "$TMPDIR_TEST/crawl-basic.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_ERR" "Sitemap written to" "writes output file"
    rm -f "$TMPDIR_TEST/crawl-basic.xml"

    log_header "Crawl -o output flag"
    log_cmd sitemap http://.../index -o crawl-custom.xml
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -o "$TMPDIR_TEST/crawl-custom.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    [ -f "$TMPDIR_TEST/crawl-custom.xml" ] && log_result "PASS" "output file exists" || log_result "FAIL" "output file missing"
    TOTAL=$((TOTAL + 1))
    rm -f "$TMPDIR_TEST/crawl-custom.xml"

    log_header "Crawl --max-depth 0 (unlimited)"
    log_cmd sitemap http://.../index --max-depth 0
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --max-depth 0 --no-respect-robots -o "$TMPDIR_TEST/crawl-depth.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-depth.xml"

    log_header "Crawl --max-depth 1"
    log_cmd sitemap http://.../index --max-depth 1
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --max-depth 1 --no-respect-robots -o "$TMPDIR_TEST/crawl-d1.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-d1.xml"

    log_header "Crawl --max-depth 0 (only start page)"
    log_cmd "sitemap http://.../index --max-depth 0 (via --no-follow)"
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-follow --no-respect-robots -o "$TMPDIR_TEST/crawl-nf.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/crawl-nf.xml" 2>/dev/null)
    assert_not_contains "$content" "/page1" "no-follow prevents link traversal"
    rm -f "$TMPDIR_TEST/crawl-nf.xml"

    log_header "Crawl --max-urls 1"
    log_cmd sitemap http://.../index -u 1
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" -u 1 --no-respect-robots -o "$TMPDIR_TEST/crawl-maxu.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-maxu.xml"

    log_header "Crawl --no-images"
    log_cmd sitemap http://.../with-images --no-images
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/with-images" --no-images --no-respect-robots -o "$TMPDIR_TEST/crawl-noimg.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-noimg.xml"

    log_header "Crawl --no-hreflang"
    log_cmd sitemap http://.../index --no-hreflang
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-hreflang --no-respect-robots -o "$TMPDIR_TEST/crawl-nohref.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-nohref.xml"

    log_header "Crawl --no-priority"
    log_cmd sitemap http://.../index --no-priority
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-priority --no-respect-robots -o "$TMPDIR_TEST/crawl-nopri.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/crawl-nopri.xml" 2>/dev/null)
    assert_not_contains "$content" "<priority>" "no-priority omits priority tags"
    rm -f "$TMPDIR_TEST/crawl-nopri.xml"

    log_header "Crawl --no-lastmod"
    log_cmd sitemap http://.../index --no-lastmod
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-lastmod --no-respect-robots -o "$TMPDIR_TEST/crawl-nolm.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-nolm.xml"

    log_header "Crawl --user-agent"
    log_cmd sitemap http://.../index -a "TestBot/1.0"
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" -a "TestBot/1.0" --no-respect-robots -o "$TMPDIR_TEST/crawl-ua.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-ua.xml"

    log_header "Crawl --verbose"
    log_cmd sitemap http://.../index -v
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" -v --no-respect-robots -o "$TMPDIR_TEST/crawl-verb.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Starting sitemap generation" "verbose shows start message"
    rm -f "$TMPDIR_TEST/crawl-verb.xml"

    log_header "Crawl --debug"
    log_cmd sitemap http://.../index --debug
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --debug --no-respect-robots -o "$TMPDIR_TEST/crawl-debug.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-debug.xml"

    log_header "Crawl --no-respect-robots"
    log_cmd sitemap http://.../index --no-respect-robots
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -o "$TMPDIR_TEST/crawl-norobots.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-norobots.xml"

    log_header "Crawl --no-verify-ssl"
    log_cmd sitemap http://.../index --no-verify-ssl
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-verify-ssl --no-respect-robots -o "$TMPDIR_TEST/crawl-nossl.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_contains "$_ERR" "SSL verification disabled" "warns about SSL"
    rm -f "$TMPDIR_TEST/crawl-nossl.xml"

    log_header "Crawl --exclude-dirs /admin"
    log_cmd sitemap http://.../index --exclude-dirs /admin
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --exclude-dirs /admin --no-respect-robots -o "$TMPDIR_TEST/crawl-excl.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/crawl-excl.xml" 2>/dev/null)
    assert_not_contains "$content" "/admin/secret" "/admin excluded"
    rm -f "$TMPDIR_TEST/crawl-excl.xml"

    log_header "Crawl --exclude-extensions csv"
    log_cmd sitemap http://.../index --exclude-extensions csv
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --exclude-extensions csv --no-respect-robots -o "$TMPDIR_TEST/crawl-extrepl.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-extrepl.xml"

    log_header "Crawl --extra-exclude-extensions pdf"
    log_cmd sitemap http://.../index --extra-exclude-extensions pdf
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --extra-exclude-extensions pdf --no-respect-robots -o "$TMPDIR_TEST/crawl-extadd.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-extadd.xml"

    log_header "Crawl -z (compress)"
    log_cmd sitemap http://.../index -z
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -z -o "$TMPDIR_TEST/crawl-gz.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-gz.xml" "$TMPDIR_TEST/crawl-gz.xml.gz"

    log_header "Crawl --no-pretty"
    log_cmd sitemap http://.../index --no-pretty
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots --no-pretty -o "$TMPDIR_TEST/crawl-nopretty.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-nopretty.xml"

    log_header "Crawl --xsl-url"
    log_cmd sitemap http://.../index --xsl-url
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots --xsl-url "https://example.com/sitemap.xsl" -o "$TMPDIR_TEST/crawl-xsl.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/crawl-xsl.xml" 2>/dev/null)
    assert_contains "$content" "xml-stylesheet" "XSL PI present"
    rm -f "$TMPDIR_TEST/crawl-xsl.xml"

    log_header "Crawl --format txt"
    log_cmd sitemap http://.../index -f txt
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -f txt -o "$TMPDIR_TEST/crawl.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl.txt"

    log_header "Crawl --format html"
    log_cmd sitemap http://.../index -f html
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -f html -o "$TMPDIR_TEST/crawl.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl.html"

    log_header "Crawl --format rss"
    log_cmd sitemap http://.../index -f rss
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -f rss -o "$TMPDIR_TEST/crawl.rss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl.rss"

    log_header "Crawl --format atom"
    log_cmd sitemap http://.../index -f atom
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -f atom -o "$TMPDIR_TEST/crawl.atom"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl.atom"

    log_header "Crawl --format mrss"
    log_cmd sitemap http://.../index -f mrss
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -f mrss -o "$TMPDIR_TEST/crawl.mrss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl.mrss"

    log_header "Crawl -a -Bot (option-looking value)"
    log_cmd sitemap http://.../index -a -Bot
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" --no-respect-robots -a -Bot -o "$TMPDIR_TEST/crawl-bot.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/crawl-bot.xml"

    log_header "Crawl -a --no-respect-robots rejected (ambiguous)"
    log_cmd sitemap http://.../index -a --no-respect-robots
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" -a --no-respect-robots
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "requires a value" "rejects ambiguous string+option"

    log_header "Crawl respects robots.txt (blocked paths excluded)"
    # Our test server allows /page1, disallows /admin/ — crawl from index
    # and verify robots.txt disallow is respected
    log_cmd "sitemap http://.../index (with robots respected)"
    run_cmd raku -I "$LIB" "$BIN" "http://127.0.0.1:$CRAWL_PORT/index" -o "$TMPDIR_TEST/crawl-robots.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/crawl-robots.xml" 2>/dev/null)
    assert_not_contains "$content" "/admin/secret" "robots.txt blocks /admin/"
    rm -f "$TMPDIR_TEST/crawl-robots.xml"

    stop_test_server "$CRAWL_PORT"
}

test_build() {
    log_header "Build from TXT file"
    make_txt_file "$TMPDIR_TEST/b1.txt" "https://example.com/a" "https://example.com/b"
    log_cmd sitemap build b1.txt
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -o "$TMPDIR_TEST/b1.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/b1.xml" 2>/dev/null)
    assert_contains "$content" "example.com/a" "URL present in output"
    rm -f "$TMPDIR_TEST/b1.xml"

    log_header "Build from XML sitemap"
    make_sitemap_xml "$TMPDIR_TEST/b2.xml" "https://example.com/x" "https://example.com/y"
    log_cmd sitemap build b2.xml
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b2.xml" -o "$TMPDIR_TEST/b2-out.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/b2-out.xml" 2>/dev/null)
    assert_contains "$content" "example.com/x" "URL from XML preserved"
    rm -f "$TMPDIR_TEST/b2-out.xml"

    log_header "Build from RSS feed"
    make_rss_xml "$TMPDIR_TEST/b3.rss" "https://example.com/r1" "https://example.com/r2"
    log_cmd sitemap build b3.rss
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b3.rss" -o "$TMPDIR_TEST/b3-out.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/b3-out.xml" 2>/dev/null)
    assert_contains "$content" "example.com/r1" "RSS item URL present"
    rm -f "$TMPDIR_TEST/b3-out.xml"

    log_header "Build from Atom feed"
    make_atom_xml "$TMPDIR_TEST/b4.atom" "https://example.com/a1" "https://example.com/a2"
    log_cmd sitemap build b4.atom
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b4.atom" -o "$TMPDIR_TEST/b4-out.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/b4-out.xml" 2>/dev/null)
    assert_contains "$content" "example.com/a1" "Atom entry URL present"
    rm -f "$TMPDIR_TEST/b4-out.xml"

    log_header "Build from MRSS feed"
    make_mrss_xml "$TMPDIR_TEST/b5.mrss" "https://example.com/v1"
    log_cmd sitemap build b5.mrss
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b5.mrss" -o "$TMPDIR_TEST/b5-out.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/b5-out.xml"

    log_header "Build from HTML file"
    make_html_file "$TMPDIR_TEST/b6.html" "https://example.com/h1" "https://example.com/h2"
    log_cmd sitemap build b6.html
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b6.html" -o "$TMPDIR_TEST/b6-out.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/b6-out.xml" 2>/dev/null)
    assert_contains "$content" "example.com/h1" "HTML link URL present"
    rm -f "$TMPDIR_TEST/b6-out.xml"

    log_header "Build --base-url resolves relative URLs"
    echo "/about" > "$TMPDIR_TEST/b7.txt"
    echo "https://x.com/y" >> "$TMPDIR_TEST/b7.txt"
    log_cmd sitemap build b7.txt --base-url https://e.com/
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b7.txt" --base-url "https://e.com/" -o "$TMPDIR_TEST/b7-out.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/b7-out.xml" 2>/dev/null)
    assert_contains "$content" "https://e.com/about" "relative URL resolved"
    assert_contains "$content" "https://x.com/y" "absolute URL preserved"
    rm -f "$TMPDIR_TEST/b7-out.xml"

    log_header "Build --format txt"
    log_cmd sitemap build b1.txt -f txt
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -f txt -o "$TMPDIR_TEST/b8.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/b8.txt"

    log_header "Build --format html"
    log_cmd sitemap build b1.txt -f html
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -f html -o "$TMPDIR_TEST/b8.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/b8.html"

    log_header "Build --format rss"
    log_cmd sitemap build b1.txt -f rss --base-url https://example.com
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -f rss --base-url "https://example.com" -o "$TMPDIR_TEST/b8.rss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/b8.rss" 2>/dev/null)
    assert_contains "$content" "<link>https://example.com</link>" "RSS feed link populated"
    rm -f "$TMPDIR_TEST/b8.rss"

    log_header "Build --format atom"
    log_cmd sitemap build b1.txt -f atom --base-url https://example.com
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -f atom --base-url "https://example.com" -o "$TMPDIR_TEST/b8.atom"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/b8.atom"

    log_header "Build --format mrss"
    log_cmd sitemap build b1.txt -f mrss --base-url https://example.com
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -f mrss --base-url "https://example.com" -o "$TMPDIR_TEST/b8.mrss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/b8.mrss"

    log_header "Build --compress"
    log_cmd sitemap build b1.txt -z
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -z -o "$TMPDIR_TEST/b9.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/b9.xml" "$TMPDIR_TEST/b9.xml.gz"

    log_header "Build --no-pretty"
    log_cmd sitemap build b1.txt --no-pretty
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" --no-pretty -o "$TMPDIR_TEST/b10.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/b10.xml"

    log_header "Build --verbose"
    log_cmd sitemap build b1.txt -v
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -v -o "$TMPDIR_TEST/b11.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Building sitemap from" "verbose shows source"
    rm -f "$TMPDIR_TEST/b11.xml"

    log_header "Build missing file"
    log_cmd sitemap build nonexistent.txt
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/nonexistent.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "File not found" "error names the missing file"

    log_header "Build from URL"
    start_test_server "$BUILD_PORT" "build"
    log_cmd sitemap build http://127.0.0.1:$BUILD_PORT/sitemap.xml
    run_cmd raku -I "$LIB" "$BIN" build "http://127.0.0.1:$BUILD_PORT/sitemap.xml" --no-verify-ssl -o "$TMPDIR_TEST/burl.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/burl.xml" 2>/dev/null)
    assert_contains "$content" "example.com" "remote sitemap items present"
    rm -f "$TMPDIR_TEST/burl.xml"
    stop_test_server "$BUILD_PORT"

    log_header "Build -F shorthand"
    log_cmd sitemap build b1.txt -F html
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -F html -o "$TMPDIR_TEST/bf.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/bf.html"

    log_header "Build -F=xml shorthand"
    log_cmd sitemap build b1.txt -F=xml
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/b1.txt" -F=xml -o "$TMPDIR_TEST/bf2.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/bf2.xml"
}

test_parse() {
    log_header "Parse TXT file"
    make_txt_file "$TMPDIR_TEST/p1.txt" "https://example.com/a" "https://example.com/b"
    log_cmd sitemap parse p1.txt
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p1.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/a" "URL printed"
    assert_contains "$_OUT" "Total URLs: 2" "total count correct"

    log_header "Parse XML sitemap"
    make_sitemap_xml "$TMPDIR_TEST/p2.xml" "https://example.com/x" "https://example.com/y"
    log_cmd sitemap parse p2.xml
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p2.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/x" "URL printed"
    assert_contains "$_OUT" "Total URLs: 2" "total count correct"

    log_header "Parse XML sitemap --verbose (shows lastmod)"
    make_sitemap_xml "$TMPDIR_TEST/p2v.xml" "https://example.com/z"
    # Modify to add lastmod
    cat > "$TMPDIR_TEST/p2v.xml" <<'EOF'
<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url><loc>https://example.com/z</loc><lastmod>2024-01-15</lastmod></url>
</urlset>
EOF
    log_cmd sitemap parse p2v.xml -v
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p2v.xml" -v
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Lastmod:" "verbose shows lastmod"

    log_header "Parse RSS feed"
    make_rss_xml "$TMPDIR_TEST/p3.rss" "https://example.com/r1"
    log_cmd sitemap parse p3.rss
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p3.rss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/r1" "RSS item URL printed"

    log_header "Parse Atom feed"
    make_atom_xml "$TMPDIR_TEST/p4.atom" "https://example.com/a1"
    log_cmd sitemap parse p4.atom
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p4.atom"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/a1" "Atom entry URL printed"

    log_header "Parse MRSS feed"
    make_mrss_xml "$TMPDIR_TEST/p5.mrss" "https://example.com/v1"
    log_cmd sitemap parse p5.mrss
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p5.mrss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/v1" "MRSS item URL printed"

    log_header "Parse HTML file"
    make_html_file "$TMPDIR_TEST/p6.html" "https://example.com/h1"
    log_cmd sitemap parse p6.html
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p6.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/h1" "HTML link URL printed"

    log_header "Parse missing file"
    log_cmd sitemap parse nonexistent.xml
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/nonexistent.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "File not found" "error names the missing file"

    log_header "Parse --recursive (HTTP index)"
    start_test_server "$PARSE_PORT" "parse"
    log_cmd sitemap parse http://127.0.0.1:$PARSE_PORT/sitemap-index.xml --recursive
    run_cmd raku -I "$LIB" "$BIN" parse "http://127.0.0.1:$PARSE_PORT/sitemap-index.xml" --recursive
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/page1" "child sitemap URL found"
    assert_contains "$_OUT" "Total URLs: 2" "total count correct"
    stop_test_server "$PARSE_PORT"

    log_header "Parse --recursive-depth 0 (unlimited)"
    start_test_server "$PARSE_PORT" "parse-depth"
    log_cmd sitemap parse http://127.0.0.1:$PARSE_PORT/sitemap-index.xml --recursive-depth 0
    run_cmd raku -I "$LIB" "$BIN" parse "http://127.0.0.1:$PARSE_PORT/sitemap-index.xml" --recursive-depth 0
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/page1" "child sitemap URL found via depth 0"
    stop_test_server "$PARSE_PORT"

    log_header "Parse without --recursive stays shallow"
    start_test_server "$PARSE_PORT" "parse-shallow"
    log_cmd "sitemap parse http://127.0.0.1:$PARSE_PORT/sitemap-index.xml (no --recursive)"
    run_cmd raku -I "$LIB" "$BIN" parse "http://127.0.0.1:$PARSE_PORT/sitemap-index.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "No URLs found" "shallow parse fails on index"
    stop_test_server "$PARSE_PORT"

    log_header "Parse recursive-depth -1 (not supplied sentinel)"
    make_sitemap_xml "$TMPDIR_TEST/p-depth1.xml" "https://example.com/d1"
    log_cmd sitemap parse p-depth1.xml --recursive-depth -1
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p-depth1.xml" --recursive-depth -1
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Total URLs: 1" "parses the URL"

    log_header "Parse recursive XML file"
    make_sitemap_xml "$TMPDIR_TEST/p-child.xml" "https://example.com/child1"
    log_cmd sitemap parse p-child.xml --recursive
    run_cmd raku -I "$LIB" "$BIN" parse "$TMPDIR_TEST/p-child.xml" --recursive
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "example.com/child1" "single URL parsed via recursive"
}

test_fetch() {
    start_test_server "$FETCH_PORT" "fetch"

    log_header "Fetch single sitemap"
    log_cmd sitemap fetch http://127.0.0.1:$FETCH_PORT/sitemap.xml
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap.xml" --no-verify-ssl -o "$TMPDIR_TEST/f1.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/f1.xml" 2>/dev/null)
    assert_contains "$content" "example.com" "fetched sitemap has URLs"
    rm -f "$TMPDIR_TEST/f1.xml"

    log_header "Fetch with --verbose"
    log_cmd sitemap fetch http://.../sitemap.xml -v
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap.xml" --no-verify-ssl -v -o "$TMPDIR_TEST/f2.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/f2.xml"

    log_header "Fetch with --compress"
    log_cmd sitemap fetch http://.../sitemap.xml -z
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap.xml" --no-verify-ssl -z -o "$TMPDIR_TEST/f3.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/f3.xml" "$TMPDIR_TEST/f3.xml.gz"

    log_header "Fetch --format txt"
    log_cmd sitemap fetch http://.../sitemap.xml -f txt
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap.xml" --no-verify-ssl -f txt -o "$TMPDIR_TEST/f4.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/f4.txt"

    log_header "Fetch --format html"
    log_cmd sitemap fetch http://.../sitemap.xml -f html
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap.xml" --no-verify-ssl -f html -o "$TMPDIR_TEST/f4.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/f4.html"

    log_header "Fetch --format rss"
    log_cmd sitemap fetch http://.../sitemap.xml -f rss
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap.xml" --no-verify-ssl -f rss -o "$TMPDIR_TEST/f4.rss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/f4.rss"

    # Recursive fetch writes its derived index/dir via validate-output-dir,
    # which allows an absolute -o as long as it resolves inside the CWD (the
    # test runs from the project root, so $TMPDIR_TEST qualifies).
    local fout="$TMPDIR_TEST/f-recursive"

    log_header "Fetch --recursive (sitemap index)"
    log_cmd sitemap fetch http://.../sitemap-index.xml --recursive
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap-index.xml" --no-verify-ssl --recursive -o "$fout" -v
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Saved index:" "recursive index saved"
    assert_contains "$_OUT" "Saved:" "child sitemap saved"
    assert_test -f "$fout.xml" "recursive index file on disk"
    assert_test -d "$fout" "recursive children dir on disk"
    rm -rf "$fout"* "$fout"

    log_header "Fetch --recursive --force"
    mkdir -p "$fout"
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap-index.xml" --no-verify-ssl --recursive --force -o "$fout" -v
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Saved index:" "recursive index saved (force)"
    assert_contains "$_OUT" "Saved:" "child sitemap saved (force)"
    assert_test -f "$fout.xml" "recursive index file on disk (force)"
    assert_test -d "$fout" "recursive children dir on disk (force)"
    rm -rf "$fout"* "$fout"

    log_header "Fetch --compress + --recursive rejected"
    log_cmd sitemap fetch http://.../sitemap.xml -z --recursive
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:1/sitemap.xml" -z --recursive
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--compress is not supported with --recursive" "clear error"

    log_header "Fetch missing URL"
    log_cmd sitemap fetch
    run_cmd raku -I "$LIB" "$BIN" fetch
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "fetch requires a URL" "error names the missing arg"

    log_header "Fetch discovery failure"
    # A server with no robots.txt and no sitemap.xml (404 for everything)
    local dport=$((FETCH_PORT + 100))
    local dserver="$TMPDIR_TEST/server-404.py"
    cat > "$dserver" <<PYEOF
import http.server, sys
PORT = $dport
class H(http.server.BaseHTTPRequestHandler):
    def log_message(self, fmt, *a): pass
    def do_GET(self):
        body = "Not Found"
        self.send_response(404)
        self.send_header('Content-Length', len(body.encode()))
        self.end_headers()
        self.wfile.write(body.encode())
s = http.server.HTTPServer(('127.0.0.1', PORT), H)
s.serve_forever()
PYEOF
    python3 "$dserver" &
    local dpid=$!
    echo "$dpid" > "$TMPDIR_TEST/server-${dport}.pid"
    local tries=0
    while ! (echo > /dev/tcp/127.0.0.1/$dport) 2>/dev/null; do
        sleep 0.1
        tries=$((tries + 1))
        if [ $tries -gt 50 ]; then break; fi
    done
    log_cmd sitemap fetch http://127.0.0.1:$dport/ --no-verify-ssl
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$dport/" --no-verify-ssl
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Error:" "reports discovery failure"
    kill "$dpid" 2>/dev/null; wait "$dpid" 2>/dev/null
    rm -f "$TMPDIR_TEST/server-${dport}.pid"

    log_header "Fetch .xml?query=1 direct detection"
    log_cmd sitemap fetch http://.../sitemap.xml?v=1
    run_cmd raku -I "$LIB" "$BIN" fetch "http://127.0.0.1:$FETCH_PORT/sitemap.xml?v=1" --no-verify-ssl -o "$TMPDIR_TEST/fq.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/fq.xml" 2>/dev/null)
    assert_contains "$content" "example.com" "query-string URL fetched directly"
    rm -f "$TMPDIR_TEST/fq.xml"

    stop_test_server "$FETCH_PORT"
}

test_convert() {
    log_header "Convert XML to TXT"
    make_sitemap_xml "$TMPDIR_TEST/c1.xml" "https://example.com/a" "https://example.com/b"
    log_cmd sitemap convert c1.xml -f txt
    run_cmd raku -I "$LIB" "$BIN" convert "$TMPDIR_TEST/c1.xml" -f txt -o "$TMPDIR_TEST/c1.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/c1.txt" 2>/dev/null)
    assert_contains "$content" "example.com/a" "URL in txt output"
    rm -f "$TMPDIR_TEST/c1.txt"

    log_header "Convert XML to HTML"
    log_cmd sitemap convert c1.xml -f html
    run_cmd raku -I "$LIB" "$BIN" convert "$TMPDIR_TEST/c1.xml" -f html -o "$TMPDIR_TEST/c1.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/c1.html"

    log_header "Convert XML to RSS"
    log_cmd sitemap convert c1.xml -f rss
    run_cmd raku -I "$LIB" "$BIN" convert "$TMPDIR_TEST/c1.xml" -f rss -o "$TMPDIR_TEST/c1.rss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/c1.rss"

    log_header "Convert XML to Atom"
    log_cmd sitemap convert c1.xml -f atom
    run_cmd raku -I "$LIB" "$BIN" convert "$TMPDIR_TEST/c1.xml" -f atom -o "$TMPDIR_TEST/c1.atom"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/c1.atom"

    log_header "Convert XML to MRSS"
    log_cmd sitemap convert c1.xml -f mrss
    run_cmd raku -I "$LIB" "$BIN" convert "$TMPDIR_TEST/c1.xml" -f mrss -o "$TMPDIR_TEST/c1.mrss"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/c1.mrss"

    log_header "Convert with --compress"
    log_cmd sitemap convert c1.xml -f txt -z
    run_cmd raku -I "$LIB" "$BIN" convert "$TMPDIR_TEST/c1.xml" -f txt -z -o "$TMPDIR_TEST/c2.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/c2.txt" "$TMPDIR_TEST/c2.txt.gz"

    log_header "Convert with --verbose"
    log_cmd sitemap convert c1.xml -f txt -v
    run_cmd raku -I "$LIB" "$BIN" convert "$TMPDIR_TEST/c1.xml" -f txt -v -o "$TMPDIR_TEST/c3.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Converting" "verbose shows conversion message"
    assert_contains "$_OUT" "Total URLs:" "verbose shows total"
    rm -f "$TMPDIR_TEST/c3.txt"

    log_header "Convert missing file"
    log_cmd sitemap convert nonexistent.xml
    run_cmd raku -I "$LIB" "$BIN" convert "$TMPDIR_TEST/nonexistent.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "File not found" "error names missing file"

    log_header "Convert missing file argument"
    log_cmd sitemap convert
    run_cmd raku -I "$LIB" "$BIN" convert
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "convert requires a file" "error names missing arg"
}

test_dir() {
    log_header "Dir basic scan"
    mkdir -p "$TMPDIR_TEST/d1"
    echo '<html><body><a href="page.html">P</a></body></html>' > "$TMPDIR_TEST/d1/index.html"
    echo '<html><body>Page</body></html>' > "$TMPDIR_TEST/d1/page.html"
    log_cmd sitemap dir d1
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d1" --base-url "https://example.com" -o "$TMPDIR_TEST/d1.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/d1.xml" 2>/dev/null)
    # Sitemap::DirScanner maps index.html/index.htm to its directory root,
    # matching a web server's default-document behavior.
    assert_contains "$content" "https://example.com/" "index resolved to root"
    assert_contains "$content" "example.com/page.html" "page.html in sitemap"
    rm -f "$TMPDIR_TEST/d1.xml"
    rm -rf "$TMPDIR_TEST/d1"

    log_header "Dir --max-depth 0 (unlimited)"
    mkdir -p "$TMPDIR_TEST/d2/sub"
    echo '<html><body><a href="sub/child.html">C</a></body></html>' > "$TMPDIR_TEST/d2/index.html"
    echo '<html><body>Child</body></html>' > "$TMPDIR_TEST/d2/sub/child.html"
    log_cmd sitemap dir d2 --max-depth 0
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d2" --base-url "https://example.com" --max-depth 0 -o "$TMPDIR_TEST/d2.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/d2.xml" 2>/dev/null)
    assert_contains "$content" "sub/child.html" "child found at depth 1"
    rm -f "$TMPDIR_TEST/d2.xml"
    rm -rf "$TMPDIR_TEST/d2"

    log_header "Dir --max-depth 1 (only top level)"
    mkdir -p "$TMPDIR_TEST/d3/sub"
    echo '<html><body><a href="sub/child.html">C</a></body></html>' > "$TMPDIR_TEST/d3/index.html"
    echo '<html><body>Child</body></html>' > "$TMPDIR_TEST/d3/sub/child.html"
    log_cmd sitemap dir d3 --max-depth 1
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d3" --base-url "https://example.com" --max-depth 1 -o "$TMPDIR_TEST/d3.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d3.xml"
    rm -rf "$TMPDIR_TEST/d3"

    log_header "Dir --max-urls 1"
    mkdir -p "$TMPDIR_TEST/d4"
    echo '<html><body><a href="a.html">A</a><a href="b.html">B</a></body></html>' > "$TMPDIR_TEST/d4/index.html"
    echo '<html><body>A</body></html>' > "$TMPDIR_TEST/d4/a.html"
    echo '<html><body>B</body></html>' > "$TMPDIR_TEST/d4/b.html"
    log_cmd sitemap dir d4 -u 1
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d4" --base-url "https://example.com" -u 1 -o "$TMPDIR_TEST/d4.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d4.xml"
    rm -rf "$TMPDIR_TEST/d4"

    log_header "Dir --format txt"
    mkdir -p "$TMPDIR_TEST/d5"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d5/index.html"
    log_cmd sitemap dir d5 -f txt
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d5" --base-url "https://example.com" -f txt -o "$TMPDIR_TEST/d5.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d5.txt"
    rm -rf "$TMPDIR_TEST/d5"

    log_header "Dir --format html"
    mkdir -p "$TMPDIR_TEST/d6"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d6/index.html"
    log_cmd sitemap dir d6 -f html
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d6" --base-url "https://example.com" -f html -o "$TMPDIR_TEST/d6.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d6.html"
    rm -rf "$TMPDIR_TEST/d6"

    log_header "Dir --compress"
    mkdir -p "$TMPDIR_TEST/d7"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d7/index.html"
    log_cmd sitemap dir d7 -z
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d7" --base-url "https://example.com" -z -o "$TMPDIR_TEST/d7.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d7.xml" "$TMPDIR_TEST/d7.xml.gz"
    rm -rf "$TMPDIR_TEST/d7"

    log_header "Dir --no-pretty"
    mkdir -p "$TMPDIR_TEST/d8"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d8/index.html"
    log_cmd sitemap dir d8 --no-pretty
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d8" --base-url "https://example.com" --no-pretty -o "$TMPDIR_TEST/d8.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d8.xml"
    rm -rf "$TMPDIR_TEST/d8"

    log_header "Dir --xsl-url"
    mkdir -p "$TMPDIR_TEST/d9"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d9/index.html"
    log_cmd sitemap dir d9 --xsl-url
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d9" --base-url "https://example.com" --xsl-url "https://example.com/sitemap.xsl" -o "$TMPDIR_TEST/d9.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/d9.xml" 2>/dev/null)
    assert_contains "$content" "xml-stylesheet" "XSL PI present"
    rm -f "$TMPDIR_TEST/d9.xml"
    rm -rf "$TMPDIR_TEST/d9"

    log_header "Dir --verbose"
    mkdir -p "$TMPDIR_TEST/d10"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d10/index.html"
    log_cmd sitemap dir d10 -v
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d10" --base-url "https://example.com" -v -o "$TMPDIR_TEST/d10.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Crawling directory" "verbose shows crawling message"
    rm -f "$TMPDIR_TEST/d10.xml"
    rm -rf "$TMPDIR_TEST/d10"

    log_header "Dir --no-images"
    mkdir -p "$TMPDIR_TEST/d11"
    echo '<html><body><img src="img.png"></body></html>' > "$TMPDIR_TEST/d11/index.html"
    log_cmd sitemap dir d11 --no-images
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d11" --base-url "https://example.com" --no-images -o "$TMPDIR_TEST/d11.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d11.xml"
    rm -rf "$TMPDIR_TEST/d11"

    log_header "Dir --videos"
    mkdir -p "$TMPDIR_TEST/d12"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d12/index.html"
    log_cmd sitemap dir d12 --videos
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d12" --base-url "https://example.com" --videos -o "$TMPDIR_TEST/d12.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d12.xml"
    rm -rf "$TMPDIR_TEST/d12"

    log_header "Dir --no-priority"
    mkdir -p "$TMPDIR_TEST/d13"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d13/index.html"
    log_cmd sitemap dir d13 --no-priority
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d13" --base-url "https://example.com" --no-priority -o "$TMPDIR_TEST/d13.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/d13.xml" 2>/dev/null)
    assert_not_contains "$content" "<priority>" "no-priority omits priority tags"
    rm -f "$TMPDIR_TEST/d13.xml"
    rm -rf "$TMPDIR_TEST/d13"

    log_header "Dir --no-hreflang"
    mkdir -p "$TMPDIR_TEST/d14"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d14/index.html"
    log_cmd sitemap dir d14 --no-hreflang
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d14" --base-url "https://example.com" --no-hreflang -o "$TMPDIR_TEST/d14.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/d14.xml"
    rm -rf "$TMPDIR_TEST/d14"

    log_header "Dir --news (with NewsArticle JSON-LD)"
    mkdir -p "$TMPDIR_TEST/d15"
    local stamp
    stamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
    cat > "$TMPDIR_TEST/d15/index.html" <<HTMLEOF
<html><head><script type="application/ld+json">
{ "@type": "NewsArticle", "headline": "Test News", "datePublished": "$stamp",
  "publisher": { "@type": "Organization", "name": "Test Pub" } }
</script></head><body>news</body></html>
HTMLEOF
    log_cmd sitemap dir d15 --news
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d15" --base-url "https://example.com" --news -o "$TMPDIR_TEST/d15.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "-news.xml" "news sitemap file written"
    rm -f "$TMPDIR_TEST/d15.xml" "$TMPDIR_TEST/d15-news.xml"
    rm -rf "$TMPDIR_TEST/d15"

    log_header "Dir missing directory"
    log_cmd sitemap dir /nonexistent/dir
    run_cmd raku -I "$LIB" "$BIN" dir "/nonexistent/dir"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Directory not found" "error names missing dir"

    log_header "Dir --base-url must be http(s)"
    mkdir -p "$TMPDIR_TEST/d16"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d16/index.html"
    log_cmd sitemap dir d16 --base-url file:///tmp
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d16" --base-url "file:///tmp"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "must start with http:// or https://" "rejects non-http base-url"
    rm -rf "$TMPDIR_TEST/d16"

    log_header "Dir --max-depth -1 rejected"
    mkdir -p "$TMPDIR_TEST/d17"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d17/index.html"
    log_cmd sitemap dir d17 -d -1
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d17" -d -1
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--max-depth must be 0 or greater" "names the option"
    rm -rf "$TMPDIR_TEST/d17"

    log_header "Dir --max-urls -1 rejected"
    mkdir -p "$TMPDIR_TEST/d18"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d18/index.html"
    log_cmd sitemap dir d18 -u -1
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d18" -u -1
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--max-urls must be 0 or greater" "names the option"
    rm -rf "$TMPDIR_TEST/d18"

    log_header "Dir --max-urls-per-file 0 rejected"
    mkdir -p "$TMPDIR_TEST/d19"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d19/index.html"
    log_cmd sitemap dir d19 -m 0
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d19" -m 0
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--max-urls-per-file must be 1 or greater" "names the option"
    rm -rf "$TMPDIR_TEST/d19"

    log_header "Dir --concurrency 0 rejected"
    mkdir -p "$TMPDIR_TEST/d20"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d20/index.html"
    log_cmd sitemap dir d20 -c 0
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d20" -c 0
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--concurrency must be 1 or greater" "names the option"
    rm -rf "$TMPDIR_TEST/d20"

    log_header "Dir RSS/Atom without --base-url warns"
    mkdir -p "$TMPDIR_TEST/d21"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d21/index.html"
    log_cmd "sitemap dir d21 -f atom (no --base-url)"
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d21" -f atom -o "$TMPDIR_TEST/d21.atom"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_contains "$_ERR" "no --base-url" "warns about missing base-url"
    rm -f "$TMPDIR_TEST/d21.atom"
    rm -rf "$TMPDIR_TEST/d21"

    log_header "Dir RSS/Atom with --base-url no warning"
    mkdir -p "$TMPDIR_TEST/d22"
    echo '<html><body>Hi</body></html>' > "$TMPDIR_TEST/d22/index.html"
    log_cmd sitemap dir d22 -f atom --base-url
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d22" -f atom --base-url "https://example.com" -o "$TMPDIR_TEST/d22.atom"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_not_contains "$_ERR" "no --base-url" "no warning when base-url provided"
    rm -f "$TMPDIR_TEST/d22.atom"
    rm -rf "$TMPDIR_TEST/d22"

    log_header "Dir --news with non-XML format warns"
    mkdir -p "$TMPDIR_TEST/d23"
    local stamp
    stamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
    cat > "$TMPDIR_TEST/d23/index.html" <<HTMLEOF
<html><head><script type="application/ld+json">
{ "@type": "NewsArticle", "headline": "Test", "datePublished": "$stamp",
  "publisher": { "@type": "Organization", "name": "Pub" } }
</script></head><body>news</body></html>
HTMLEOF
    log_cmd sitemap dir d23 --news -f html
    run_cmd raku -I "$LIB" "$BIN" dir "$TMPDIR_TEST/d23" --base-url "https://example.com" --news -f html -o "$TMPDIR_TEST/d23.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_contains "$_ERR" "ignoring it for format" "warns news only applies to xml"
    rm -f "$TMPDIR_TEST/d23.html"
    rm -rf "$TMPDIR_TEST/d23"
}

test_tree() {
    log_header "Tree from JSON"
    log_cmd sitemap tree tree.json
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.json" -o "$TMPDIR_TEST/t1.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/t1.xml" 2>/dev/null)
    assert_contains "$content" "https://example.com/about/team" "nested URL present"
    assert_contains "$content" "https://example.com/products/widget/widget-pro" "deeply nested URL present"
    rm -f "$TMPDIR_TEST/t1.xml"

    log_header "Tree from YAML"
    log_cmd sitemap tree tree.yml
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.yml" -o "$TMPDIR_TEST/t2.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/t2.xml" 2>/dev/null)
    assert_contains "$content" "https://example.com/about/team" "nested URL present from YAML"
    rm -f "$TMPDIR_TEST/t2.xml"

    log_header "Tree --format txt"
    log_cmd sitemap tree tree.json -f txt
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.json" -f txt -o "$TMPDIR_TEST/t3.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/t3.txt"

    log_header "Tree --format html"
    log_cmd sitemap tree tree.json -f html
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.json" -f html -o "$TMPDIR_TEST/t4.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/t4.html"

    log_header "Tree --compress"
    log_cmd sitemap tree tree.json -z
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.json" -z -o "$TMPDIR_TEST/t5.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/t5.xml" "$TMPDIR_TEST/t5.xml.gz"

    log_header "Tree --no-pretty"
    log_cmd sitemap tree tree.json --no-pretty
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.json" --no-pretty -o "$TMPDIR_TEST/t6.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    rm -f "$TMPDIR_TEST/t6.xml"

    log_header "Tree --xsl-url"
    log_cmd sitemap tree tree.json --xsl-url
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.json" --xsl-url "https://example.com/sitemap.xsl" -o "$TMPDIR_TEST/t7.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/t7.xml" 2>/dev/null)
    assert_contains "$content" "xml-stylesheet" "XSL PI present"
    rm -f "$TMPDIR_TEST/t7.xml"

    log_header "Tree --verbose"
    log_cmd sitemap tree tree.json -v
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.json" -v -o "$TMPDIR_TEST/t8.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_OUT" "Loading tree from" "verbose shows source"
    assert_contains "$_OUT" "Tree structure:" "verbose shows tree"
    rm -f "$TMPDIR_TEST/t8.xml"

    log_header "Tree missing file"
    log_cmd sitemap tree nonexistent.json
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/nonexistent.json"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "File not found" "error names missing file"

    log_header "Tree non-object data rejected"
    echo '[1,2,3]' > "$TMPDIR_TEST/tbad.json"
    log_cmd "sitemap tree tbad.json (array input)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/tbad.json"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "must contain an object" "clean error"
    rm -f "$TMPDIR_TEST/tbad.json"

    log_header "Tree non-string base_url rejected"
    echo '{ "base_url": 123, "pages": [] }' > "$TMPDIR_TEST/tbad2.json"
    log_cmd "sitemap tree tbad2.json (numeric base_url)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/tbad2.json"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "'base_url' must be a non-empty string" "clean error"
    rm -f "$TMPDIR_TEST/tbad2.json"

    log_header "Tree empty stub rejected"
    echo '{ "base_url": "https://example.com", "pages": [{"stub": "", "priority": 1.0}] }' > "$TMPDIR_TEST/tbad3.json"
    log_cmd "sitemap tree tbad3.json (empty stub)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/tbad3.json"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "missing 'stub'" "clean error"
    rm -f "$TMPDIR_TEST/tbad3.json"

    log_header "Tree invalid priority rejected"
    cat > "$TMPDIR_TEST/tbad4.yml" <<'EOF'
base_url: https://example.com
pages:
  - stub: index
    priority: abc
EOF
    log_cmd "sitemap tree tbad4.yml (invalid priority)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/tbad4.yml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid priority" "clean error"
    rm -f "$TMPDIR_TEST/tbad4.yml"

    log_header "Tree string priority coerced"
    cat > "$TMPDIR_TEST/tgood.yml" <<'EOF'
base_url: https://example.com
pages:
  - stub: index
    priority: "0.5"
EOF
    log_cmd "sitemap tree tgood.yml (string priority)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/tgood.yml" -o "$TMPDIR_TEST/tgood.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    local content
    content=$(cat "$TMPDIR_TEST/tgood.xml" 2>/dev/null)
    assert_contains "$content" "<priority>0.5</priority>" "string priority coerced"
    rm -f "$TMPDIR_TEST/tgood.xml" "$TMPDIR_TEST/tgood.yml"

    log_header "Tree invalid changefreq rejected"
    cat > "$TMPDIR_TEST/tbad5.yml" <<'EOF'
base_url: https://example.com
pages:
  - stub: index
    changefreq: bogus
EOF
    log_cmd "sitemap tree tbad5.yml (invalid changefreq)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/tbad5.yml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid changefreq" "clean error"
    rm -f "$TMPDIR_TEST/tbad5.yml"

    log_header "Tree unknown keys warned"
    cat > "$TMPDIR_TEST/twarn.json" <<'EOF'
{
  "base_url": "https://example.com",
  "pages": [
    {"stub": "home", "prority": 0.9, "lastmods": "2024-01-01", "childs": [{"stub": "lost"}]}
  ]
}
EOF
    log_cmd "sitemap tree twarn.json (unknown keys)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/twarn.json" -o "$TMPDIR_TEST/twarn.xml"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    assert_contains "$_ERR" "unknown key" "warns on typo'd keys"
    rm -f "$TMPDIR_TEST/twarn.json" "$TMPDIR_TEST/twarn.xml"

    log_header "Tree invalid lastmod rejected"
    cat > "$TMPDIR_TEST/tbad6.json" <<'EOF'
{
  "base_url": "https://example.com",
  "pages": [{"stub": "home", "lastmod": "not-a-date"}]
}
EOF
    log_cmd "sitemap tree tbad6.json (invalid lastmod)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/tbad6.json"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid lastmod" "clean error"
    rm -f "$TMPDIR_TEST/tbad6.json"

    log_header "Tree invalid priority (out of range)"
    cat > "$TMPDIR_TEST/tbad7.json" <<'EOF'
{
  "base_url": "https://example.com",
  "pages": [{"stub": "home", "priority": 1.5}]
}
EOF
    log_cmd "sitemap tree tbad7.json (priority > 1)"
    run_cmd raku -I "$LIB" "$BIN" tree "$TMPDIR_TEST/tbad7.json"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "Invalid priority" "rejects out of range"
    rm -f "$TMPDIR_TEST/tbad7.json"

    log_header "Tree --max-urls-per-file 0 rejected"
    log_cmd sitemap tree tree.json -m 0
    run_cmd raku -I "$LIB" "$BIN" tree "$SCRIPT_DIR/data/tree.json" -m 0
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 1
    assert_contains "$_ERR" "--max-urls-per-file must be 1 or greater" "names the option"
}

test_compress_streams() {
    log_header "Build txt + gzip round-trip"
    make_txt_file "$TMPDIR_TEST/gz1.txt" "https://example.com/gz-a" "https://example.com/gz-b"
    log_cmd sitemap build gz1.txt -f txt -z
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/gz1.txt" -f txt -z -o "$TMPDIR_TEST/gz1.txt"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    [ -f "$TMPDIR_TEST/gz1.txt.gz" ] && log_result "PASS" ".gz file created" || log_result "FAIL" ".gz file missing"
    TOTAL=$((TOTAL + 1))
    rm -f "$TMPDIR_TEST/gz1.txt" "$TMPDIR_TEST/gz1.txt.gz"

    log_header "Build html + gzip round-trip"
    make_txt_file "$TMPDIR_TEST/gz2.txt" "https://example.com/gz-c"
    log_cmd sitemap build gz2.txt -f html -z
    run_cmd raku -I "$LIB" "$BIN" build "$TMPDIR_TEST/gz2.txt" -f html -z -o "$TMPDIR_TEST/gz2.html"
    log_output "$_OUT" "$_ERR" "$_EXIT"
    assert_exit 0
    [ -f "$TMPDIR_TEST/gz2.html.gz" ] && log_result "PASS" ".gz file created" || log_result "FAIL" ".gz file missing"
    TOTAL=$((TOTAL + 1))
    rm -f "$TMPDIR_TEST/gz2.txt" "$TMPDIR_TEST/gz2.html" "$TMPDIR_TEST/gz2.html.gz"
}

# =========================================================================
# MAIN
# =========================================================================

setup

echo "Starting tests..." | tee -a "$LOG"
echo "" | tee -a "$LOG"

test_help
test_global_validation
test_url_validation
test_crawl
test_build
test_parse
test_fetch
test_convert
test_dir
test_tree
test_compress_streams

cleanup

echo "" | tee -a "$LOG"
echo "============================================" | tee -a "$LOG"
echo "SUMMARY" | tee -a "$LOG"
echo "============================================" | tee -a "$LOG"
echo "Total: $TOTAL" | tee -a "$LOG"
echo -e "${GREEN}Passed: $PASS${NC}" | tee -a "$LOG"
echo -e "${RED}Failed: $FAIL${NC}" | tee -a "$LOG"
echo -e "${YELLOW}Skipped: $SKIP${NC}" | tee -a "$LOG"
echo "" | tee -a "$LOG"
echo "Finished: $(date)" | tee -a "$LOG"

if [ "$FAIL" -gt 0 ]; then
    exit 1
else
    exit 0
fi
