#!/usr/bin/env raku

use v6.d;

use Sitemap;
use Sitemap::Crawler;
use Sitemap::Parser;
use Sitemap::Fetcher;
use Sitemap::Config;
use Sitemap::InputParser;
use Sitemap::Item;
use Sitemap::DirScanner;
use Sitemap::SiteTree;
use Sitemap::Format::TXT;
use Sitemap::Format::HTML;
use Compress::Zlib;
use JSON::Fast;
use YAMLish;

#| Canonical names of every option that takes a value (positional after =).
#| Shared by the BEGIN block (which rewrites `--opt value` into `--opt=value`)
#| and the CHECK block (which validates MAIN signatures at compile time).
my constant %VALUE-OPTS = (
    '-o' => 1, '--output'    => 1,
    '-f' => 1, '-F' => 1, '--format'   => 1,
    '-d' => 1, '--max-depth' => 1,
    '-u' => 1, '--max-urls'  => 1,
    '-m' => 1, '--max-urls-per-file' => 1,
    '-a' => 1, '--user-agent' => 1,
    '-c' => 1, '--concurrency' => 1,
    '--xsl' => 1, '--xsl-url' => 1,
    '--base-url'          => 1,
    '--recursive-depth'   => 1,
    '--exclude-dirs'      => 1,
    '--exclude-extensions' => 1,
    '--extra-exclude-extensions' => 1,
    '--max-images'        => 1,
    '--max-videos'        => 1,
    '--max-redirects'     => 1,
);

#| Every recognised option name (value-taking + Bool flags). Single source of
#| truth: the BEGIN block's absorption guard is DERIVED from this table, so an
#| option added to MAIN and listed here can never go missing from the
#| normalizer again (the drift that previously ate --allow-http-fallback when
#| it appeared in a value slot). The CHECK block uses it to catch undeclared
#| options in MAIN signatures.
my constant %KNOWN-OPTS = %VALUE-OPTS, (
    '-z' => 1, '--compress'   => 1,
    '--pretty'  => 1,
    '-v' => 1, '--verbose'    => 1,
    '--debug'   => 1,
    '-r' => 1, '--respect-robots' => 1,
    '--verify-ssl' => 1, '--ssl-verify' => 1,
    '--allow-http-fallback' => 1,
    '--images'  => 1, '--img' => 1,
    '--follow'  => 1,
    '--hreflang' => 1,
    '--videos'  => 1,
    '--news'    => 1,
    '--lastmod' => 1,
    '--priority' => 1,
    '-h' => 1, '--help'      => 1,
    '--force'   => 1,
    '--follow-foreign-children' => 1,
);

BEGIN {
    # Rewrite `--opt value` into `--opt=value`. Raku's native CLI parsing only
    # binds a separated value to the preceding option when the option comes
    # before the positional (e.g. `-o out.xml https://site` works but
    # `https://site -o out.xml` does not), so normalize everything upfront.
    # Value-taking options are exactly %VALUE-OPTS — kept in one place above.
    my %takes-value = %VALUE-OPTS.keys.map: { $_ => True };

    # The help text advertises `--xsl`, but every multi MAIN declares the
    # option as `--xsl-url`; canonicalize the alias here. `-F` is similarly
    # an undocumented shorthand for `--format` (a value-taking rewrite of the
    # short token `-F` would otherwise reach the parser undeclared).
    @*ARGS = @*ARGS.map: {
        when '--xsl'   { '--xsl-url' }
        when /^ '--xsl=' (.+) $/ { "--xsl-url=$0" }
        when '-F'      { '--format' }
        when /^ '-F=' (.+) $/ { "--format=$0" }
        default        { $_ }
    };

    # A value-taking option absorbs the next token UNLESS that token is a
    # *known* option (flag, value-option, or --no-* negation). Binding only on
    # shape (`-x`, `--foo`) made a following value that merely *looked* like an
    # option -- e.g. `-a -Bot` -- be silently dropped, so the user agent never
    # took effect. Only recognized options are excluded from absorption;
    # anything else, including a bare `-` or a negative number, is a value.
    #
    # The known set is derived from %KNOWN-OPTS rather than duplicated here:
    # every declared option (plus the --no-* negation of every long-form Bool
    # flag) guards itself against absorption automatically.
    my %known-option = %KNOWN-OPTS.keys.map: { $_ => True };
    for %KNOWN-OPTS.keys.grep({ .starts-with('--') && !%VALUE-OPTS{$_} }) -> $opt {
        %known-option{"--no-" ~ $opt.substr(2)} = True;
    }

    # Numeric value options (Int/Num) are handled correctly by Raku's native
    # parser: it refuses to bind a following option-looking token as a number,
    # so `-u --allow-http-fallback` already passes the flag through unchanged.
    # Only string-typed value options are misparsed: Raku greedily binds the
    # next option token as the value (e.g. `-a --no-respect-robots` makes the
    # user agent "--no-respect-robots" and silently leaves robots enabled), so
    # the ambiguity must be refused rather than misparsed. This list keeps the
    # numeric options out of that refusal.
    my %numeric-value = %(
        '-d' => 1, '--max-depth' => 1,
        '-u' => 1, '--max-urls' => 1,
        '-m' => 1, '--max-urls-per-file' => 1,
        '-c' => 1, '--concurrency' => 1,
        '--max-images' => 1, '--max-videos' => 1, '--recursive-depth' => 1,
    );

    my @new;
    my $i = 0;
    while $i < @*ARGS.elems {
        my $arg = @*ARGS[$i];
        my $next = $i + 1 < @*ARGS.elems ?? @*ARGS[$i+1] !! '';
        if $arg ~~ /^^ ('--'? <[a..zA..Z0..9\-]>+) '=' <(.+)> $/
                && %known-option{$0}
                && !%takes-value{$0}
                && !%numeric-value{$0} {
            # A Bool flag written as --flag=value reaches Raku's native parser,
            # which aborts with a bare usage dump instead of a friendly error.
            # Refuse it up front; Booleans are set with --flag or --no-flag.
            my %bool-long = '-z' => '--compress', '-v' => '--verbose',
                '-r' => '--respect-robots', '-h' => '--help';
            my $shown = %bool-long{$0} // $0;
            my $sans = $shown.substr(2);
            # A --no-* form is itself the negation; its base is the bare name,
            # so the suggestion must be "set with --base or --no-base", never a
            # doubled --no-no-*.
            my $base = $sans.starts-with('no-') ?? '--' ~ $sans.substr(3) !! $shown;
            die "Option $shown does not take a value (drop '=…'); Booleans are set with $base or --no-$base.substr(2).";
        }
        if %takes-value{$arg} && $i + 1 < @*ARGS.elems && !%known-option{$next} {
            @new.push: "$arg=" ~ @*ARGS[$i+1];
            $i += 2;
        }
        elsif %takes-value{$arg} && $i + 1 < @*ARGS.elems && %known-option{$next}
                && !%numeric-value{$arg} {
            # A string-typed value option immediately followed by another
            # *known* option is ambiguous: Raku's native parser would bind the
            # next option token as this option's value. Refuse rather than
            # misparse into silently-wrong output.
            die "Option $arg requires a value, but '$next' is itself an option here. "
              ~ "Write it as '$arg=$next' if you meant it as the value, or reorder "
              ~ "so the value option is not followed by another option.";
        }
        else {
            @new.push: $arg;
            $i++;
        }
    }
    @*ARGS = @new;
}

my %*SUB-MAIN-OPTS =
    :named-anywhere,
    :allow-no,
;

#| Compile-time validation: every named parameter in every MAIN candidate
#| must be a known option, and every value-taking option must appear in at
#| least one candidate.  Catches typos, missing declarations and option
#| table drift at compile time instead of silently mis-parsing CLI args.
# NOTE: iterates &MAIN.candidates via Raku multi-dispatch introspection.
# Fragile if Raku's compilation model changes; keep in sync with MAIN sigs.
CHECK {
    for &MAIN.candidates -> $candidate {
        for $candidate.signature.params -> $p {
            next if $p.positional;
            my @names = $p.named_names;
            next unless @names;
            my $is-bool = $p.type ~~ Bool;
            # named_names returns bare names (e.g. "output", "o") — build
            # both --long and -short forms for lookup in the option hashes.
            my @lookup = @names.map({ .chars == 1 ?? "-$_" !! "--$_" });
            my $canonical = @lookup.grep({ %KNOWN-OPTS{$_} }).first // @lookup[0];
            unless %KNOWN-OPTS{$canonical} {
                note "CHECK: unknown option --{$canonical} in MAIN candidate '$($candidate.name)'";
            }
            if !$is-bool && !%VALUE-OPTS{$canonical} {
                # Allow short aliases that are in %VALUE-OPTS
                unless @lookup.first({ %VALUE-OPTS{$_} }) {
                    note "CHECK: non-Bool option --{$canonical} not in %VALUE-OPTS in MAIN candidate '$($candidate.name)'";
                }
            }
            if $is-bool && %VALUE-OPTS{$canonical} {
                note "CHECK: Bool flag --{$canonical} is in %VALUE-OPTS in MAIN candidate '$($candidate.name)'";
            }
        }
    }
}

my constant $OUTPUT-OPTS = q:to/END/;
    -o, --output <file>     Output file (default: sitemap.<fmt>)
    -f, --format <fmt>      Output format: xml, html, txt, rss, atom, mrss (default: xml)
    -z, --compress          Gzip compress output file
        --no-pretty         Disable pretty printing
        --xsl <url>         Add XSL stylesheet reference
    -v, --verbose           Verbose output
        --debug             Debug output (crawl tracing)
    -h, --help              Show this help message
END

my constant $CRAWL-OPTS = qq:to/END/;
    -d, --max-depth <n>      Maximum crawl depth (default: 0 = unlimited)
    -u, --max-urls <n>       Maximum number of URLs (default: 0 = unlimited;
                            unbounded crawls buffer every URL in memory — set
                            a limit for large sites to cap memory use)
    -m, --max-urls-per-file <n>  Max URLs per sitemap file (default: 50000)
    -a, --user-agent <str>   User agent string (default: Raku-Sitemap/{$Sitemap::Config::VERSION})
        --no-respect-robots  Don't respect robots.txt (default: respect)
        --no-verify-ssl      Don't verify SSL certificates (default: verify)
        --allow-http-fallback  Allow HTTPS→HTTP downgrade on SSL/TLS errors (default: off)
        --no-images          Don't extract images during crawl (default: extract; --img is an alias for --images)
        --max-images <n>     Max images per page (default: 1000)
        --max-videos <n>     Max videos per page (default: 100)
        --exclude-dirs <list>
                             Skip URL paths starting with any of these comma-
                             separated prefixes; matches whole path segments,
                             so /admin also excludes /admin/x
        --exclude-extensions <list>
                             Comma-separated extensions to exclude from the
                             crawl, REPLACING the default list (ico, css, js,
                             woff, woff2, ttf, eot, zip, tar, gz, pdf, doc,
                             docx, rss, atom, xml)
        --extra-exclude-extensions <list>
                             Comma-separated extensions to exclude, ADDED to
                             the default exclusion list
        --no-follow          Crawl only the start URL; don't follow links
        --no-hreflang        Don't extract hreflang links during crawl (default: extract)
        --videos             Extract video metadata during crawl (default: off)
        --news               Extract NewsArticle JSON-LD and write a separate news sitemap (default: off)
        --no-lastmod         Don't extract lastmod from HTTP headers
        --no-priority        Don't calculate priority from crawl depth
        --max-redirects <n>  Max redirects followed per chain (default: 5)
    -c, --concurrency <n>    Max concurrent page fetches (default: 20)
END

#| A minimal gzip-writing handle. Feed it Str chunks via print and it deflates
#| them incrementally into the underlying raw file handle, so the source content
#| is never materialized as one string. Rendering, buffering and closing work
#| exactly like a plain file handle, letting the streaming TXT/HTML render-to
#| methods compress as they go (audit F).
class GzipWriter {
    has $!fh;
    has $!stream;
    submethod BUILD(IO::Path :$path!) {
        $!fh = $path.open(:w, :bin);
        $!stream = Compress::Zlib::Stream.new(:gzip);
    }
    method print(+@out) {
        my $chunk = @out.join('');
        return if $chunk eq '';
        my $compressed = $!stream.deflate($chunk.encode('utf-8'));
        $!fh.write($compressed) if $compressed.elems;
    }
    method close {
        my $tail = $!stream.finish;
        $!fh.write($tail) if $tail.elems;
        $!fh.close;
    }
}

#| Render a Sitemap::Builder to disk in the requested format. The xml branch
#| hands off to Sitemap::Builder's own writer (which splits by max-entries and
#| honors its compress/pretty settings); every other format renders via
#| format-output and gzips when :$compress is set. All write sites in this
#| script converge here so the branches stay identical.
sub write-output(
    $builder,
    Str $output,
    Str $fmt,
    Str $base-for-relative,
    Bool :$compress = False,
    Bool :$pretty = True,
) {
    if $fmt eq 'xml' {
        my @written = $builder.write: $output.IO;
        note "Sitemap written to: @written.join(', ')";
    }
    else {
        my @items = $builder.get-items;
        # The base URL fills both the feed title and the feed <link>: RSS/Atom
        # render() reads them from the 3rd/4th positionals, and passing the base
        # only as $title left the feed with an empty <link>.
        if $fmt eq 'txt' || $fmt eq 'html' {
            # Stream the line/document-oriented formats (TXT, HTML) directly to
            # the sink so a 50k+ item sitemap never materializes its full output
            # string in memory alongside the item list (audit F). TXT/HTML are
            # inherently serial: one line / one element at a time.
            my $gzip-out = out-path($output, $fmt, :$compress);
            my $fh = $compress
                ?? GzipWriter.new(path => $gzip-out)
                !! $gzip-out.open(:w);
            if $fmt eq 'txt' {
                Sitemap::Format::TXT.render-to($fh, @items.List);
            }
            else {
                Sitemap::Format::HTML.render-to($fh, @items.List, :title($base-for-relative), :$pretty);
            }
            $fh.close;
            note "{$fmt.uc} sitemap written to: $gzip-out";
        }
        else {
            my $content = format-output(@items.List, $fmt, $base-for-relative, $base-for-relative, pretty => $pretty);
            my $out = out-path($output, $fmt, :$compress);
            if $compress {
                gzspurt($out, $content);
            }
            else {
                $out.spurt($content);
            }
            note "{$fmt.uc} sitemap written to: $out";
        }
    }
}

#| The actual output path for a format, appending .gz for the feeds as the rest
#| of the CLI does (txt/html streaming and rss/atom/mrss share this).
sub out-path(Str $output, Str $fmt, Bool :$compress --> IO::Path) {
    my $path = $output.IO;
    return $path unless $compress;
    IO::Path.new($path.path.subst(/\.gz $/, '') ~ '.gz')
}


#| Write the optional news sitemap produced by a crawl or directory scan.
#| Shared by the crawl and dir MAINs, which used to carry near-identical
#| copies of this block. The news builder inherits the same output options
#| (max entries per file, gzip) as the main sitemap instead of always
#| writing stock settings.
sub write-news-output(
    $news-builder,
    Str $output,
    Bool :$verbose = False,
    Int :$stale-count = 0,
    Str :$stage = 'crawl',
) {
    if $news-builder.item-count > 0 {
        my $news-stem = $output;
        $news-stem ~~ s:i/\.gz$//;
        $news-stem ~~ s:i/\.xml$//;
        my $news-out = $news-stem ~ '-news.xml';
        # write() returns the actually-written paths — with compress enabled
        # that is '<stem>-news.xml.gz', not the name requested here.
        my @written = $news-builder.write: $news-out.IO;
        say "News sitemap written to: {@written.join(', ')}";

        if $stale-count > 0 {
            note "Warning: $stale-count news item(s) skipped during $stage (older than 48 hours).";
            note "Consider regenerating or removing the news sitemap.";
        }
    } else {
        say "Warning: --news enabled but no NewsArticle JSON-LD found during $stage." if $verbose;
    }
}

# URL validation for the crawl command. The start URL is either an explicit
# http(s) URL or a scheme-less host guess (example.com, localhost:8080,
# 127.0.0.1, [::1]) that normalize-start-url will later turn into https://….
# Rejecting obviously-mistaken inputs here gives a clear message instead of a
# downstream connection error; recognized non-http schemes (ftp://x) are
# refused because the crawler can only fetch HTTP(S).
my regex url-label { <[a..zA..Z0..9\-]>+ }
my regex url-host {
    || '[' <[0..9a..fA..F:.]>+ ']'                 # IPv6 literal
    || <url-label> ( '.' <url-label> )+           # dotted hostname
    || <url-label>                                # single token (localhost)
}
sub url-port-ok(Str $port --> Bool) { $port ~~ /^ \d+ $/ && $port.Int >= 1 && $port.Int <= 65535 }

# Everything up to the first of / ? # (the scheme has already been split off).
sub url-authority-of(Str $rest --> Str) {
    for < / ? # > -> $sep {
        my $i = $rest.index($sep);
        return $rest.substr(0, $i) if $i.defined;
    }
    $rest;
}

# A host[:port] authority, with userinfo stripped. Port presence is tested via
# a required-capture branch so the port capture is never an empty optional.
sub url-authority-ok(Str $authority --> Bool) {
    my $a = $authority;
    $a = $a.substr($a.index('@') + 1) if $a.contains('@');
    return False if $a eq '';
    if my $m = $a ~~ /^ <url-host> ':' (\d+) $/ {
        return url-port-ok(~$m[0]);
    }
    so $a ~~ /^ <url-host> $/;
}

sub is-valid-crawl-url(Str $url --> Bool) {
    if $url ~~ /^ (<[a..zA..Z]>+) '://' (.*) $/ {
        # Read captures into lexicals before any later match: a subsequent ~~
        # rebinds $0/$1, so using them inline would silently read Nil.
        my $scheme = ~$0;
        my $rest   = ~$1;
        return False unless $scheme ~~ /^ (:i 'http' s? ) $/;
        return url-authority-ok(url-authority-of($rest));
    }
    if $url ~~ /^ '//' (.*) $/ {
        my $rest = ~$0;
        return url-authority-ok(url-authority-of($rest));
    }
    if my $m = $url ~~ /^ (<url-host>) ':' (\d+) ( [ '/' | '?' ] .* )? $/ {
        return url-port-ok(~$m[1]);
    }
    so $url ~~ /^ <url-host> ( [ '/' | '?' ] .* )? $/;
}

# Crawl command - crawl a website
multi MAIN(
    Str $url,  #= URL to crawl (e.g., https://raku.org or raku.org)
    Str :o(:$output) is copy = '',                      #= Output file
    Str :f(:$format) = 'xml',                          #= Output format (xml, html, txt)
    Int :d(:$max-depth) = 0,                           #= Maximum crawl depth
    Int :u(:$max-urls) = 0,                            #= Maximum number of URLs
    Int :m(:$max-urls-per-file) = 50000,               #= Max URLs per sitemap file
    Str :a(:$user-agent) = "Raku-Sitemap/{$Sitemap::Config::VERSION}",      #= User agent string
    Bool :$respect-robots = True,                 #= Respect robots.txt (default: true, use --no-respect-robots to disable)
    Bool :verify-ssl(:$ssl-verify) = True,             #= Verify SSL certificates (use --no-verify-ssl to disable)
    Bool :$allow-http-fallback = False,                    #= Allow HTTPS→HTTP downgrade on SSL/TLS errors (default: false)
    Bool :z(:$compress) = False,                       #= Gzip compress output
    Bool :pretty(:$do-pretty) = True,                  #= Pretty print output (use --no-pretty to disable)
    Str :$xsl-url = '',                                #= XSL stylesheet URL
    Bool :v(:$verbose) = False,                       #= Verbose output
    Bool :$debug = False,                         #= Debug output (crawl tracing)
    Bool :images(:img(:$extract-images)) = True,   #= Extract images (use --no-images/--no-img to disable)
    Int :$max-images = 1000,                        #= Max images per page (default: 1000)
    Int :$max-videos = 100,                         #= Max videos per page (default: 100)
    Int :$max-redirects = 5,                        #= Max redirects per chain (default: 5)
    Str :$exclude-dirs = '',                       #= Comma-separated URL path prefixes to skip
    Str :$exclude-extensions = '',                 #= Comma-separated extensions to EXCLUDE (replaces the default list)
    Str :$extra-exclude-extensions = '',           #= Comma-separated extensions to add to the default exclude list
    Bool :follow(:$follow-links) = True,            #= Follow discovered links (use --no-follow to crawl only the start URL)
    Bool :hreflang(:$extract-hreflang) = True, #= Extract hreflang links (use --no-hreflang to disable)
    Bool :videos(:$extract-videos) = False,    #= Extract video metadata (default: off)
    Bool :news(:$extract-news) = False,        #= Extract NewsArticle JSON-LD (default: off)
    Int :c(:$concurrency) = 20,                         #= Max concurrent page fetches
    Bool :lastmod(:$extract-lastmod) = True,            #= Extract lastmod from HTTP headers (use --no-lastmod to disable)
    Bool :priority(:$calculate-priority) = True,         #= Calculate priority from crawl depth (use --no-priority to disable)
    Bool :h(:$help) = False,                         #= Show help
) {
    if $help {
        print q:to/END/;
sitemap - Crawl a website and generate a sitemap

Usage:  sitemap <url> [options]

OUTPUT OPTIONS:
END
        print $OUTPUT-OPTS;
        print q:to/END/;

CRAWL OPTIONS:
END
        print $CRAWL-OPTS;
        exit 0;
    }
    note "Warning: SSL verification disabled — connections may be intercepted" unless $ssl-verify;
    # Redirect subcommand names to their MAIN handlers before URL validation.
    # Multi-dispatch normally routes these away, but if one slips through as the
    # crawl URL the message must say so instead of a confusing "Invalid URL".
    if $url ~~ /^ [build | parse | fetch | convert | dir | tree] $/ {
        note "Error: Use 'sitemap $url ...' for that command";
        exit 1;
    }
    if $url eq 'crawl' {
        note "Error: crawl is the default command; run 'sitemap <url>' directly (e.g. 'sitemap https://example.com/')";
        exit 1;
    }
    if $max-depth < 0 {
        note "Error: --max-depth must be 0 or greater (got $max-depth)";
        exit 1;
    }
    if $max-urls < 0 {
        note "Error: --max-urls must be 0 or greater (got $max-urls); use 0 for unlimited";
        exit 1;
    }
    if $max-urls-per-file < 1 {
        note "Error: --max-urls-per-file must be 1 or greater (got $max-urls-per-file)";
        exit 1;
    }
    if $concurrency < 1 {
        note "Error: --concurrency must be 1 or greater (got $concurrency)";
        exit 1;
    }
    if $max-redirects < 1 {
        note "Error: --max-redirects must be 1 or greater (got $max-redirects)";
        exit 1;
    }
    # Validate URL - must look like a URL: an explicit http(s) scheme, or a
    # host optionally with :port and/or a path (example.com/blog). Rejects
    # non-http schemes, out-of-range ports and hostname typos up front.
    unless is-valid-crawl-url($url) {
        note "Error: Invalid URL: $url";
        say "URL must start with http:// or https://, or be a domain like 'example.com', 'example.com:8080' or 'example.com/blog'";
        exit 1;
    }

    # Normalize URL - prepend https:// if missing
    my $normalized-url = normalize-start-url($url);
    
    my $fmt = validate-format($format);
    $output = "sitemap.$fmt" if $output eq '';

    say "Starting sitemap generation for: $normalized-url" if $verbose;

    # Build {ext => True} exclusion seed hashes from the comma-separated CLI
    # values, mirroring the Crawler's default exclusion hash shape. Each
    # extension becomes a True-valued key; empty input yields an empty hash so
    # the Crawler keeps its defaults.
    my sub ext-seed(Str $csv) {
        my %seed;
        %seed{$_} = True for $csv.split(',').map(*.trim).grep(*.chars).map(*.lc);
        %seed;
    }
    my %exclude-extensions-seed = ext-seed($exclude-extensions);
    my %extra-exclude-extensions-seed = ext-seed($extra-exclude-extensions);

    # Only bind the extension exclusions when the caller actually supplied a
    # value; an empty --exclude-extensions must leave the Crawler defaults
    # intact (the slipped hash stays empty when nothing was given).
    # NOTE: this must be a HASH slip, not an array of pairs. `|@array-of-pairs`
    # splats positionally (breaking Crawler.new's named-arg-only constructor),
    # and `push(... exclude-extensions => %h)` with an unquoted dashed key is
    # silently dropped by the parser; `|%hash` is the robust form.
    my %exclusion;
    %exclusion<exclude-extensions> = %exclude-extensions-seed if %exclude-extensions-seed.elems;
    %exclusion<extra-exclude-extensions> = %extra-exclude-extensions-seed if %extra-exclude-extensions-seed.elems;

    my $crawler = Sitemap::Crawler.new(
        url => $normalized-url,
        :$user-agent,
        :$max-depth,
        :$max-urls,
        :$concurrency,
        :$respect-robots,
        :verify-ssl($ssl-verify),
        :$extract-images,
        :$max-images,
        :$max-videos,
        :$max-redirects,
        :exclude-dirs($exclude-dirs.split(',')».trim.grep(*.chars)),
        |%exclusion,
        :$follow-links,
        :$extract-hreflang,
        :$extract-videos,
        :$extract-news,
        :$extract-lastmod,
        :$calculate-priority,
        :$allow-http-fallback,
        :$verbose,
        :$debug,
    );
    # The CLI is a one-shot crawl, so finish the event suppliers too instead of
    # leaving them open; library callers that reuse a Crawler for a second
    # crawl() should call close() only and keep the suppliers open.
    LEAVE { $crawler.close; $crawler.close-suppliers } if $crawler.defined;

    $crawler.on-add: -> $url {
        say "Found: $url" if $verbose;
    } if $verbose;

    $crawler.on-error: -> $err {
        if $verbose {
            if $err ~~ Hash {
                note "Error: {$err<error>} for {$err<url>}";
            } else {
                note "Error: $err";
            }
        }
    } if $verbose;

    my $builder = $crawler.crawl;
    $builder.max-entries = $max-urls-per-file;
    $builder.compress = $compress;
    $builder.pretty = $do-pretty;
    $builder.xsl-url = $xsl-url if $xsl-url;

    # The news builder gets the same chunking/compression/stylesheet options as
    # the main sitemap instead of silently writing stock settings.
    if $extract-news {
        with $crawler.news-builder -> $nb {
            $nb.max-entries = $max-urls-per-file;
            $nb.compress = $compress;
            $nb.pretty = $do-pretty;
            $nb.xsl-url = $xsl-url if $xsl-url;
        }
    }

    say "Found {$builder.item-count} URLs" if $verbose;

    unless $builder.item-count > 0 {
        note "Error: No URLs found (crawl produced no results)";
        exit 1;
    }

    # Write output using Sitemap::Builder
    write-output($builder, $output, $fmt, $normalized-url, :$compress, :pretty($do-pretty));

    # Write news sitemap if enabled
    if $extract-news && $crawler.news-builder && $fmt ne 'xml' {
        note "Warning: --news applies only to XML sitemaps; ignoring it for format '$fmt'";
    }
    if $extract-news && $crawler.news-builder && $fmt eq 'xml' {
        write-news-output($crawler.news-builder, $output,
            :$verbose, :stale-count($crawler.stale-news-count), :stage('crawl'));
    }
}

# Build command - build from URL list file (supports TXT, XML, RSS, Atom, MRSS, HTML)
multi MAIN(
    'build',
    Str $file is copy = '',                              #= File containing URLs (TXT, XML, RSS, Atom, MRSS, HTML)
    Str :o(:$output) is copy = '',                      #= Output file
    Str :f(:$format) = 'xml',                          #= Output format (xml, html, txt, rss, atom, mrss)
    Int :m(:$max-urls-per-file) = 50000,               #= Max URLs per sitemap file
    Str :$base-url = '',                                #= Base URL for relative URLs
    Bool :verify-ssl(:$ssl-verify) = True,              #= Verify SSL (use --no-verify-ssl to disable)
    Bool :z(:$compress) = False,                       #= Gzip compress output
    Bool :pretty(:$do-pretty) = True,                  #= Pretty print output (use --no-pretty to disable)
    Str :$xsl-url = '',                                #= XSL stylesheet URL
    Bool :v(:$verbose) = False,                        #= Verbose output
    Bool :h(:$help) = False,                           #= Show help
) {
    if $help {
        print q:to/END/;
sitemap build - Build sitemap from URL list file

Usage:  sitemap build <file> [options]

Supports input formats: TXT (one URL per line), XML sitemap, RSS 2.0,
Atom 1.0, Media RSS (MRSS), and HTML (extracts <a href> links).
The input may also be a URL (e.g. https://example.com/sitemap.xml).

OUTPUT OPTIONS:
END
        print $OUTPUT-OPTS;
        print q:to/END/;

BUILD OPTIONS:
    -m, --max-urls-per-file <n>  Max URLs per sitemap file (default: 50000)
        --base-url <url>     Base URL for relative URLs
        --no-verify-ssl      Don't verify SSL when the input is a URL
END
        exit 0;
    }
    note "Warning: SSL verification disabled — connections may be intercepted" unless $ssl-verify;

    # The input is a local file, or a URL when no such file exists (mirroring
    # `sitemap parse`, so `sitemap build https://.../sitemap.xml` works and
    # honors --no-verify-ssl). The hostname guess requires a dot or an explicit
    # port so a typo'd filename is not silently fetched.
    my $input-type = detect-input-type($file);
    my $is-file = $input-type eq 'file';
    my $is-url  = $input-type eq 'url';
    unless $is-file || $is-url {
        note "Error: File not found: {$file // '(none)'}";
        say "Usage: sitemap build <file> [options]";
        exit 1;
    }

    my $fmt = validate-format($format);
    $output = "sitemap.$fmt" if $output eq '';

    # A bare domain ("example.com"), host:port or scheme-relative "//host"
    # is a URL too; normalize it so parse-url gets a usable scheme (mirrors
    # the parse command's normalize-start-url at its dispatch). The hostname
    # guess requires a dot or explicit port so a typo'd filename is not
    # silently fetched.
    $file = normalize-start-url($file) if $is-url && $file !~~ /^ <[a..zA..Z]>+ '://' /;

    say "Building sitemap from: $file" if $verbose;

    my @items = $is-url
        ?? parse-url($file, :$ssl-verify, :$verbose, |($base-url ?? (:$base-url) !! Empty))
        !! parse-file($file, :$verbose, |($base-url ?? (:$base-url) !! Empty));
    unless @items.elems > 0 {
        note "Error: No URLs found in $file";
        exit 1;
    }

    say "Found {@items.elems} URLs" if $verbose;

    my $builder = Sitemap::Builder.new(:max-entries($max-urls-per-file), :$compress, :$xsl-url, :pretty($do-pretty));

    for @items -> $item {
        $builder.add-item-from($item);
    }

    say "Added {$builder.item-count} URLs" if $verbose;

    # Write output using Sitemap::Builder
    write-output($builder, $output, $fmt, $base-url, :$compress, :pretty($do-pretty));
}

# Parse command - parse existing sitemap (supports XML, TXT, RSS, Atom, MRSS, HTML)
multi MAIN(
    'parse',
    Str $file is copy = '',                              #= Sitemap file or URL to parse
    Bool :r(:$recursive) = False,                      #= Enable recursive parsing (XML only)
    Int :$recursive-depth = -1,                        #= Max depth (0=unlimited); any >=0 implies --recursive (-1 = not supplied)
    Bool :$follow-foreign-children = False,            #= Follow child sitemaps on other hosts (⚠ SSRF risk: disables same-origin guard)
    Int :c(:$concurrency) = 4,                         #= Parallel child fetches for --recursive
    Bool :verify-ssl(:$ssl-verify) = True,              #= Verify SSL (use --no-verify-ssl to disable)
    Bool :v(:$verbose) = False,                         #= Verbose output (show lastmod)
    Bool :h(:$help) = False,                            #= Show help
) {
    if $help {
        print q:to/END/;
sitemap parse - Parse existing sitemap file or URL

Usage:  sitemap parse <file> [options]
        sitemap parse <url>  [options]

Supports input formats: XML, TXT, RSS 2.0, Atom 1.0, MRSS, HTML.

PARSE OPTIONS:
    -r, --recursive           Enable recursive parsing (XML sitemaps only)
        --recursive-depth=<n>  Max depth; any >=0 enables recursion (0=unlimited)
        --follow-foreign-children
                              Follow child sitemaps on other hosts (default: same-host only)
                              ⚠ SECURITY: disables same-origin guard; a malicious sitemap
                              index can fetch arbitrary hosts including internal endpoints
    -c, --concurrency=<n>     Parallel child fetches for --recursive (default: 4)
        --no-verify-ssl       Don't verify SSL certificates (default: verify)
    -v, --verbose             Verbose output (show lastmod)

EXAMPLES:
    sitemap parse sitemap.xml
    sitemap parse sitemap.xml --recursive
    sitemap parse sample.rss
    sitemap parse https://example.com/feed.atom
END
        exit 0;
    }

    my @items;
    my $input-type = detect-input-type($file);
    my $is-file = $input-type eq 'file';
    # A bare domain ("example.com"), host:port or scheme-relative "//host"
    # input is a URL too, not a missing file; normalize it so downstream
    # parsers get a usable scheme. The hostname guess requires a dot or
    # an explicit port so a typo'd filename is not silently fetched.
    my $is-url = $input-type eq 'url';
    if $is-url {
        $file = normalize-start-url($file) unless $file ~~ /^ <[a..zA..Z]>+ '://' /;
        say "Parsing URL: $file" if $verbose;

        my $do-recursive = $recursive || $recursive-depth >= 0;

        if $do-recursive {
            my $effective-depth = recursive-depth(:$recursive-depth);
            my $result = Sitemap::Parser.parse-xml-url-recursive($file, :max-depth($effective-depth), :verify-ssl($ssl-verify), :$follow-foreign-children, :concurrency($concurrency));
            @items = |$result<items>;
            say "Parsed {$result<urls-parsed>} URL(s), found {$result<sitemaps>.elems} sitemap index(es)" if $verbose;
            if $result<errors>.elems > 0 {
                note "Errors: {$result<errors>.elems}" if $verbose;
            }
        } else {
            @items = parse-url($file, :$ssl-verify, :$verbose);
        }

    } elsif $is-file {
        say "Parsing: $file" if $verbose;

        my $do-recursive = $recursive || $recursive-depth >= 0;

        if $do-recursive {
            my $effective-depth = recursive-depth(:$recursive-depth);
            my $result = Sitemap::Parser.parse-xml-file-recursive($file.IO, :max-depth($effective-depth));
            @items = |$result<items>;
            say "Parsed {$result<files-parsed>} file(s), found {$result<sitemaps>.elems} sitemap index(es)" if $verbose;
            if $result<errors>.elems > 0 {
                note "Errors: {$result<errors>.elems}" if $verbose;
            }
        } else {
            @items = parse-file($file, :$verbose);
        }
    } else {
        note "Error: File not found: {$file // '(none)'}";
        say "Usage: sitemap parse <file> [options]";
        exit 1;
    }

    unless @items.elems > 0 {
        note "Error: No URLs found in $file";
        exit 1;
    }

    for @items -> $item {
        say $item.url;
        if $verbose && $item.lastmod {
            say "  Lastmod: {$item.lastmod}";
        }
    }

    say "Total URLs: {@items.elems}";
}

# Fetch command - download sitemap from server
multi MAIN(
    'fetch',
    Str $url = '',                                       #= Sitemap URL to fetch
    Str :o(:$output) = '',                                #= Output file (default: preserve original name)
    Str :f(:$format) = 'xml',                            #= Output format: xml, html, txt, rss, atom, mrss
    Bool :r(:$recursive) = False,                        #= Fetch all sitemaps in a sitemap index
    Bool :$force = False,                                 #= Delete existing output dir before recursive fetch
    Int :c(:$concurrency) = 10,                           #= Max concurrent child sitemap fetches
    Bool :$follow-foreign-children = False,               #= Fetch child sitemaps on other hosts (⚠ SSRF risk: disables same-origin guard)
    Bool :verify-ssl(:$ssl-verify) = True,                #= Verify SSL certificates
    Bool :z(:$compress) = False,                          #= Gzip compress output file (single fetch only)
    Bool :v(:$verbose) = False,                         #= Verbose output
    Bool :h(:$help) = False,                            #= Show help
) {
    if $help {
        print q:to/END/;
sitemap fetch - Download sitemap from server

Usage:  sitemap fetch <url> [options]

OPTIONS:
    -o, --output <file>     Output file (default: preserve original filename)
    -f, --format <fmt>      Output format: xml (default), html, txt, rss, atom, mrss
    -r, --recursive         Fetch all sitemaps in a sitemap index
        --force             Delete existing output dir before recursive fetch (no-op without --recursive)
        --follow-foreign-children
                            Fetch child sitemaps on other hosts (default: same-host only)
                            ⚠ SECURITY: disables same-origin guard; a malicious sitemap
                            index can fetch arbitrary hosts including internal endpoints
        --no-verify-ssl     Don't verify SSL certificates
    -z, --compress          Gzip the fetched sitemap (single fetch only; not with --recursive)
    -v, --verbose           Verbose output
    -h, --help              Show this help message
END
        exit 0;
    }
    note "Warning: SSL verification disabled — connections may be intercepted" unless $ssl-verify;

    unless $url {
        note "Error: fetch requires a URL (see 'sitemap fetch --help')";
        exit 1;
    }

    # Normalize format case-insensitively; everything below compares against
    # the lowercased value.
    my $fmt = validate-format($format);

    # Normalize URL
    my $normalized = normalize-start-url($url);

    # If URL doesn't look like a sitemap, discover from robots.txt (with
    # fallback). The .xml check ignores any query/fragment so a URL like
    # "sitemap.xml?v=2" is still fetched directly (query preserved), not
    # misdetected as a site root.
    my $sitemap-url = $normalized;
    my $no-query = $normalized.subst(/[ '?' | '#' ] .* $/, '');
    unless $no-query ~~ /:i ['.xml' | '.xml.gz'] $/ {
        # discover-sitemap dies when neither robots.txt nor /sitemap.xml yield
        # a sitemap; surface that as a friendly error + exit 1 instead of an
        # unhandled exception backtrace.
        my $discovered = try discover-sitemap($normalized, :$ssl-verify, :$verbose);
        if $! {
            note "Error: {$!.message}";
            exit 1;
        }
        $sitemap-url = $discovered;
    }

    if $recursive {
        if $compress {
            note "Error: --compress is not supported with --recursive; fetch the children uncompressed or use a single fetch";
            exit 1;
        }
        # Use Sitemap::Fetcher for recursive fetch
        my %result;
        try {
            %result = fetch-recursive($sitemap-url, :format($fmt), :output($output),
                :$force, :$ssl-verify, :$verbose, :$concurrency, :$follow-foreign-children);
        }
        if $! {
            note "Error: {$!.message}";
            exit 1;
        }
        if %result<errors>.elems > 0 {
            note "Errors: {%result<errors>.elems} child sitemap(s) could not be fetched" if $verbose;
        }
    } else {
        # Single sitemap fetch
        say "Fetching $sitemap-url..." if $verbose;
        my $content = fetch($sitemap-url, :$ssl-verify, :$verbose);

        # Determine output filename
        my $out-file = output-filename($sitemap-url, :format($fmt), :output-override($output));

        if Sitemap::Parser.is-sitemap-index($content) {
            # parse-string on a sitemap index yields 0 items, so handle it
            # directly instead of silently writing an empty sitemap.
            if $fmt eq 'xml' {
                if $compress {
                    my $gz-file = $out-file ~ '.gz';
                    gzspurt($gz-file, $content);
                    say "Saved index as-is: $gz-file";
                } else {
                    $out-file.IO.spurt($content);
                    say "Saved index as-is: $out-file";
                }
                say "Hint: use --recursive to also fetch the child sitemaps." if $verbose;
            } else {
                note "Error: $sitemap-url is a sitemap index; $fmt conversion requires --recursive";
                exit 1;
            }
        } else {
            my @items = parse-string($content, '');

            unless @items.elems > 0 {
                note "Error: No URLs found in $sitemap-url (unrecognized or empty content; is it a sitemap?)";
                exit 1;
            }

            # Re-serialize through Sitemap::Builder for xml; the other formats
            # render from the same items.
            my $builder = Sitemap::Builder.new;
            $builder.compress = $compress;
            for @items -> $item {
                $builder.add-item-from($item);
            }
            write-output($builder, $out-file, $fmt, $sitemap-url, :$compress);
        }
    }
}

sub USAGE() {
    print q:to/END/;
sitemap - Raku sitemap generator

Usage:
    sitemap <url> [options]           Crawl a website and generate sitemap
    sitemap build <file> [options]    Build sitemap from URL list file
    sitemap parse <file> [options]    Parse existing sitemap file or URL
    sitemap fetch <url> [options]     Fetch sitemap from server (--recursive creates <name>/ dir)
    sitemap convert <file> [options]  Convert local sitemap to another format
    sitemap dir <dir> [options]       Crawl local directory and generate sitemap
    sitemap tree <file> [options]     Build sitemap from JSON or YAML site tree definition

COMMON OPTIONS:
END
    print $OUTPUT-OPTS;
    print q:to/END/;

CRAWL OPTIONS (sitemap <url>):
END
    print $CRAWL-OPTS;
    print q:to/END/;

Run 'sitemap <subcommand> --help' for subcommand-specific options.
END
}

sub validate-format(Str $format --> Str) {
    my %valid = xml => 1, html => 1, txt => 1, rss => 1, atom => 1, mrss => 1;
    my $normalized = $format.lc;
    unless %valid{$normalized} {
        # Friendly CLI error like every other validation failure in this
        # script: a bare die would dump an internal backtrace on the user.
        note "Error: Invalid format '$format'. Valid formats: xml, html, txt, rss, atom, mrss";
        exit 1;
    }
    $normalized;
}

sub recursive-depth(Int :$recursive-depth --> Int) {
    # -1 is the "not supplied" sentinel (see parse MAIN). An explicit value
    # (>= 0) wins as the max depth; 0 means unlimited. When reached here with
    # no explicit depth, --recursive alone was passed → unlimited (0).
    $recursive-depth >= 0 ?? $recursive-depth !! 0;
}

# Convert command - convert local sitemap file to another format
multi MAIN(
    'convert',
    Str $file = '',                                        #= Local sitemap file to convert
    Str :o(:$output) = '',                                #= Output file (default: derived from input)
    Str :f(:$format) = 'txt',                             #= Output format: xml, html, txt, rss, atom, mrss
    Bool :z(:$compress) = False,                          #= Gzip compress output file
    Bool :v(:$verbose) = False,                         #= Verbose output
    Bool :h(:$help) = False,                            #= Show help
) {
    if $help {
        print q:to/END/;
sitemap convert - Convert local sitemap to another format

Usage:  sitemap convert <file> [options]

OPTIONS:
    -o, --output <file>     Output file (default: derived from input)
    -f, --format <fmt>      Output format: xml, html, txt, rss, atom, mrss (default: txt)
    -z, --compress          Gzip compress the output file
    -v, --verbose           Verbose output
    -h, --help              Show this help message
END
        exit 0;
    }

    unless $file {
        note "Error: convert requires a file (see 'sitemap convert --help')";
        exit 1;
    }
    unless $file.IO.f {
        note "Error: File not found: $file";
        exit 1;
    }

    my $fmt = validate-format($format);
    my $out-file = output-filename($file, :format($fmt), :output-override($output));

    say "Converting $file to $fmt..." if $verbose;

    my @items = parse-file($file, :$verbose);

    unless @items.elems > 0 {
        note "Error: No URLs found in $file";
        exit 1;
    }

    # Re-serialize through Sitemap::Builder for xml; the other formats render
    # from the same items.
    my $builder = Sitemap::Builder.new;
    $builder.compress = $compress;
    for @items -> $item {
        $builder.add-item-from($item);
    }
    write-output($builder, $out-file, $fmt, '', :$compress);

    say "Total URLs: {@items.elems}" if $verbose;
}

# Dir command - crawl local directory and generate sitemap
multi MAIN(
    'dir',
    Str $directory = '',                                      #= Directory to crawl
    Str :$base-url = '',                                      #= Base URL for sitemap entries
    Str :o(:$output) is copy = '',                           #= Output file
    Str :f(:$format) = 'xml',                                #= Output format
    Int :d(:$max-depth) = 0,                                 #= Maximum crawl depth
    Int :u(:$max-urls) = 0,                                  #= Maximum URLs
    Int :m(:$max-urls-per-file) = 50000,                    #= Max URLs per file
    Bool :z(:$compress) = False,                             #= Gzip compress
    Bool :pretty(:$do-pretty) = True,                        #= Pretty print
    Str :$xsl-url = '',                                      #= XSL stylesheet
    Bool :images(:$extract-images) = True,                   #= Extract images
    Bool :hreflang(:$extract-hreflang) = True,               #= Extract hreflang
    Bool :videos(:$extract-videos) = False,                  #= Extract video metadata (default: off)
    Bool :news(:$extract-news) = False,                      #= Extract NewsArticle JSON-LD (default: off)
    Bool :priority(:$calculate-priority) = True,             #= Calculate priority
    Int :c(:$concurrency) = 1,                               #= Max concurrent file processors
    Int :$max-images = 1000,                                 #= Max images per page (default: 1000)
    Int :$max-videos = 100,                                  #= Max videos per page (default: 100)
    Bool :v(:$verbose) = False,                              #= Verbose output
    Bool :h(:$help) = False,                                 #= Show help
) {
    if $help {
        print q:to/END/;
sitemap dir - Crawl local directory and generate sitemap

Usage:  sitemap dir <directory> [options]

Walks a local directory tree, following links between HTML files
(like a web crawler), and generates a sitemap.

OUTPUT OPTIONS:
END
        print $OUTPUT-OPTS;
        print q:to/END/;

DIR OPTIONS:
    --base-url <url>          Base URL for sitemap entries (auto-detect from robots.txt, fallback: file:///)
    -d, --max-depth <n>       Maximum crawl depth (default: 0 = unlimited)
    -u, --max-urls <n>        Maximum number of URLs (default: 0 = unlimited;
                              unbounded scans buffer every URL in memory — set
                              a limit for large sites to cap memory use)
    -m, --max-urls-per-file <n>  Max URLs per sitemap file (default: 50000)
    -c, --concurrency <n>     Max concurrent file processors (default: 1)
    --max-images <n>          Max images per page (default: 1000)
    --max-videos <n>          Max videos per page (default: 100)
    --no-images               Don't extract images (default: extract)
    --no-hreflang             Don't extract hreflang links (default: extract)
    --videos                  Extract video metadata (default: off)
    --news                    Extract NewsArticle JSON-LD and write a separate news sitemap (default: off)
    --no-priority             Don't calculate priority from depth (default: calculate)
END
        exit 0;
    }

    unless $directory && $directory.IO.d {
        note "Error: Directory not found: {$directory // '(none)'}";
        say "Usage: sitemap dir <directory> [options]";
        exit 1;
    }
    if $max-depth < 0 {
        note "Error: --max-depth must be 0 or greater (got $max-depth)";
        exit 1;
    }
    if $max-urls < 0 {
        note "Error: --max-urls must be 0 or greater (got $max-urls); use 0 for unlimited";
        exit 1;
    }
    if $max-urls-per-file < 1 {
        note "Error: --max-urls-per-file must be 1 or greater (got $max-urls-per-file)";
        exit 1;
    }
    if $concurrency < 1 {
        note "Error: --concurrency must be 1 or greater (got $concurrency)";
        exit 1;
    }

    my $fmt = validate-format($format);
    $output = "sitemap.$fmt" if $output eq '';

    # Reject any non-http(s) --base-url up front: a bare domain or an ftp:// /
    # file:// base would be emitted verbatim into every entry, producing
    # broken loc values. The crawl command normalizes bare domains to https://;
    # dir does not, so refuse anything that would yield garbage instead.
    if $base-url && $base-url !~~ /^ :i 'http' 's'? '://' / {
        note "Error: --base-url must start with http:// or https:// (got '$base-url')";
        exit 1;
    }

    say "Crawling directory: $directory" if $verbose;

    my %scan-result = scan-dir($directory,
        :$base-url, :$max-depth, :$max-urls,
        :$extract-images, :$extract-hreflang, :$extract-videos, :$extract-news,
        :$calculate-priority, :$concurrency, :$max-images, :$max-videos, :$verbose,
    );
    my $builder = %scan-result<builder>;
    my $news-builder = %scan-result<news-builder>;
    my $stale-count = %scan-result<stale-news-count>;

    $builder.max-entries = $max-urls-per-file;
    $builder.compress = $compress;
    $builder.pretty = $do-pretty;
    $builder.xsl-url = $xsl-url if $xsl-url;

    # Same output options for the news builder as for the main sitemap.
    if $extract-news && $news-builder {
        $news-builder.max-entries = $max-urls-per-file;
        $news-builder.compress = $compress;
        $news-builder.pretty = $do-pretty;
        $news-builder.xsl-url = $xsl-url if $xsl-url;
    }

    say "Found {$builder.item-count} URLs" if $verbose;

    unless $builder.item-count > 0 {
        note "Error: No URLs found in $directory";
        exit 1;
    }

    # Without --base-url (and no Sitemap: line in robots.txt) the dir scanner
    # roots entry URLs at file:/// — deliberate for a local tree — but a web
    # RSS/Atom feed carrying file:/// links is almost certainly unintended.
    # Surface it as a warning only; the URL semantics are left untouched.
    if !$base-url && $fmt ~~ /^(rss|atom)$/ {
        note "Warning: no --base-url; RSS/Atom entries will use file:/// URLs. Pass --base-url for web URLs.";
    }

    # Write output
    write-output($builder, $output, $fmt, $base-url || 'file:///', :$compress, :pretty($do-pretty));

    # Write news sitemap if enabled
    if $extract-news && $news-builder && $fmt ne 'xml' {
        note "Warning: --news applies only to XML sitemaps; ignoring it for format '$fmt'";
    }
    if $extract-news && $news-builder && $fmt eq 'xml' {
        write-news-output($news-builder, $output,
            :$verbose, :$stale-count, :stage('scan'));
    }
}

# Tree command - build sitemap from JSON site tree definition
multi MAIN(
    'tree',
    Str $file = '',                                          #= JSON or YAML file defining the site tree
    Str :o(:$output) is copy = '',                          #= Output file
    Str :f(:$format) = 'xml',                               #= Output format (xml, html, txt, rss, atom, mrss)
    Int :m(:$max-urls-per-file) = 50000,                   #= Max URLs per sitemap file
    Bool :z(:$compress) = False,                            #= Gzip compress output
    Bool :pretty(:$do-pretty) = True,                       #= Pretty print output (use --no-pretty to disable)
    Str :$xsl-url = '',                                     #= XSL stylesheet URL
    Bool :v(:$verbose) = False,                             #= Verbose output
    Bool :h(:$help) = False,                                #= Show help
) {
    if $help {
        print q:to/END/;
sitemap tree - Build sitemap from JSON or YAML site tree definition

Usage:  sitemap tree <file> [options]

Reads a JSON (.json) or YAML (.yml/.yaml) file defining a hierarchical
site structure and generates a sitemap. URL priorities are automatically
inferred from tree depth.

JSON/YAML format:
{
  "base_url": "https://example.com",
  "pages": [
    {"stub": "home", "priority": 1.0},
    {"stub": "about",
     "children": [
       {"stub": "team"}
     ]},
    {"stub": "blog",
     "children": [
       {"stub": "first-post"},
       {"stub": "second-post"}
     ]}
  ]
}

OUTPUT OPTIONS:
END
        print $OUTPUT-OPTS;
        print q:to/END/;

TREE OPTIONS:
END
        exit 0;
    }

    unless $file && $file.IO.f {
        note "Error: File not found: {$file // '(none)'}";
        say "Usage: sitemap tree <file> [options]";
        exit 1;
    }

    # Reject excessively large files to prevent unbounded memory allocation
    # during YAML/JSON parsing (deeply nested structures can blow the stack).
    my $file-size = $file.IO.s;
    if $file-size > 10 * 1024 * 1024 {
        note "Error: Tree file '$file' is $file-size bytes (max 10 MB)";
        exit 1;
    }

    if $max-urls-per-file < 1 {
        note "Error: --max-urls-per-file must be 1 or greater (got $max-urls-per-file)";
        exit 1;
    }

    my $fmt = validate-format($format);
    $output = "sitemap.$fmt" if $output eq '';

    say "Loading tree from: $file" if $verbose;

    my $data;
    if $file ~~ /:i '.' (yml|yaml) $/ {
        $data = try { load-yaml($file.IO.slurp) };
        unless $data.defined {
            note "Error: Invalid YAML in '$file': {$! ?? $!.message !! 'parse failed'}";
            exit 1;
        }
    } elsif $file ~~ /:i '.json' $/ {
        $data = try { from-json($file.IO.slurp) };
        unless $data.defined {
            note "Error: Invalid JSON in '$file': {$! ?? $!.message !! 'parse failed'}";
            exit 1;
        }
    } else {
        # Try JSON first, then YAML (read once to avoid double I/O)
        my $content = $file.IO.slurp;
        $data = try { from-json($content) };
        unless $data.defined {
            $data = try { load-yaml($content) };
        }
        unless $data.defined {
            note "Error: Unrecognised format in '$file'. Use .json, .yml, or .yaml extension.";
            exit 1;
        }
    }
    unless $data ~~ Hash {
        note "Error: '{$file}' must contain an object with 'base_url' and 'pages' keys (found {$data.^name}).";
        exit 1;
    }
    my $base-url = $data<base_url>;
    unless $base-url ~~ Str && $base-url.chars {
        note "Error: 'base_url' must be a non-empty string in tree JSON";
        exit 1;
    }
    $base-url = $base-url.subst(/\/+$/, '');
    my $pages = $data<pages>;
    unless $pages.defined && $pages ~~ Positional {
        note "Error: 'pages' must be an array in tree JSON (found {$pages.defined ?? $pages.^name !! 'nothing'})";
        exit 1;
    }
    my @page-data = @$pages;

    sub build-tree(Sitemap::SiteTree $parent, @pages) {
        for @pages -> $p {
            unless $p ~~ Hash {
                die "page entry is not an object: {$p.raku}";
            }
            unless $p<stub>:exists && $p<stub> {
                die "page missing 'stub' key: {$p.raku}";
            }

            my %item-attrs;
            for <lastmod changefreq priority expires title> -> $key {
                %item-attrs{$key} = $p{$key} if $p{$key}:exists;
            }
            # Any key that is neither a recognized item attribute nor the
            # structural 'stub'/'children' is almost certainly a typo
            # (e.g. 'prority' or 'childs') that silently dropped data below.
            # Warn without aborting, so a batch build still runs while the
            # mistake is surfaced on stderr.
            for $p.keys.grep({ $_ ne 'stub' && $_ ne 'children' && $_ !~~ any(<lastmod changefreq priority expires title>) }) -> $unknown {
                note "Warning: unknown key '$unknown' for stub '$p<stub>' (ignored)";
            }
            if $p<lastmod> ~~ Str {
                my $dt = try { DateTime.new($p<lastmod>) };
                unless $dt.defined {
                    die "Invalid lastmod '$p<lastmod>' for stub '$p<stub>': {$! ?? $!.message !! 'parse failed'}";
                }
                %item-attrs<lastmod> = $dt;
            }
            if %item-attrs<changefreq>:exists && %item-attrs<changefreq> ~~ Str {
                my $cf = coerce-changefreq(%item-attrs<changefreq>);
                unless $cf.defined {
                    die "Invalid changefreq '{%item-attrs<changefreq>}' for stub '$p<stub>'";
                }
                %item-attrs<changefreq> = $cf;
            }
            # priority arrives as a string from YAML/JSON parsers; add-child's
            # priority attribute is Numeric, so coerce and range-check here
            # (lastmod and changefreq are handled the same way above) instead
            # of letting add-child explode with a type check deep in the tree.
            if %item-attrs<priority>:exists && %item-attrs<priority> ~~ Str {
                my $pr = try { %item-attrs<priority>.Numeric };
                unless $pr.defined {
                    die "Invalid priority '{%item-attrs<priority>}' for stub '$p<stub>': {$! ?? $!.message !! 'not a number'}";
                }
                unless $pr >= 0 && $pr <= 1 {
                    die "Invalid priority '{%item-attrs<priority>}' for stub '$p<stub>': must be between 0 and 1";
                }
                %item-attrs<priority> = $pr;
            }

            my $child = $parent.add-child($p<stub>, |%item-attrs);
            say "  Added: {$child.full-url($base-url)}" if $verbose;

            my @kids = @($p<children> // ());
            if @kids {
                build-tree($child, @kids);
            }
        }
    }

    my $root = Sitemap::SiteTree.new(stub => '');
    say "Building tree..." if $verbose;
    try { build-tree($root, @page-data) };
    if $! {
        note "Error: {$!.message}";
        exit 1;
    }

    say "Tree structure:" if $verbose;
    say $root.tree if $verbose;

    my $builder = $root.to-builder($base-url,
        :max-entries($max-urls-per-file), :$compress, :pretty($do-pretty), :$xsl-url,
    );

    say "Generated {$builder.item-count} URLs from tree" if $verbose;

    write-output($builder, $output, $fmt, $base-url, :$compress, :pretty($do-pretty));
}
