#!/bin/bash
# Crawls telugu-good-lyrics.blogspot.com by following "Older Posts" links.
# Saves each page's full HTML to data/telugulyrics/page_NNN.html
# Stops when no "Older Posts" link is found.
# Usage: ./pipelines/telugulyrics.sh

OUT_DIR="$(dirname "$0")/../data/telugulyrics"
mkdir -p "$OUT_DIR"

START_URL="https://telugu-good-lyrics.blogspot.com/"
DELAY=1  # seconds between requests
page=1
url="$START_URL"

while true; do
    filename="$OUT_DIR/page_$(printf '%04d' $page).html"

    # Skip if already downloaded (resume support)
    if [ -f "$filename" ]; then
        echo "↷ Page $page already exists, skipping" >&2
    else
        echo "↓ Page $page → $filename" >&2
        curl -s -L -A "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36" \
            "$url" -o "$filename"

        if [ $? -ne 0 ] || [ ! -s "$filename" ]; then
            echo "✗ Failed to download page $page, stopping" >&2
            rm -f "$filename"
            break
        fi
        sleep "$DELAY"
    fi

    # Extract the "Older Posts" href
    next_url=$(grep -oE 'href='"'"'[^'"'"']*blog-pager-older-link[^'"'"']*'"'"'|href="[^"]*blog-pager-older-link[^"]*"' "$filename" \
        | grep -oE 'https://[^"'"'"']+' | head -1)

    # Fallback: match by class name in the anchor tag
    if [ -z "$next_url" ]; then
        next_url=$(grep -oE 'blog-pager-older-link[^>]+href="[^"]+"' "$filename" \
            | grep -oE 'https://[^"]+' | head -1)
    fi

    # Fallback: match the anchor containing "Older Posts" text
    if [ -z "$next_url" ]; then
        next_url=$(grep -iE "Older Posts" "$filename" \
            | grep -oE 'https://telugu-good-lyrics\.blogspot\.com/[^"'"'"' >]+' | head -1)
    fi

    # Decode HTML entities (e.g. & → &)
    next_url=$(echo "$next_url" | sed 's/&/\&/g')

    if [ -z "$next_url" ]; then
        echo "✓ No more older posts. Done at page $page." >&2
        break
    fi

    url="$next_url"
    page=$((page + 1))
done

total=$(ls "$OUT_DIR"/*.html 2>/dev/null | wc -l | tr -d ' ')
echo "Total pages saved: $total" >&2