#!/bin/bash

OUTPUT_FILE="articles.csv"
FEEDS_FILE="feeds.csv"

# Write CSV header
echo "Feed Name,Title,Link,Date" > "$OUTPUT_FILE"

while IFS='|' read -r FEED_NAME FEED_URL; do
    echo "Scraping: $FEED_NAME ($FEED_URL)"
    
    # Use the new node script for robust RSS/Atom parsing
    DATA=$(node scrape_rss.js "$FEED_URL" "$FEED_NAME" 2>/dev/null)
    
    if [ -n "$DATA" ] && [ "$DATA" != "[]" ]; then
        # Convert JSON to CSV lines using jq and append to output
        echo "$DATA" | jq -r '.[] | [.feedName, .title, .link, .date] | @csv' >> "$OUTPUT_FILE"
    else
        echo "  Failed to scrape or no items found for $FEED_NAME"
    fi

done < "$FEEDS_FILE"

echo "Done! Articles saved to $OUTPUT_FILE"