Revert previous breaking change: back to old feed list

rss-generator.py

@@ -1,55 +1,112 @@
-import requests
+#!/usr/bin/env python3
-import os
+"""
-from datetime import datetime
+Generate RSS feeds from Pebble Search API data.
-from feedgen.feed import FeedGenerator
+Requires: feedgen (pip install feedgen)
-from feedgen.entry import FeedEntry
+"""
 
-API_BASE_URL = "https://repebblesearch.com/api/v1/apps"
+import requests
-OUTPUT_FOLDER = "feeds"
+import json
+import os
+from datetime import datetime, timezone
+from feedgen.feed import FeedGenerator
+
+# Define your feeds here
+FEEDS = [
+    {
+        'url': 'https://repebblesearch.com/api/v1/apps?sort=release_date&order=desc&new_only=true&limit=50&platform=basalt',
+        'name': 'New Apps',
+        'description': 'Latest new Pebble Time (basalt) apps'
+    },
+    {
+        'url': 'https://repebblesearch.com/api/v1/apps?sort=release_date&order=desc&recently_updated=true&limit=50&platform=basalt',
+        'name': 'Updated Apps',
+        'description': 'Recently updated Pebble Time (basalt) apps'
+    }
+]
+
+# Output folder for feeds
+FEEDS_FOLDER = 'feeds'
+
+def fetch_apps_data(url):
+    """Fetch apps data from the API endpoint."""
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        print(f"Error fetching data: {e}")
+        return None
+
+def fetch_app_description(app_id):
+    """Fetch the description for a specific app from its detail endpoint."""
+    detail_url = f"https://repebblesearch.com/api/v1/apps/{app_id}"
+    try:
+        response = requests.get(detail_url)
+        response.raise_for_status()
+        
+        # Try parsing as JSON first
+        try:
+            data = response.json()
+            return data.get('description', '')
+        except json.JSONDecodeError:
+            # If not JSON, try to extract from response text
+            print(f"Warning: Detail endpoint returned non-JSON for {app_id}")
+            return ""
+    except requests.exceptions.RequestException as e:
+        print(f"Error fetching description for {app_id}: {e}")
+        return ""
 
 def format_description(text):
-    """Convert release notes text to HTML with paragraph and list support."""
+    """Convert plain text description to formatted HTML with proper line breaks and lists."""
     if not text:
         return ""
     
-    html = ""
+    # Normalize line breaks (handle both \r\n and \n)
-    lines = text.split("\n")
+    text = text.replace('\r\n', '\n')
+    
+    lines = text.split('\n')
+    formatted_lines = []
     in_list = False
     
     for line in lines:
-        line = line.strip()
+        stripped = line.strip()
-        if not line:
-            if in_list:
-                html += "</ul>"
-                in_list = False
-            continue
         
-        # Check if line is a list item (starts with -, *, or •)
+        # Check if this is a bullet point
-        if line.startswith(("-", "*", "•")):
+        if stripped and stripped[0] in ('-', '*', '•'):
             if not in_list:
-                html += "<ul>"
+                formatted_lines.append('<ul>')
                 in_list = True
-            item_text = line[1:].strip()
+            # Remove the leading dash/bullet and clean up
-            html += f"<li>{item_text}</li>"
+            item_text = stripped[1:].strip()
+            formatted_lines.append(f'<li>{item_text}</li>')
         else:
+            # Close list if we were in one
             if in_list:
-                html += "</ul>"
+                formatted_lines.append('</ul>')
                 in_list = False
-            html += f"<p>{line}</p>"
+            # Add non-empty paragraphs
+            if stripped:
+                formatted_lines.append(f'<p>{stripped}</p>')
     
+    # Close any open list
     if in_list:
-        html += "</ul>"
+        formatted_lines.append('</ul>')
     
-    return html
+    return '\n'.join(formatted_lines)
 
-def create_description_with_screenshots(app_data):
+def build_screenshots_html(app):
-    """Build description with screenshots on a single line, then release notes."""
+    """Build HTML for screenshots on a single line."""
-    description = ""
+    screenshots_html = ""
+    
+    # Check for new-style screenshots array
+    screenshots = app.get("screenshots", [])
+    
+    # Fallback to old-style screenshot_url if no array
+    if not screenshots and 'screenshot_url' in app and app['screenshot_url']:
+        screenshots = [{'url': app['screenshot_url']}]
     
-    # Add screenshots on a single line at the top
-    screenshots = app_data.get("screenshots", [])
     if screenshots:
-        screenshot_html = '<div style="display: flex; gap: 10px; margin-bottom: 20px; flex-wrap: nowrap; overflow-x: auto;">'
+        screenshots_html = '<div style="display: flex; gap: 10px; margin-bottom: 20px; flex-wrap: nowrap; overflow-x: auto;">'
         for idx, screenshot in enumerate(screenshots, 1):
             url = screenshot.get("url", "")
             width = screenshot.get("width", "auto")
@@ -60,13 +117,18 @@ def create_description_with_screenshots(app_data):
             if hardware:
                 label += f" ({hardware})"
             
-            screenshot_html += f'<img src="{url}" alt="{label}" title="{label}" width="{width}" height="{height}" style="max-height: 300px; flex-shrink: 0;" />'
+            if url:
+                screenshots_html += f'<img src="{url}" alt="{label}" title="{label}" width="{width}" height="{height}" style="max-height: 300px; flex-shrink: 0;" />'
         
-        screenshot_html += '</div>'
+        screenshots_html += '</div>'
-        description += screenshot_html
+    
+    return screenshots_html
+
+def build_changelog_html(app):
+    """Build HTML for changelog entries."""
+    changelog_html = ""
+    changelog = app.get("changelog", [])
     
-    # Add release notes below screenshots
-    changelog = app_data.get("changelog", [])
     if changelog:
         for entry in changelog:
             version = entry.get("version", "Unknown")
@@ -83,66 +145,109 @@ def create_description_with_screenshots(app_data):
             else:
                 formatted_date = "Unknown"
             
-            description += f"<h4>Version {version}</h4>"
+            changelog_html += f"<h4>Version {version}</h4>"
-            description += f"<em>Released: {formatted_date}</em><br/>"
+            changelog_html += f"<em>Released: {formatted_date}</em><br/>"
-            description += format_description(release_notes)
+            changelog_html += format_description(release_notes)
-    else:
-        # Fallback to latest_release_notes
-        latest_notes = app_data.get("latest_release_notes", "")
-        if latest_notes:
-            description += format_description(latest_notes)
     
-    return description
+    return changelog_html
 
-def fetch_apps(query, platform="basalt"):
+def create_rss_feed(apps_data, feed_name, feed_description, output_file):
-    """Fetch apps from the Pebble Search API."""
+    """Create an RSS feed from apps data using feedgen."""
-    params = {
-        "query": query,
-        "platform": platform,
-        "limit": 100
-    }
     
-    try:
+    # Extract the apps list from 'items' key
-        response = requests.get(API_BASE_URL, params=params)
+    if not isinstance(apps_data, dict) or 'items' not in apps_data:
-        response.raise_for_status()
+        print(f"  Unexpected data structure: 'items' key not found")
-        data = response.json()
+        return False
-        return data.get("apps", [])
+    
-    except requests.exceptions.RequestException as e:
+    apps = apps_data['items']
-        print(f"Error fetching apps for query '{query}': {e}")
+    
-        return []
+    if not isinstance(apps, list):
-
+        print(f"  Error: 'items' is not a list")
-def create_rss_feed(filename, title, description, apps):
+        return False
-    """Create an RSS feed from app data."""
+    
+    print(f"  Found {len(apps)} apps to process")
+    
+    # Initialize the feed generator
     fg = FeedGenerator()
-    fg.title(title)
+    fg.id(f'https://repebblesearch.com/api/v1/apps')
-    fg.link(href="https://repebblesearch.com", rel="alternate")
+    fg.title(feed_name)
-    fg.description(description)
+    fg.link(href='https://repebblesearch.com', rel='alternate')
-    fg.language("en")
+    fg.description(feed_description)
+    fg.language('en')
     
-    for app in apps:
+    # Process each app
+    for i, app in enumerate(apps):
+        # Skip entries without required fields
+        if not isinstance(app, dict) or 'title' not in app:
+            continue
+        
+        app_id = app.get('id', app['title'])
+        print(f"    Processing app {i+1}/{len(apps)}: {app['title']}")
+        
+        # Fetch description from detail endpoint
+        description = fetch_app_description(app_id)
+        # Format the description with HTML
+        description = format_description(description)
+        # Ensure description is never None
+        if description is None:
+            description = ""
+        
+        # Build description with sections
+        description_sections = []
+        
+        # Add screenshots on a single line at the top
+        screenshots_html = build_screenshots_html(app)
+        if screenshots_html:
+            description_sections.append(screenshots_html)
+        
+        # Add main description
+        if description:
+            description_sections.append('<h3>Description</h3>')
+            description_sections.append(description)
+        
+        # Add changelog if available
+        changelog_html = build_changelog_html(app)
+        if changelog_html:
+            description_sections.append('<h3>Changelog</h3>')
+            description_sections.append(changelog_html)
+        
+        # Fallback to latest release notes if no changelog
+        if not changelog_html and 'latest_release_notes' in app and app['latest_release_notes']:
+            release_notes = str(app['latest_release_notes']).strip()
+            if release_notes:
+                formatted_notes = format_description(release_notes)
+                description_sections.append('<h3>Latest Release Notes</h3>')
+                description_sections.append(formatted_notes)
+        
+        # Join all sections
+        final_description = '\n'.join(description_sections)
+        
+        # Create a feed entry
         fe = fg.add_entry()
+        fe.id(str(app_id))  # Ensure ID is a string
+        fe.title(str(app['title']))  # Ensure title is a string
+        fe.link(href=f'https://repebblesearch.com/apps/{app_id}', rel='alternate')
         
-        app_title = app.get("title", "Unknown App")
+        # Set description (now includes all sections with headings)
-        app_id = app.get("uuid", "")
+        if final_description:
-        app_url = f"https://repebblesearch.com/app/{app_id}" if app_id else "https://repebblesearch.com"
+            fe.description(final_description)
+        else:
+            fe.description("No description available")
         
-        fe.title(app_title)
+        # Add screenshot URLs as enclosures (for readers that support them)
-        fe.link(href=app_url)
-        fe.guid(app_url)
-        
-        # Build description with screenshots and release notes
-        description_html = create_description_with_screenshots(app)
-        fe.description(description_html)
-        
-        # Add enclosures for screenshots (for readers that support them)
         screenshots = app.get("screenshots", [])
+        if not screenshots and 'screenshot_url' in app and app['screenshot_url']:
+            screenshots = [{'url': app['screenshot_url']}]
+        
         for screenshot in screenshots:
             url = screenshot.get("url", "")
             if url:
                 fe.enclosure(url=url, type="image/png")
         
-        # Set publication date from latest changelog entry or fallback
+        # Set publication date if available
         pub_date = None
+        
+        # Try changelog first
         changelog = app.get("changelog", [])
         if changelog and len(changelog) > 0:
             pub_date_str = changelog[0].get("published_date", "")
@@ -150,49 +255,62 @@ def create_rss_feed(filename, title, description, apps):
                 try:
                     pub_date = datetime.fromisoformat(pub_date_str.replace('Z', '+00:00'))
                 except:
-                    pub_date = datetime.now()
+                    pass
-        else:
-            pub_date = datetime.now()
         
-        if pub_date:
+        # Fallback to release_date
-            fe.pubDate(pub_date)
+        if pub_date is None and 'release_date' in app and app['release_date']:
+            try:
+                pub_date = datetime.fromisoformat(app['release_date'].replace('Z', '+00:00'))
+            except (ValueError, AttributeError):
+                pass
+        
+        # If no valid date, use current time with UTC timezone
+        if pub_date is None:
+            pub_date = datetime.now(timezone.utc)
+        
+        fe.pubDate(pub_date)
+        
+        # Add author if available
+        if 'author' in app and app['author']:
+            fe.author(name=str(app['author']))
     
-    # Ensure output folder exists
+    # Write the RSS feed to file
-    os.makedirs(OUTPUT_FOLDER, exist_ok=True)
+    try:
-    
+        fg.rss_file(output_file, pretty=True)
-    # Write RSS feed to file
+        print(f"  RSS feed successfully created: {output_file}")
-    output_path = os.path.join(OUTPUT_FOLDER, filename)
+        return True
-    fg.rss_file(output_path, pretty=True)
+    except Exception as e:
-    print(f"Feed created: {output_path}")
+        print(f"  Error writing RSS file: {e}")
+        return False
 
 def main():
-    """Main function to generate RSS feeds."""
+    # Create feeds folder if it doesn't exist
-    print("Fetching Pebble apps and generating RSS feeds...")
+    if not os.path.exists(FEEDS_FOLDER):
+        os.makedirs(FEEDS_FOLDER)
+        print(f"Created '{FEEDS_FOLDER}' folder\n")
     
-    # Fetch new apps
+    print("Generating Pebble apps RSS feeds...\n")
-    print("Fetching new apps...")
-    new_apps = fetch_apps("*", platform="basalt")
     
-    # Fetch updated apps (you might want to adjust the query)
+    for feed_config in FEEDS:
-    print("Fetching updated apps...")
+        feed_name = feed_config['name']
-    updated_apps = fetch_apps("*", platform="basalt")
+        feed_url = feed_config['url']
+        feed_description = feed_config['description']
+        output_file = os.path.join(FEEDS_FOLDER, f"{feed_name.lower().replace(' ', '_')}_feed.xml")
+        
+        print(f"Processing: {feed_name}")
+        print(f"  Fetching data from API...")
+        apps_data = fetch_apps_data(feed_url)
+        
+        if apps_data:
+            print(f"  Creating RSS feed...")
+            if create_rss_feed(apps_data, feed_name, feed_description, output_file):
+                print(f"  ✓ Done!\n")
+            else:
+                print(f"  ✗ Failed to create RSS feed\n")
+        else:
+            print(f"  ✗ Failed to fetch apps data\n")
     
-    # Create RSS feeds
+    print("All feeds generated!")
-    create_rss_feed(
-        "new_apps.xml",
-        "New Pebble Apps",
-        "Recently released Pebble apps from the Pebble Search database",
-        new_apps[:20]  # Limit to 20 most recent
-    )
-    
-    create_rss_feed(
-        "updated_apps.xml",
-        "Updated Pebble Apps",
-        "Recently updated Pebble apps from the Pebble Search database",
-        updated_apps[:20]  # Limit to 20 most recent
-    )
-    
-    print("Done!")
 
 if __name__ == "__main__":
     main()