news-aggregation

from concurrent.futures import ThreadPoolExecutor
import xml.etree.ElementTree as ET

RSS_FEEDS = [
    ("TechCrunch",     "https://techcrunch.com/feed/"),
    ("Ars Technica",   "https://feeds.arstechnica.com/arstechnica/index"),
    ("BBC News",       "http://feeds.bbci.co.uk/news/rss.xml"),
    ("Guardian World", "https://www.theguardian.com/world/rss"),
    ("Wired",          "https://www.wired.com/feed/rss"),
    ("NPR",            "https://feeds.npr.org/1001/rss.xml"),
    ("Wired",          "https://www.wired.com/feed/rss"),
]

def fetch_rss(name_url):
    name, url = name_url
    xml_data = http_get(url)
    root = ET.fromstring(xml_data)
    items = root.findall('.//item')
    return name, items

with ThreadPoolExecutor(max_workers=len(RSS_FEEDS)) as ex:
    results = list(ex.map(fetch_rss, RSS_FEEDS))

for name, items in results:
    for item in items[:5]:
        title = item.find('title').text
        link  = item.find('link').text
        print(f"[{name}] {title}")

import xml.etree.ElementTree as ET

NS = {'atom': 'http://www.w3.org/2005/Atom'}

xml_data = http_get("https://www.theverge.com/rss/index.xml")
root = ET.fromstring(xml_data)
entries = root.findall('.//atom:entry', NS)   # 10 entries

for e in entries:
    title = e.find('atom:title', NS).text
    link  = e.find('atom:link', NS).get('href')
    print(title, link)

Reuters: ERROR HTTP Error 401: HTTP Forbidden   # with AND without User-Agent

headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"}
html = http_get("https://www.nytimes.com", headers=headers)  # 1.1MB, works
html = http_get("https://news.ycombinator.com")               # 34KB, works without UA

import re
html = http_get("https://news.ycombinator.com")
stories = re.findall(r'class="titleline"><a href="([^"]+)"[^>]*>([^<]+)<', html)
# Returns list of (url, title) tuples — 30 stories on the front page

goto_url("https://www.bbc.com/news")
wait_for_load()
wait(2)

headlines = js("""
  Array.from(document.querySelectorAll('article h2'))
    .map(h => ({
      title: h.innerText.trim(),
      url: h.closest('a')?.href || h.closest('[href]')?.href || 
           h.parentElement.querySelector('a')?.href
    }))
    .filter(h => h.title.length > 10)
""")
# Returns 50+ articles. First one is typically LIVE/breaking.

accept = js("""
  var btns = Array.from(document.querySelectorAll('button'));
  var btn = btns.find(b => /accept all|agree|continue/i.test(b.innerText));
  if (btn) { btn.click(); return 'clicked: ' + btn.innerText; }
  return 'no banner';
""")

goto_url("https://techcrunch.com")
wait_for_load()
wait(2)

articles = js("""
  Array.from(document.querySelectorAll('h3'))
    .map(h => ({
      title: h.innerText?.trim(),
      url: h.closest('a')?.href || h.querySelector('a')?.href || 
           h.parentElement?.querySelector('a')?.href
    }))
    .filter(a => a.title && a.title.length > 20)
""")
# Returns ~10 articles. RSS is preferred (20 items, no JS required).

goto_url("https://www.reuters.com")
wait_for_load()
wait(3)
js("window.scrollTo(0, 500)")
wait(1)
# Category links work for topic nav:
links = js("""
  Array.from(document.querySelectorAll('a[href*="/world/"], a[href*="/technology/"]'))
    .filter(a => a.innerText.trim().length > 20)
    .map(a => ({text: a.innerText.trim(), href: a.href}))
""")

Does the site have an RSS/Atom feed?
  YES → http_get + XML parse (fastest, ~0.1s per feed)
         - RSS 2.0: root.findall('.//item')
         - Atom:    root.findall('.//atom:entry', {'atom': 'http://www.w3.org/2005/Atom'})
  NO  → Does http_get return valid HTML (not 403/401/JS shell)?
          YES → http_get + regex/BeautifulSoup (fast, ~0.2-0.3s)
          NO  → goto + wait_for_load + wait(2) + js() extraction (slow, 3-8s)