twitch

import json, re, time, urllib.parse

CATEGORY_SLUG = "valorant"      # the <slug> in /directory/category/<slug> (block 2 lists them)
SORT = "VIEWER_COUNT"           # RELEVANCE | VIEWER_COUNT | VIEWER_COUNT_ASC | RECENT
MAX_CHANNELS = 150              # ~24-26 cards load per scroll step
PAUSE = 2.0                     # seconds between scroll steps (polite pacing)

BASE = "https://www.twitch.tv/directory/category/" + urllib.parse.quote(CATEGORY_SLUG, safe="")
URL = BASE + "?" + urllib.parse.urlencode({"sort": SORT})

# Generic bot-check markers (none were seen on Twitch at this pace). On a hit: stop and raise, never retry around it.
CHALLENGE_JS = r"""(() => {
  const t = (document.title + ' ' + (document.body ? document.body.innerText.slice(0, 3000) : '')).toLowerCase();
  return /verify (that )?you are (a )?human|are you a robot|unusual traffic from your|press (and|&) hold/.test(t)
    || !!document.querySelector('iframe[src*="captcha"], iframe[src*="challenges.cloudflare.com"], iframe[src*="arkoselabs"], iframe[src*="funcaptcha"], #challenge-form');
})()"""

def safe_js(expr):
    try:
        return js(expr)
    except Exception:       # the page context is replaced mid-navigation
        return None

def open_page(url, timeout=20):
    """Navigate the attached tab (new tab only if it is not on twitch yet); confirm the exact URL, query included."""
    href = safe_js("location.href") or ""
    if href.startswith("https://www.twitch.tv/"):
        goto_url(url)
    else:
        new_tab(url)
    deadline = time.time() + timeout
    while True:
        href = safe_js("location.href") or ""
        if safe_js(CHALLENGE_JS):
            raise RuntimeError(f"bot check on {href}; stopping (do not retry around it)")
        if href.split("#")[0] == url and safe_js("document.readyState") == "complete":
            return href
        if time.time() > deadline:
            raise RuntimeError(f"expected {url}, tab is on {href}")
        time.sleep(0.5)

CARDS_JS = r"""(() => [...document.querySelectorAll('article')].map(art => {
  const link = art.querySelector('a[data-a-target="preview-card-channel-link"]');
  const img = art.querySelector('a[data-a-target="preview-card-image-link"]');
  if (!link || !img) return null;                     // skips the "May Contain Labeled Content" notice
  const title = link.querySelector('h4');
  const name = link.querySelector('p[title]');
  const stat = img.querySelector('.tw-media-card-stat');
  return {
    login: (link.getAttribute('href') || '').replace(/^\//, ''),
    display_name: name ? name.getAttribute('title') : null,
    title: title ? (title.getAttribute('title') || title.textContent) : null,
    viewers_text: stat ? stat.textContent.trim() : null,
    tags: [...art.querySelectorAll('button.tw-tag')].map(b => b.textContent.trim()),
    verified: [...link.querySelectorAll('svg title')].some(t => t.textContent === 'Verified'),
    thumbnail: (img.querySelector('img.tw-image') || {}).src || null,
  };
}).filter(Boolean))()"""

STATE_JS = r"""(() => {
  const t = (document.querySelector('main') || document.body).innerText;
  if (/Category does not exist/.test(t)) return 'missing';
  if (/No results available/.test(t)) return 'empty'; // category exists, nobody live right now
  return '';
})()"""

# The grid scrolls this inner container; at the true end it sits at its bottom and no new cards arrive.
AT_END_JS = r"""(() => {
  const s = document.querySelector('[data-a-target="root-scroller"]');
  return s ? s.scrollHeight - s.scrollTop - s.clientHeight <= 2 : null;
})()"""
REQUIRED = ("login", "display_name", "title", "viewers_text")
EMAIL = re.compile(r"[\w.+-]+@[\w-]+(?:\.[\w-]+)+")
PHONE = re.compile(r"\+?\d[\d\s().-]{7,}\d")

def redact(s):
    """Mask emails and phone-like numbers (9+ digits) in free text such as stream titles."""
    if not s:
        return s
    s = EMAIL.sub("[email]", s)
    return PHONE.sub(lambda m: "[phone]" if sum(ch.isdigit() for ch in m.group()) >= 9 else m.group(), s)

def viewers_to_int(s):
    """'27.3K viewers' -> 27300, '762 viewers' -> 762, '1 viewer' -> 1 (Twitch rounds K/M)."""
    m = re.match(r"([\d.,]+)\s*([KM]?)", s or "")
    if not m:
        return None
    n = float(m.group(1).replace(",", ""))
    return int(round(n * {"": 1, "K": 1_000, "M": 1_000_000}[m.group(2)]))

def read_cards(seen, incomplete):
    """Add fully rendered cards to seen; remember logins whose card still lacks a required field."""
    if js(CHALLENGE_JS):
        raise RuntimeError("bot check appeared; stopping, no partial list returned")
    for c in js(CARDS_JS):
        if all(c.get(k) for k in REQUIRED):
            seen.setdefault(c["login"], c)
            incomplete.discard(c["login"])
        elif c.get("login") and c["login"] not in seen:
            incomplete.add(c["login"])

open_page(URL)

# Wait for the first cards (the grid renders client-side after load).
deadline, empty_polls = time.time() + 20, 0
while not js(CARDS_JS):
    if js(CHALLENGE_JS):
        raise RuntimeError("bot check on the category page; stopping")
    state = js(STATE_JS)
    if state == "missing":
        raise SystemExit(f"Twitch says category '{CATEGORY_SLUG}' does not exist")
    empty_polls = empty_polls + 1 if state == "empty" else 0
    if empty_polls >= 3:      # stable empty state, not a loading flash
        break
    if time.time() > deadline:
        raise RuntimeError("no channel cards and no empty-state message within 20 s")
    time.sleep(1)

# Infinite scroll: wheel over the grid until MAX_CHANNELS or a verified end of list.
seen, incomplete, stalls, at_end = {}, set(), 0, empty_polls >= 3
while not at_end and len(seen) < MAX_CHANNELS:
    before = len(seen)
    read_cards(seen, incomplete)
    stalls = stalls + 1 if len(seen) == before else 0
    if len(seen) >= MAX_CHANNELS:
        break
    if stalls >= 3:           # 3 scroll steps with no new card
        if not js(AT_END_JS):
            raise RuntimeError(f"cards stopped loading at {len(seen)} but the list is not at its end; no partial list returned")
        at_end = True
        break
    scroll(js("Math.round(innerWidth * 0.55)"), 450, dy=3000)   # pointer over the grid, not the sidebar
    time.sleep(PAUSE)

deadline = time.time() + 10   # give half-rendered cards a moment, then fail loudly
while incomplete and time.time() < deadline:
    time.sleep(1)
    read_cards(seen, incomplete)
if incomplete:
    raise RuntimeError(f"{len(incomplete)} cards never rendered {REQUIRED}; no partial list returned")

channels = list(seen.values())[:MAX_CHANNELS]
for pos, c in enumerate(channels, 1):
    c["title"] = redact(c["title"])
    c["grid_position"] = pos          # order the cards were appended while scrolling
    c["viewers"] = viewers_to_int(c["viewers_text"])
    c["url"] = "https://www.twitch.tv/" + c["login"]
# Counts change while you scroll, so grid order is only roughly sorted; re-sort by the card counts.
if SORT in ("VIEWER_COUNT", "VIEWER_COUNT_ASC"):
    channels.sort(key=lambda c: c["viewers"] if c["viewers"] is not None else -1, reverse=SORT == "VIEWER_COUNT")
for rank, c in enumerate(channels, 1):
    c["rank"] = rank

status = {"complete": at_end, "stopped_at_cap": not at_end, "loaded": len(seen), "returned": len(channels)}
if at_end:
    print(len(channels), "live channels in", CATEGORY_SLUG, "sorted by", SORT, "- complete, reached the end of the list")
else:
    more = "more were already loaded" if len(seen) > MAX_CHANNELS else "the list may continue"
    print(f"TRUNCATED: stopped at MAX_CHANNELS={MAX_CHANNELS} ({more}); {len(channels)} channels in {CATEGORY_SLUG} sorted by {SORT}")
print(json.dumps(status))
print(json.dumps(channels[:2], indent=1, ensure_ascii=False))

TRUNCATED: stopped at MAX_CHANNELS=150 (more were already loaded); 150 channels in valorant sorted by VIEWER_COUNT
{"complete": false, "stopped_at_cap": true, "loaded": 167, "returned": 150}
[
 {
  "login": "<login>",
  "display_name": "<display name>",
  "title": "<stream title>",
  "viewers_text": "6.4K viewers",
  "tags": ["<tag>", "<tag>", "<tag>"],
  "verified": false,
  "thumbnail": "https://static-cdn.jtvnw.net/previews-ttv/live_user_<login>-640x360.jpg",
  "grid_position": 27,
  "viewers": 6400,
  "url": "https://www.twitch.tv/<login>",
  "rank": 1
 },
 ...
]

import json, re, time

MAX_CATEGORIES = 100   # ~30 box-art cards load per scroll step
PAUSE = 2.0

# Generic bot-check markers (none were seen on Twitch at this pace). On a hit: stop and raise, never retry around it.
CHALLENGE_JS = r"""(() => {
  const t = (document.title + ' ' + (document.body ? document.body.innerText.slice(0, 3000) : '')).toLowerCase();
  return /verify (that )?you are (a )?human|are you a robot|unusual traffic from your|press (and|&) hold/.test(t)
    || !!document.querySelector('iframe[src*="captcha"], iframe[src*="challenges.cloudflare.com"], iframe[src*="arkoselabs"], iframe[src*="funcaptcha"], #challenge-form');
})()"""

def safe_js(expr):
    try:
        return js(expr)
    except Exception:       # the page context is replaced mid-navigation
        return None

def open_page(url, timeout=20):
    """Navigate the attached tab (new tab only if it is not on twitch yet); confirm the exact URL, query included."""
    href = safe_js("location.href") or ""
    if href.startswith("https://www.twitch.tv/"):
        goto_url(url)
    else:
        new_tab(url)
    deadline = time.time() + timeout
    while True:
        href = safe_js("location.href") or ""
        if safe_js(CHALLENGE_JS):
            raise RuntimeError(f"bot check on {href}; stopping (do not retry around it)")
        if href.split("#")[0] == url and safe_js("document.readyState") == "complete":
            return href
        if time.time() > deadline:
            raise RuntimeError(f"expected {url}, tab is on {href}")
        time.sleep(0.5)

CATS_JS = r"""(() => [...document.querySelectorAll('div.game-card')].map(c => {
  const a = c.querySelector('a[data-a-target="tw-box-art-card-link"]');
  const h = c.querySelector('[data-a-target="tw-card-title"] h2');
  const v = c.querySelector('p a[aria-label*="viewers for"]');
  if (!a || !h) return null;
  return {
    name: h.getAttribute('title') || h.textContent,
    path: a.getAttribute('href'),
    viewers_text: v ? v.textContent.trim() : null,
    tags: [...c.querySelectorAll('.tw-tag')].map(t => t.textContent.trim()),
    box_art: (a.querySelector('img') || {}).src || null,
  };
}).filter(Boolean))()"""

def viewers_to_int(s):
    m = re.match(r"([\d.,]+)\s*([KM]?)", s or "")
    if not m:
        return None
    return int(round(float(m.group(1).replace(",", "")) * {"": 1, "K": 1_000, "M": 1_000_000}[m.group(2)]))

AT_END_JS = r"""(() => {
  const s = document.querySelector('[data-a-target="root-scroller"]');
  return s ? s.scrollHeight - s.scrollTop - s.clientHeight <= 2 : null;
})()"""
REQUIRED = ("name", "path", "viewers_text")

def read_cats(seen, incomplete):
    if js(CHALLENGE_JS):
        raise RuntimeError("bot check appeared; stopping, no partial list returned")
    for c in js(CATS_JS):
        if all(c.get(k) for k in REQUIRED):
            seen.setdefault(c["path"], c)
            incomplete.discard(c["path"])
        elif c.get("path") and c["path"] not in seen:
            incomplete.add(c["path"])

open_page("https://www.twitch.tv/directory")
deadline = time.time() + 20
while not js(CATS_JS):
    if js(CHALLENGE_JS):
        raise RuntimeError("bot check on /directory; stopping")
    if time.time() > deadline:
        raise RuntimeError("no category cards rendered within 20 s")
    time.sleep(1)

seen, incomplete, stalls, at_end = {}, set(), 0, False
while len(seen) < MAX_CATEGORIES:
    before = len(seen)
    read_cats(seen, incomplete)
    stalls = stalls + 1 if len(seen) == before else 0
    if len(seen) >= MAX_CATEGORIES:
        break
    if stalls >= 3:
        if not js(AT_END_JS):
            raise RuntimeError(f"cards stopped loading at {len(seen)} but the list is not at its end; no partial list returned")
        at_end = True
        break
    scroll(js("Math.round(innerWidth * 0.55)"), 450, dy=3000)   # pointer over the grid, not the sidebar
    time.sleep(PAUSE)

deadline = time.time() + 10
while incomplete and time.time() < deadline:
    time.sleep(1)
    read_cats(seen, incomplete)
if incomplete:
    raise RuntimeError(f"{len(incomplete)} category cards never rendered {REQUIRED}; no partial list returned")

cats = []
for rank, c in enumerate(list(seen.values())[:MAX_CATEGORIES], 1):
    kind, _, slug = c["path"].strip("/").partition("/")[2].partition("/")   # "category"/"event", slug
    cats.append({**c, "rank": rank, "kind": kind, "slug": slug,
                 "viewers": viewers_to_int(c["viewers_text"]),
                 "url": "https://www.twitch.tv" + c["path"]})

status = {"complete": at_end, "stopped_at_cap": not at_end, "loaded": len(seen), "returned": len(cats)}
if not at_end:
    more = "more were already loaded" if len(seen) > MAX_CATEGORIES else "the list may continue"
    print(f"TRUNCATED: stopped at MAX_CATEGORIES={MAX_CATEGORIES} ({more})")
print(len(cats), "categories;", sum(c["kind"] == "category" for c in cats), "are /directory/category/ pages")
print(json.dumps(status))
print(json.dumps(cats[:3], indent=1, ensure_ascii=False))

TRUNCATED: stopped at MAX_CATEGORIES=100 (more were already loaded)
100 categories; 99 are /directory/category/ pages
{"complete": false, "stopped_at_cap": true, "loaded": 109, "returned": 100}
[
 {"name": "Stream-o-Ween", "path": "/directory/event/stream-o-ween-2026", "viewers_text": "34.4K viewers", "tags": [], "kind": "event", "slug": "stream-o-ween-2026", "viewers": 34400, ...},
 {"name": "Just Chatting", "path": "/directory/category/just-chatting", "viewers_text": "200K viewers", "tags": ["IRL"],
  "box_art": "https://static-cdn.jtvnw.net/ttv-boxart/509658-285x380.jpg", "rank": 2, "kind": "category", "slug": "just-chatting", "viewers": 200000,
  "url": "https://www.twitch.tv/directory/category/just-chatting"},
 {"name": "Call of Duty: Modern Warfare 4", "path": "/directory/category/call-of-duty-modern-warfare-4", "viewers_text": "67.4K viewers", "tags": ["FPS", "Shooter", "Action"], "rank": 3, ...}
]

import gzip, html, json, re, time, urllib.error, urllib.parse, urllib.request

LOGINS = ["twitchgaming", "twitch", "zzqx_no_such_channel_9871"]
UA = "twitch-channel-about/1.0 (python-urllib; one request per second)"   # truthful client UA
PAUSE = 1.0

BUDGET = 120   # one deadline per channel for every request, backoff and refetch
EMAIL = re.compile(r"[\w.+-]+@[\w-]+(?:\.[\w-]+)+")
PHONE = re.compile(r"\+?\d[\d\s().-]{7,}\d")

def redact(s):
    """Mask emails and phone-like numbers (9+ digits) in free text (bios, titles)."""
    if not s:
        return s
    s = EMAIL.sub("[email]", s)
    return PHONE.sub(lambda m: "[phone]" if sum(ch.isdigit() for ch in m.group()) >= 9 else m.group(), s)

# Bot-check markers searched in the WHOLE body of every response (200 or error). A bare "captcha" is not one:
# normal Twitch pages (4 of 4 saved) name a "features.auth.components.captcha.fun-captcha" script chunk.
CHALLENGE_RE = re.compile(
    r"challenges\.cloudflare\.com|/cdn-cgi/challenge-platform|captcha-delivery\.com|px-captcha|g-recaptcha|h-captcha"
    r"|arkoselabs|verify (that )?you are (a )?human|are you a robot|unusual traffic from your"
    r"|<title[^>]*>[^<]*(captcha|access denied|attention required|just a moment)", re.I)

def body(data, headers):
    if (headers.get("Content-Encoding") or "") == "gzip":
        data = gzip.decompress(data)
    return data.decode("utf-8", "replace")

def check_challenge(url, page):
    m = CHALLENGE_RE.search(page)
    if m:
        raise RuntimeError(f"{url}: bot check page (marker {m.group(0)[:60]!r}); stopping, not retrying")

def fetch(url, deadline):
    """GET with exponential backoff on 429/5xx until `deadline`; no sleep when the next try would miss it.
    The body is checked for a bot check before any retry or acceptance."""
    delay = 2
    while True:
        left = deadline - time.time()
        if left <= 0:
            raise TimeoutError(f"{url}: deadline reached")
        req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept-Encoding": "gzip", "Accept-Language": "en-US"})
        try:
            with urllib.request.urlopen(req, timeout=min(20, left)) as r:
                page = body(r.read(), r.headers)
        except urllib.error.HTTPError as e:
            try:
                err_page = body(e.read(), e.headers or {})
            except Exception:          # no readable error body
                err_page = ""
            check_challenge(url, err_page)   # a challenge on a 403/429/5xx stops here, before any retry
            if e.code not in (429, 500, 502, 503, 504) or time.time() + delay >= deadline:
                raise          # 403 and other blocks surface at once; retryable codes stop at the deadline
            time.sleep(delay)
            delay *= 2
            continue
        check_challenge(url, page)
        return page

def meta(page, key):
    for pat in (rf'<meta (?:property|name)="{key}" content="([^"]*)"', rf'<meta content="([^"]*)" (?:property|name)="{key}"'):
        m = re.search(pat, page)
        if m:
            return html.unescape(m.group(1))
    return None

def channel_about(login, attempts=4):
    url = "https://www.twitch.tv/" + urllib.parse.quote(login.lower(), safe="")
    deadline = time.time() + BUDGET          # all requests and waits for one login share this deadline
    for attempt in range(attempts):
        page = fetch(url, deadline)
        graph = []
        for block in re.findall(r'<script type="application/ld\+json">(.*?)</script>', page, re.S):
            graph += json.loads(block).get("@graph", [])
        profile = next((g for g in graph if g.get("@type") == "ProfilePage"), None)
        if profile:
            break
        # Unknown (or unavailable) login: Twitch's generic page, og:title "Twitch" and robots "noindex".
        # A real channel never carries both, so this is a positive "missing" signal, not a fallback.
        if meta(page, "og:title") == "Twitch" and meta(page, "robots") == "noindex":
            return {"login": login, "error": "no such channel (or unavailable)"}
        # Anything else (e.g. a ~217 KB shell with no meta tags and no JSON-LD) is transient: refetch.
        wait = 5 * 2 ** attempt              # 5, 10, 20 s
        if attempt == attempts - 1 or time.time() + wait > deadline:
            break
        time.sleep(wait)
    if not profile:
        raise RuntimeError(f"{url}: no ProfilePage JSON-LD and no 'missing channel' marker after {attempt + 1} tries")
    person = profile.get("mainEntity") or {}
    stat = person.get("interactionStatistic") or {}
    if not person.get("name") or not isinstance(stat.get("userInteractionCount"), int):
        raise RuntimeError(f"{url}: ProfilePage lacks name or follower count; not accepting it")
    videos = next((g for g in graph if g.get("@type") == "ItemList"), {}).get("itemListElement", [])
    title = meta(page, "og:title") or ""
    desc = meta(page, "og:description") or ""
    live = title.endswith(" - Live on Twitch")
    # Live og:description is "<title> | Streaming <category> for N viewers." OR "<title> | Streaming <category>."
    # N is NOT used: it was off by up to 300x versus the live count (see Gotchas). Use block 1 or 4 for viewers.
    m = re.match(r"(.*)\|\s*Streaming (.+?)(?: for \d+ viewers?)?\.$", desc, re.S) if live else None
    if live and not m:
        raise RuntimeError(f"{url}: live, but og:description has an unknown shape: {desc[-80:]!r}")
    return {
        "login": person.get("alternateName") or login,
        "display_name": person.get("name"),
        "url": person.get("url") or url,
        "bio": redact(person.get("description")),
        "profile_image": person.get("image"),
        "followers": stat.get("userInteractionCount"),
        "social_links": [u for u in person.get("sameAs", []) if not u.startswith(("mailto:", "tel:"))],
        "live": live,
        "live_title": redact(m.group(1).strip()) if m else None,
        "live_category": m.group(2) if m else None,   # lowercase category name, e.g. "just chatting"
        "recent_videos": [{"title": redact(v.get("name")), "url": v.get("url"), "uploaded": v.get("uploadDate"),
                           "duration": v.get("duration"),
                           "views": (v.get("interactionStatistic") or {}).get("userInteractionCount")}
                          for v in videos if v.get("@type") == "VideoObject"],
        "page_modified": profile.get("dateModified"),
    }

out = []
for i, login in enumerate(LOGINS):
    if i:
        time.sleep(PAUSE)
    out.append(channel_about(login))
print(json.dumps([{**o, "recent_videos": o["recent_videos"][:2]} if "recent_videos" in o else o for o in out], indent=1, ensure_ascii=False))

[
 {
  "login": "twitchgaming",
  "display_name": "twitchgaming",
  "url": "https://www.twitch.tv/twitchgaming",
  "bio": "Join /twitchgaming and be a part of the conversation! ...",
  "profile_image": "https://static-cdn.jtvnw.net/jtv_user_pictures/77eb821f-...-profile_image-300x300.jpeg",
  "followers": 584895,
  "social_links": ["https://twitter.com/twitch", "https://instagram.com/twitch"],
  "live": false, "live_title": null, "live_category": null,
  "recent_videos": [],
  "page_modified": "2023-10-15T02:56:52Z"
 },
 {
  "login": "twitch", "display_name": "Twitch", "followers": 2483231,
  "social_links": ["https://twitter.com/twitch", "https://www.instagram.com/twitch/", "https://www.tiktok.com/@twitch", "https://www.youtube.com/twitch"],
  "recent_videos": [
   {"title": "Let's Chat Episode 14 | ...", "url": "https://www.twitch.tv/videos/2908640776", "uploaded": "2026-10-23T16:55:16Z", "duration": "PT3353S", "views": 8},
   ...
  ], ...
 },
 {"login": "zzqx_no_such_channel_9871", "error": "no such channel (or unavailable)"}
]

import json, re, time, urllib.parse

LOGINS = ["twitchgaming", "twitch"]   # channel logins, e.g. the "login" field from block 1
PAUSE = 2.0                           # seconds between page loads
RENDER_TIMEOUT = 20                   # seconds to wait for a positive live or offline marker

# Generic bot-check markers (none were seen on Twitch at this pace). On a hit: stop and raise, never retry around it.
CHALLENGE_JS = r"""(() => {
  const t = (document.title + ' ' + (document.body ? document.body.innerText.slice(0, 3000) : '')).toLowerCase();
  return /verify (that )?you are (a )?human|are you a robot|unusual traffic from your|press (and|&) hold/.test(t)
    || !!document.querySelector('iframe[src*="captcha"], iframe[src*="challenges.cloudflare.com"], iframe[src*="arkoselabs"], iframe[src*="funcaptcha"], #challenge-form');
})()"""

def safe_js(expr):
    try:
        return js(expr)
    except Exception:       # the page context is replaced mid-navigation
        return None

def open_page(url, timeout=20):
    """Navigate the attached tab (new tab only if it is not on twitch yet); confirm the exact URL, query included."""
    href = safe_js("location.href") or ""
    if href.startswith("https://www.twitch.tv/"):
        goto_url(url)
    else:
        new_tab(url)
    deadline = time.time() + timeout
    while True:
        href = safe_js("location.href") or ""
        if safe_js(CHALLENGE_JS):
            raise RuntimeError(f"bot check on {href}; stopping (do not retry around it)")
        if href.split("#")[0] == url and safe_js("document.readyState") == "complete":
            return href
        if time.time() > deadline:
            raise RuntimeError(f"expected {url}, tab is on {href}")
        time.sleep(0.5)

ABOUT_JS = r"""(() => {
  const main = document.querySelector('main') || document.body;
  if (!main) return null;
  if (/Sorry\. Unless you've got a time machine, that content is unavailable/.test(main.innerText)) return {missing: true};
  const p = document.querySelector('[data-a-target="about-panel"]');
  if (!p || !p.querySelector('h3')) return null;   // panel shell renders before its content
  const f = [...p.querySelectorAll('div')].find(d => / followers?$/.test(d.textContent.trim()) && d.children.length <= 1);
  const name = (document.querySelector('main h1') || {}).textContent || null;
  if (!name || !f) return null;                    // required fields not rendered yet: keep polling
  return {
    display_name: name,
    about_heading: (p.querySelector('h3') || {}).textContent || null,
    followers_text: f.textContent.trim(),
    bio: (p.querySelector('p[dir="auto"]') || {}).textContent || null,
    partner: !!p.querySelector('svg[aria-label="Verified Partner"]'),
    social_links: [...p.querySelectorAll('a[role="link"][target="_blank"]')].map(a => ({label: a.textContent.trim(), url: a.href})),
    panels: [...document.querySelectorAll('[data-test-selector="channel_panel_test_selector"]')].map(x => ({
      text: x.innerText.trim().slice(0, 500),
      images: [...x.querySelectorAll('img')].map(i => ({src: i.src, alt: i.alt || null})),
      links: [...x.querySelectorAll('a[href^="http"]')].map(a => a.href),
    })),
  };
})()"""

# {live: false} only on Twitch's own OFFLINE marker; the live dict only once title, viewers and uptime
# all exist; null while neither has rendered (the live header appeared 3.9-5.1 s after navigation).
LIVE_JS = r"""(() => {
  const t = s => { const e = document.querySelector(s); return e ? e.textContent.trim() : null; };
  const st = document.querySelector('.channel-status-info');
  if (document.querySelector('.channel-root__player--offline') && st && /^OFFLINE\b/.test(st.innerText.trim())) return {live: false};
  const title = t('[data-a-target="stream-title"]');
  const viewers = t('[data-a-target="animated-channel-viewers-count"]');
  const uptime = ((document.querySelector('.live-time') || {}).innerText || '').split('\n')[0] || null;
  if (!title || !viewers || !uptime) return null;
  const g = document.querySelector('[data-a-target="stream-game-link"]');
  return {
    live: true,
    stream_title: title,
    category: g ? g.textContent.trim() : null,
    category_url: g ? g.href : null,
    viewers_text: viewers,
    uptime: uptime,
    tags: [...document.querySelectorAll('main .tw-tag')].map(b => b.textContent.trim()),
  };
})()"""

EMAIL = re.compile(r"[\w.+-]+@[\w-]+(?:\.[\w-]+)+")
PHONE = re.compile(r"\+?\d[\d\s().-]{7,}\d")

def redact(s):
    """Mask emails and phone-like numbers (9+ digits) in free text (bios, panels, titles)."""
    if not s:
        return s
    s = EMAIL.sub("[email]", s)
    return PHONE.sub(lambda m: "[phone]" if sum(ch.isdigit() for ch in m.group()) >= 9 else m.group(), s)

def count(s):
    """'585K followers' -> 585000, '1,175' -> 1175 (follower counts are rounded by Twitch)."""
    m = re.match(r"([\d.,]+)\s*([KM]?)", s or "")
    if not m:
        return None
    return int(round(float(m.group(1).replace(",", "")) * {"": 1, "K": 1_000, "M": 1_000_000}[m.group(2)]))

def poll(expr, what, timeout=RENDER_TIMEOUT):
    deadline, err = time.time() + timeout, None
    while True:
        try:
            blocked, r = js(CHALLENGE_JS), js(expr)
        except Exception as e:      # navigation in progress, or a tab that stopped answering
            blocked, r, err = False, None, e
        if blocked:
            raise RuntimeError(f"bot check while waiting for {what}; stopping")
        if r:
            return r
        if time.time() > deadline:
            raise RuntimeError(f"{what} did not render within {timeout} s" + (f"; last JS error: {str(err)[:120]}" if err else ""))
        time.sleep(1)

results = []
for i, login in enumerate(LOGINS):
    if i:
        time.sleep(PAUSE)
    base = "https://www.twitch.tv/" + urllib.parse.quote(login.lower(), safe="")
    open_page(base + "/about")
    about = poll(ABOUT_JS, f"about panel for {login}")
    if about.get("missing"):   # unknown logins: "Sorry. Unless you've got a time machine, that content is unavailable."
        results.append({"login": login, "error": "content unavailable"})
        continue
    about["followers"] = count(about["followers_text"])
    about["bio"] = redact(about["bio"])
    about["social_links"] = [l for l in about["social_links"] if not l["url"].startswith(("mailto:", "tel:"))]
    for x in about["panels"]:
        x["text"] = redact(x["text"])
        x["links"] = [u for u in x["links"] if not u.startswith(("mailto:", "tel:"))]
    time.sleep(PAUSE)
    open_page(base)
    live = poll(LIVE_JS, f"live header or OFFLINE marker for {login}")   # never infer offline from absence
    if live["live"]:
        live["stream_title"] = redact(live["stream_title"])
        live["viewers"] = count(live["viewers_text"])
    results.append({"login": login, "url": base, **about, **live})

print(json.dumps([{**r, "panels": r.get("panels", [])[:2]} for r in results], indent=1, ensure_ascii=False))

[
 {
  "login": "twitchgaming",
  "url": "https://www.twitch.tv/twitchgaming",
  "display_name": "twitchgaming",
  "about_heading": "About twitchgaming",
  "followers_text": "585K followers",
  "bio": "Join /twitchgaming and be a part of the conversation! ...",
  "partner": true,
  "social_links": [{"label": "Twitter", "url": "https://twitter.com/twitch"}, {"label": "Instagram", "url": "https://instagram.com/twitch"}],
  "panels": [{"text": "", "images": [...], "links": []}, ...],
  "followers": 585000,
  "live": false
 },
 {"login": "twitch", "display_name": "Twitch", "followers_text": "2.5M followers", "partner": true,
  "panels": [{"text": "", "images": [{"src": "https://panels.twitch.tv/panel-12826-image-...", "alt": "Panel Content"}], "links": ["https://twitter.com/Twitch"]}, ...],
  "live": false, ...}
]