# A creator's profile, recent posts and the comments on them
#
# Price for profile, one page of videos, first comment page on five videos: 7 credits held up front (exact; the unused hold is refunded).
# Measured on 25/10/2026: 7 credits in 7 calls for ITEMS=5 (the quote held 7). The TypeScript and Python programs both ran.
#
# ITEMS caps the rows a step collects or loops over (default 5, profile, one page of videos, first comment page on five videos); the price scales with it.
# FRESH=1 skips the cache and bills every call; a cache hit is free.
# Inputs, set to a working example (change them to yours):
#   handle: The TikTok handle
#
# Run it with Python 3.9 or newer, nothing to install: SOCIALCRAWL_API_KEY=sc_... python3 recipe.py

import json
import os
import random
import re
import sys
import time
import http.client
import urllib.error
import urllib.parse
import urllib.request
import uuid

API_KEY = os.environ.get("SOCIALCRAWL_API_KEY") or sys.exit("Set SOCIALCRAWL_API_KEY to your SocialCrawl API key")
BASE_URL = os.environ.get("SOCIALCRAWL_BASE_URL", "https://www.socialcrawl.dev").rstrip("/")
TIMEOUT_S = 300  # the longest route time limit, per attempt
MAX_RETRIES = 3
# FRESH=1 skips the cache: every call is fetched again and billed normally (a cache hit is free).
FRESH = os.environ.get("FRESH") == "1"
WAIT_LIMIT_S = 15 * 60  # how long to follow one run or job
RUNNING = {"queued", "running", "pending"}
# A job read answers the job's status; read it again until it is terminal.
JOB_READ = re.compile(r"^(web/jobs|cohort-queries)/[^/]+$")

call_count = 0
credits_spent = 0
run_credits = {}
created_paths = []


def dig(value, *keys):
    for key in keys:
        if not isinstance(value, dict):
            return None
        value = value.get(key)
    return value


def backoff(attempt):
    # Exponential backoff with jitter: about 1 s, 2 s, then 4 s.
    return 2 ** attempt * (0.5 + random.random() / 2)


def send(url, method="GET", body=None, idempotency_key=None):
    # One request. Retries a network error and an error the API marks retryable
    # (Retry-After wins), but a POST only when it carries an Idempotency-Key, so
    # a retry never runs a paid write twice. Returns the parsed body, or None for
    # RESOURCE_NOT_FOUND (refunded: nothing to read). Exits on any other error.
    global call_count, credits_spent
    headers = {"x-api-key": API_KEY, "accept": "application/json"}
    if FRESH:
        headers["Cache-Control"] = "no-cache"
    data = None
    if body is not None:
        data = json.dumps(body).encode()
        headers["content-type"] = "application/json"
    if idempotency_key:
        headers["Idempotency-Key"] = idempotency_key
    retries = MAX_RETRIES if method == "GET" or idempotency_key else 0
    for attempt in range(retries + 1):
        request = urllib.request.Request(url, data=data, method=method, headers=headers)
        try:
            with urllib.request.urlopen(request, timeout=TIMEOUT_S) as res:
                status, retry_after, payload = res.status, res.headers.get("Retry-After"), json.loads(res.read() or b"null")
        except urllib.error.HTTPError as err:
            status, retry_after = err.code, err.headers.get("Retry-After")
            try:
                payload = json.loads(err.read() or b"null")
            except (ValueError, OSError, http.client.HTTPException):
                payload = None
        # A network error, a timeout, a connection cut mid-body, or a body that is not JSON.
        except (OSError, http.client.HTTPException, ValueError):
            if attempt == retries:
                raise
            time.sleep(backoff(attempt))
            continue
        call_count += 1
        if 200 <= status < 300 and dig(payload, "success") is not False:
            # A run (prism/discover) reports its whole spend on data.run.credits.spent.
            run = dig(payload, "data", "run") or {}
            spent = dig(run, "credits", "spent")
            if run.get("run_id") and isinstance(spent, (int, float)):
                run_credits[run["run_id"]] = spent
            elif isinstance(payload, dict):
                credits_spent += payload.get("credits_used") or 0
            return payload
        # A gateway page that is not our JSON (a 502 or 503 in front of the API) is retryable.
        error = dig(payload, "error") or {"type": f"HTTP_{status}", "message": "no error body", "retryable": status >= 500}
        # Branch on error type, never on the message text.
        if error.get("type") == "RESOURCE_NOT_FOUND":
            print(f"Skipped, not found (refunded): {url}", file=sys.stderr)
            return None
        if not error.get("retryable") or attempt == retries:
            sys.exit(f"{error.get('type')}: {error.get('message')} (request_id {dig(payload, 'request_id')})")
        time.sleep(float(retry_after) if retry_after else backoff(attempt))


def note_created(path, method, body):
    # What a run created and keeps (a monitor, a cohort), printed with how to delete it.
    if method != "POST" or not body:
        return
    ids = {"monitors": dig(body, "monitor", "id"), "cohorts": dig(body, "data", "id"), "web/monitors": dig(body, "data", "monitor_id")}
    created = ids.get(path)
    if created:
        created_paths.append(f"{path}/{created}")
        print(f"Created {path}/{created}. Delete it with DELETE /v1/{path}/{created} when you no longer need it.")


def sc(path, params=None, method="GET", body=None, idempotency_key=None):
    # One API call by its path under /v1. Follows an async run (202) and a job to its end.
    url = f"{BASE_URL}/v1/{path}"
    query = {k: (str(v).lower() if isinstance(v, bool) else str(v)) for k, v in (params or {}).items() if v is not None}
    if query:
        url += "?" + urllib.parse.urlencode(query)
    result = send(url, method, body, idempotency_key)
    deadline = time.time() + WAIT_LIMIT_S
    # An async run answers 202 and keeps going: follow data.next.status_url until it ends.
    status_url = dig(result, "data", "next", "status_url")
    while status_url and dig(result, "data", "run", "status") in RUNNING:
        if time.time() > deadline:
            sys.exit(f"{path} is still running: follow {status_url}")
        time.sleep(1)
        result = send(status_url)
        status_url = dig(result, "data", "next", "status_url") or status_url
    while JOB_READ.match(path) and dig(result, "data", "status") in RUNNING:
        if time.time() > deadline:
            sys.exit(f"{path} is still running: read it again later")
        wait_s = dig(result, "data", "poll_after_s")
        time.sleep(5 if wait_s is None else wait_s)
        result = send(url)
    note_created(path, method, result)
    return result


def rows_in(value):
    # Rows in a step's result: a page's items, or the answers of a step run once per row.
    if isinstance(value, list):
        return len([v for v in value if v is not None])
    if not value:
        return 0
    items = dig(value, "data", "items")
    return len(items) if isinstance(items, list) else 1


ITEMS = int(os.environ.get("ITEMS", "5"))

profile = sc("tiktok/profile", {"handle": "charlidamelio"})
videos = sc("tiktok/profile/videos", {"handle": "charlidamelio"})
comments = []
for row in videos["data"]["items"][:ITEMS]:
    comments.append(sc("tiktok/post/comments", {"url": row["post"]["url"]}))

total_credits = credits_spent + sum(run_credits.values())
print(f"Spent {total_credits} credits in {call_count} calls.")
row_counts = {"profile": rows_in(profile), "videos": rows_in(videos), "comments": rows_in(comments)}
print(
    "RECIPE_RESULT "
    + json.dumps(
        {
            "recipe": "creator-profile-posts-comments",
            "calls": call_count,
            "credits_used": total_credits,
            "rows": row_counts,
            "created": created_paths,
        }
    )
)
