Monthly brand reaction

// How people reacted to a brand and its competitors last month
//
// Price for a brand and its competitors, one month: at most 150 credits (max_credits caps the whole run; each search bills at its platform's price only when it returns rows, so it settles 0-150; nothing beyond the judging bound is held up front).
// Measured on 25/10/2026: 35 credits in 5 calls (the quote held 150). The TypeScript and Python programs both ran.
//
// FRESH=1 skips the cache and bills every call; a cache hit is free.
// Inputs, set to a working example (change them to yours):
//   brands: The brand and its competitors, comma-separated
//   exclude_terms: Words that mark another meaning of a brand name, comma-separated
//
// Run it with Node 22 or newer: SOCIALCRAWL_API_KEY=sc_... npx tsx recipe.ts

const API_KEY = process.env.SOCIALCRAWL_API_KEY;
if (!API_KEY) throw new Error("Set SOCIALCRAWL_API_KEY to your SocialCrawl API key");
const BASE_URL = (process.env.SOCIALCRAWL_BASE_URL ?? "https://www.socialcrawl.dev").replace(/\/+$/, "");
const TIMEOUT_MS = 300_000; // the longest route time limit, per attempt
const MAX_RETRIES = 3;
// FRESH=1 skips the cache: every call is fetched again and billed normally (a cache hit is free).
const FRESH = process.env.FRESH === "1";
const WAIT_LIMIT_MS = 15 * 60_000; // how long to follow one run or job
const RUNNING = new Set(["queued","running","pending"]);
// A job read answers the job's status; read it again until it is terminal.
const JOB_READ = /^(web\/jobs|cohort-queries)\/[^/]+$/;

let callCount = 0;
let creditsSpent = 0;
const runCredits = new Map<string, number>();
const createdPaths: string[] = [];

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
// Exponential backoff with jitter: about 1 s, 2 s, then 4 s.
const backoff = (attempt: number) => 2 ** attempt * (500 + Math.random() * 500);

interface CallOptions {
  method?: string;
  body?: unknown;
  idempotencyKey?: string;
}

// One request. Retries a network error and an error the API marks retryable
// (Retry-After wins), but a POST only when it carries an Idempotency-Key, so
// a retry never runs a paid write twice. Returns the parsed body, or null for
// RESOURCE_NOT_FOUND (refunded: nothing to read). Throws on any other error.
async function send(url: string, opts: CallOptions = {}): Promise<any> {
  const method = opts.method ?? "GET";
  const retries = method === "GET" || opts.idempotencyKey ? MAX_RETRIES : 0;
  for (let attempt = 0; ; attempt++) {
    let res: Response;
    let body: any;
    try {
      res = await fetch(url, {
        method,
        headers: {
          "x-api-key": API_KEY!,
          accept: "application/json",
          ...(FRESH ? { "cache-control": "no-cache" } : {}),
          ...(opts.body === undefined ? {} : { "content-type": "application/json" }),
          ...(opts.idempotencyKey ? { "idempotency-key": opts.idempotencyKey } : {}),
        },
        body: opts.body === undefined ? undefined : JSON.stringify(opts.body),
        signal: AbortSignal.timeout(TIMEOUT_MS),
      });
      body = await res.json();
    } catch (err) {
      if (attempt >= retries) throw err;
      await sleep(backoff(attempt));
      continue;
    }
    callCount++;
    if (res.ok && body?.success !== false) {
      // A run (prism/discover) reports its whole spend on data.run.credits.spent.
      const run = body?.data?.run;
      if (run?.run_id && typeof run.credits?.spent === "number") runCredits.set(run.run_id, run.credits.spent);
      else creditsSpent += body?.credits_used ?? 0;
      return body;
    }
    const error = body?.error ?? { type: `HTTP_${res.status}`, message: res.statusText };
    // Branch on error.type, never on the message text.
    if (error.type === "RESOURCE_NOT_FOUND") {
      console.warn(`Skipped, not found (refunded): ${url}`);
      return null;
    }
    const retryAfter = res.headers.get("retry-after");
    if (!error.retryable || attempt >= retries)
      throw new Error(`${error.type}: ${error.message} (request_id ${body?.request_id ?? "none"})`);
    await sleep(retryAfter ? Number(retryAfter) * 1000 : backoff(attempt));
  }
}

// What a run created and keeps (a monitor, a cohort), printed with how to delete it.
function noteCreated(path: string, method: string | undefined, body: any) {
  if (method !== "POST" || !body) return;
  const id =
    path === "monitors" ? body.monitor?.id
    : path === "cohorts" ? body.data?.id
    : path === "web/monitors" ? body.data?.monitor_id
    : undefined;
  if (!id) return;
  createdPaths.push(`${path}/${id}`);
  console.log(`Created ${path}/${id}. Delete it with DELETE /v1/${path}/${id} when you no longer need it.`);
}

// One API call by its path under /v1. Follows an async run (202) and a job to its end.
async function sc(path: string, params: Record<string, unknown> = {}, opts: CallOptions = {}): Promise<any> {
  const url = new URL(`${BASE_URL}/v1/${path}`);
  for (const [key, value] of Object.entries(params))
    if (value !== undefined && value !== null) url.searchParams.set(key, String(value));
  let body = await send(url.toString(), opts);
  const deadline = Date.now() + WAIT_LIMIT_MS;
  // An async run answers 202 and keeps going: follow data.next.status_url until it ends.
  let statusUrl = body?.data?.next?.status_url;
  while (statusUrl && RUNNING.has(body?.data?.run?.status)) {
    if (Date.now() > deadline) throw new Error(`${path} is still running: follow ${statusUrl}`);
    await sleep(1000);
    body = await send(statusUrl);
    statusUrl = body?.data?.next?.status_url ?? statusUrl;
  }
  while (JOB_READ.test(path) && RUNNING.has(body?.data?.status)) {
    if (Date.now() > deadline) throw new Error(`${path} is still ${body.data.status}: read it again later`);
    await sleep((body.data.poll_after_s ?? 5) * 1000);
    body = await send(url.toString());
  }
  noteCreated(path, opts.method, body);
  return body;
}

// Rows in a step's result: a page's items, or the answers of a step run once per row.
function rowsIn(value: any): number {
  if (Array.isArray(value)) return value.filter((v) => v != null).length;
  if (!value) return 0;
  return Array.isArray(value.data?.items) ? value.data.items.length : 1;
}

const ITEMS = Number(process.env.ITEMS ?? 1);

const reactions = await sc("prism/discover", { brands: "aruba,curacao,bonaire", period: "last_month", exclude_terms: "networks", max_credits: "150" });

const totalCredits = creditsSpent + [...runCredits.values()].reduce((a, b) => a + b, 0);
console.log(`Spent ${totalCredits} credits in ${callCount} calls.`);
const rowCounts = { reactions: rowsIn(reactions) };
console.log(
  "RECIPE_RESULT " +
    JSON.stringify({
      recipe: "monthly-brand-reaction",
      calls: callCount,
      credits_used: totalCredits,
      rows: rowCounts,
      created: createdPaths,
    }),
);

# How people reacted to a brand and its competitors last month
#
# Price for a brand and its competitors, one month: at most 150 credits (max_credits caps the whole run; each search bills at its platform's price only when it returns rows, so it settles 0-150; nothing beyond the judging bound is held up front).
# Measured on 25/10/2026: 35 credits in 5 calls (the quote held 150). The TypeScript and Python programs both ran.
#
# FRESH=1 skips the cache and bills every call; a cache hit is free.
# Inputs, set to a working example (change them to yours):
#   brands: The brand and its competitors, comma-separated
#   exclude_terms: Words that mark another meaning of a brand name, comma-separated
#
# Run it with Python 3.9 or newer, nothing to install: SOCIALCRAWL_API_KEY=sc_... python3 recipe.py

import json
import os
import random
import re
import sys
import time
import http.client
import urllib.error
import urllib.parse
import urllib.request
import uuid

API_KEY = os.environ.get("SOCIALCRAWL_API_KEY") or sys.exit("Set SOCIALCRAWL_API_KEY to your SocialCrawl API key")
BASE_URL = os.environ.get("SOCIALCRAWL_BASE_URL", "https://www.socialcrawl.dev").rstrip("/")
TIMEOUT_S = 300  # the longest route time limit, per attempt
MAX_RETRIES = 3
# FRESH=1 skips the cache: every call is fetched again and billed normally (a cache hit is free).
FRESH = os.environ.get("FRESH") == "1"
WAIT_LIMIT_S = 15 * 60  # how long to follow one run or job
RUNNING = {"queued", "running", "pending"}
# A job read answers the job's status; read it again until it is terminal.
JOB_READ = re.compile(r"^(web/jobs|cohort-queries)/[^/]+$")

call_count = 0
credits_spent = 0
run_credits = {}
created_paths = []


def dig(value, *keys):
    for key in keys:
        if not isinstance(value, dict):
            return None
        value = value.get(key)
    return value


def backoff(attempt):
    # Exponential backoff with jitter: about 1 s, 2 s, then 4 s.
    return 2 ** attempt * (0.5 + random.random() / 2)


def send(url, method="GET", body=None, idempotency_key=None):
    # One request. Retries a network error and an error the API marks retryable
    # (Retry-After wins), but a POST only when it carries an Idempotency-Key, so
    # a retry never runs a paid write twice. Returns the parsed body, or None for
    # RESOURCE_NOT_FOUND (refunded: nothing to read). Exits on any other error.
    global call_count, credits_spent
    headers = {"x-api-key": API_KEY, "accept": "application/json"}
    if FRESH:
        headers["Cache-Control"] = "no-cache"
    data = None
    if body is not None:
        data = json.dumps(body).encode()
        headers["content-type"] = "application/json"
    if idempotency_key:
        headers["Idempotency-Key"] = idempotency_key
    retries = MAX_RETRIES if method == "GET" or idempotency_key else 0
    for attempt in range(retries + 1):
        request = urllib.request.Request(url, data=data, method=method, headers=headers)
        try:
            with urllib.request.urlopen(request, timeout=TIMEOUT_S) as res:
                status, retry_after, payload = res.status, res.headers.get("Retry-After"), json.loads(res.read() or b"null")
        except urllib.error.HTTPError as err:
            status, retry_after = err.code, err.headers.get("Retry-After")
            try:
                payload = json.loads(err.read() or b"null")
            except (ValueError, OSError, http.client.HTTPException):
                payload = None
        # A network error, a timeout, a connection cut mid-body, or a body that is not JSON.
        except (OSError, http.client.HTTPException, ValueError):
            if attempt == retries:
                raise
            time.sleep(backoff(attempt))
            continue
        call_count += 1
        if 200 <= status < 300 and dig(payload, "success") is not False:
            # A run (prism/discover) reports its whole spend on data.run.credits.spent.
            run = dig(payload, "data", "run") or {}
            spent = dig(run, "credits", "spent")
            if run.get("run_id") and isinstance(spent, (int, float)):
                run_credits[run["run_id"]] = spent
            elif isinstance(payload, dict):
                credits_spent += payload.get("credits_used") or 0
            return payload
        # A gateway page that is not our JSON (a 502 or 503 in front of the API) is retryable.
        error = dig(payload, "error") or {"type": f"HTTP_{status}", "message": "no error body", "retryable": status >= 500}
        # Branch on error type, never on the message text.
        if error.get("type") == "RESOURCE_NOT_FOUND":
            print(f"Skipped, not found (refunded): {url}", file=sys.stderr)
            return None
        if not error.get("retryable") or attempt == retries:
            sys.exit(f"{error.get('type')}: {error.get('message')} (request_id {dig(payload, 'request_id')})")
        time.sleep(float(retry_after) if retry_after else backoff(attempt))


def note_created(path, method, body):
    # What a run created and keeps (a monitor, a cohort), printed with how to delete it.
    if method != "POST" or not body:
        return
    ids = {"monitors": dig(body, "monitor", "id"), "cohorts": dig(body, "data", "id"), "web/monitors": dig(body, "data", "monitor_id")}
    created = ids.get(path)
    if created:
        created_paths.append(f"{path}/{created}")
        print(f"Created {path}/{created}. Delete it with DELETE /v1/{path}/{created} when you no longer need it.")


def sc(path, params=None, method="GET", body=None, idempotency_key=None):
    # One API call by its path under /v1. Follows an async run (202) and a job to its end.
    url = f"{BASE_URL}/v1/{path}"
    query = {k: (str(v).lower() if isinstance(v, bool) else str(v)) for k, v in (params or {}).items() if v is not None}
    if query:
        url += "?" + urllib.parse.urlencode(query)
    result = send(url, method, body, idempotency_key)
    deadline = time.time() + WAIT_LIMIT_S
    # An async run answers 202 and keeps going: follow data.next.status_url until it ends.
    status_url = dig(result, "data", "next", "status_url")
    while status_url and dig(result, "data", "run", "status") in RUNNING:
        if time.time() > deadline:
            sys.exit(f"{path} is still running: follow {status_url}")
        time.sleep(1)
        result = send(status_url)
        status_url = dig(result, "data", "next", "status_url") or status_url
    while JOB_READ.match(path) and dig(result, "data", "status") in RUNNING:
        if time.time() > deadline:
            sys.exit(f"{path} is still running: read it again later")
        wait_s = dig(result, "data", "poll_after_s")
        time.sleep(5 if wait_s is None else wait_s)
        result = send(url)
    note_created(path, method, result)
    return result


def rows_in(value):
    # Rows in a step's result: a page's items, or the answers of a step run once per row.
    if isinstance(value, list):
        return len([v for v in value if v is not None])
    if not value:
        return 0
    items = dig(value, "data", "items")
    return len(items) if isinstance(items, list) else 1


ITEMS = int(os.environ.get("ITEMS", "1"))

reactions = sc("prism/discover", {"brands": "aruba,curacao,bonaire", "period": "last_month", "exclude_terms": "networks", "max_credits": "150"})

total_credits = credits_spent + sum(run_credits.values())
print(f"Spent {total_credits} credits in {call_count} calls.")
row_counts = {"reactions": rows_in(reactions)}
print(
    "RECIPE_RESULT "
    + json.dumps(
        {
            "recipe": "monthly-brand-reaction",
            "calls": call_count,
            "credits_used": total_credits,
            "rows": row_counts,
            "created": created_paths,
        }
    )
)

{
  "name": "SocialCrawl: How people reacted to a brand and its competitors last month",
  "nodes": [
    {
      "id": "node-1",
      "position": [
        260,
        0
      ],
      "name": "Run recipe",
      "type": "n8n-nodes-base.manualTrigger",
      "typeVersion": 1,
      "parameters": {}
    },
    {
      "id": "node-2",
      "position": [
        520,
        0
      ],
      "name": "reactions",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "parameters": {
        "method": "GET",
        "url": "https://www.socialcrawl.dev/v1/prism/discover?brands=aruba%2Ccuracao%2Cbonaire&period=last_month&exclude_terms=networks&max_credits=150",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth",
        "options": {
          "timeout": 300000
        }
      },
      "credentials": {
        "httpHeaderAuth": {
          "name": "SocialCrawl API key"
        }
      },
      "executeOnce": true,
      "retryOnFail": true,
      "maxTries": 4,
      "waitBetweenTries": 2000
    },
    {
      "id": "node-3",
      "position": [
        780,
        0
      ],
      "name": "reactions: finished?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2,
      "parameters": {
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "loose"
          },
          "conditions": [
            {
              "id": "reactions-finished",
              "leftValue": "={{ !['queued','running','pending'].includes($json.data.run.status) }}",
              "rightValue": true,
              "operator": {
                "type": "boolean",
                "operation": "true",
                "singleValue": true
              }
            }
          ],
          "combinator": "and"
        },
        "options": {}
      }
    },
    {
      "id": "node-4",
      "position": [
        1040,
        0
      ],
      "name": "reactions: wait",
      "type": "n8n-nodes-base.wait",
      "typeVersion": 1.1,
      "parameters": {
        "resume": "timeInterval",
        "amount": 5,
        "unit": "seconds"
      }
    },
    {
      "id": "node-5",
      "position": [
        1300,
        0
      ],
      "name": "reactions: follow the run",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "parameters": {
        "method": "GET",
        "url": "={{ $json.data.next.status_url }}",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth",
        "options": {
          "timeout": 300000
        }
      },
      "credentials": {
        "httpHeaderAuth": {
          "name": "SocialCrawl API key"
        }
      },
      "executeOnce": true,
      "retryOnFail": true,
      "maxTries": 4,
      "waitBetweenTries": 2000
    },
    {
      "id": "node-6",
      "position": [
        1560,
        0
      ],
      "name": "reactions: result",
      "type": "n8n-nodes-base.code",
      "typeVersion": 2,
      "parameters": {
        "jsCode": "// The finished answer.\nreturn $input.all();"
      }
    },
    {
      "id": "note",
      "name": "How this recipe works",
      "type": "n8n-nodes-base.stickyNote",
      "typeVersion": 1,
      "position": [
        0,
        -460
      ],
      "parameters": {
        "content": "## How people reacted to a brand and its competitors last month\n\nPrice for a brand and its competitors, one month: at most 150 credits (max_credits caps the whole run; each search bills at its platform's price only when it returns rows, so it settles 0-150; nothing beyond the judging bound is held up front).\n\nMeasured on 25/10/2026: 35 credits in 5 calls (the quote held 150). The TypeScript and Python programs both ran.\n\nFRESH=1 skips the cache and bills every call; a cache hit is free.\n\nInputs, set to a working example (change them to yours):\n\n  brands: The brand and its competitors, comma-separated\n\n  exclude_terms: Words that mark another meaning of a brand name, comma-separated\n\n**Key:** create a Header Auth credential named SocialCrawl API key, with Name x-api-key and your API key as Value, and pick it on every request node.\n\n**Rows:** a step that runs once per row takes the first 1 rows (change the number in its rows node).\n\n**Errors:** a request node stops the workflow on an API error. Read error.type in its output; retry only when error.retryable is true.",
        "width": 520,
        "height": 420
      }
    }
  ],
  "connections": {
    "Run recipe": {
      "main": [
        [
          {
            "node": "reactions",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "reactions": {
      "main": [
        [
          {
            "node": "reactions: finished?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "reactions: finished?": {
      "main": [
        [
          {
            "node": "reactions: result",
            "type": "main",
            "index": 0
          }
        ],
        [
          {
            "node": "reactions: wait",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "reactions: wait": {
      "main": [
        [
          {
            "node": "reactions: follow the run",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "reactions: follow the run": {
      "main": [
        [
          {
            "node": "reactions: finished?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  },
  "settings": {}
}