Social listening pulse-check

// recipe-pulse-check.ts
// Streams universal search results for a single topic.
// Run with: SOCIALCRAWL_KEY=sc_... npx tsx recipe-pulse-check.ts

const KEY = process.env.SOCIALCRAWL_KEY;
if (!KEY) throw new Error("Set SOCIALCRAWL_KEY");

const url = new URL("https://www.socialcrawl.dev/v1/search/everywhere");
url.searchParams.set("query", "anthropic claude 4");
url.searchParams.set("lookback_days", "7");

const res = await fetch(url, {
  headers: {
    "x-api-key": KEY,
    accept: "text/event-stream",
  },
});

if (!res.ok || !res.body) {
  throw new Error(`Search failed: ${res.status} ${await res.text()}`);
}

const decoder = new TextDecoder();
let buffer = "";

for await (const chunk of res.body) {
  buffer += decoder.decode(chunk as Uint8Array, { stream: true });
  const events = buffer.split("\n\n");
  buffer = events.pop() ?? "";

  for (const event of events) {
    const dataLine = event
      .split("\n")
      .find((line) => line.startsWith("data: "));
    if (!dataLine) continue;
    const payload = JSON.parse(dataLine.slice(6));

    switch (payload.type) {
      case "plan_refined":
        console.log("plan ready, sub-queries:", payload.plan.subqueries.length);
        break;
      case "clusters":
        console.log(`clustered into ${payload.clusters.length} themes`);
        break;
      case "comments_enriched":
        for (const item of payload.items) {
          const top = item.source_items[0]?.metadata?.top_comments?.[0];
          console.log(`[${item.source}] ${item.title}`);
          if (top) console.log(`  → "${top.text.slice(0, 120)}…"`);
        }
        break;
      case "done":
        console.log(
          `done: ${payload.summary.total_items} results from ${payload.summary.sources_called.length} sources, charged ${payload.summary.credits_used}cr`,
        );
        break;
      case "error":
        console.error("search error:", payload.message);
        break;
    }
  }
}

// One `comments_enriched` SSE chunk, many of these arrive over the stream
{
  "type": "comments_enriched", // <-- discriminator
  "items": [
    {
      "candidate_id": "rd_18gha22",
      "source": "reddit", // <-- one of the 14 platforms
      "title": "Claude 4 is shockingly good at refactors",
      "url": "https://reddit.com/r/ClaudeAI/comments/...",
      "snippet": "I gave it a 4k-line Rust file and...",
      "final_score": 0.91, // <-- RRF + rerank fused
      "cluster_id": "cluster_3", // <-- groups related posts
      "source_items": [
        {
          "metadata": {
            "top_comments": [
              // <-- real-people sentiment
              {
                "text": "Same. Just shipped a migration in 20 min.",
                "score": 142,
              },
            ],
          },
        },
      ],
    },
  ],
}