Hybrid search-then-enrich

// recipe-deep-research.ts
// Finds top posts on a topic, fetches their transcripts, returns the corpus.
// Run with: SOCIALCRAWL_KEY=sc_... npx tsx recipe-deep-research.ts

const KEY = process.env.SOCIALCRAWL_KEY;
if (!KEY) throw new Error("Set SOCIALCRAWL_KEY");

const BASE = "https://www.socialcrawl.dev/v1";
const topic = "ozempic side effects";
const POSTS_PER_SOURCE = 5;

// ── Step 1: Universal search, sync mode, scoped to TikTok + Instagram ─────
async function search(query: string) {
  const url = new URL(`${BASE}/search/everywhere`);
  url.searchParams.set("query", query);
  url.searchParams.set("sources", "tiktok,instagram");
  url.searchParams.set("lookback_days", "30");

  const res = await fetch(url, {
    headers: { "x-api-key": KEY!, accept: "application/json" },
  });
  const json = (await res.json()) as {
    success: boolean;
    data: { items: Array<{ source: string; url: string; title: string }> };
    credits_used: number;
  };
  if (!json.success) throw new Error("search failed");
  return json.data.items;
}

// ── Step 2: Per-platform transcript fan-out ───────────────────────────────
type Platform = "tiktok" | "instagram";
const TRANSCRIPT_PATH: Record<Platform, string> = {
  tiktok: "tiktok/post/transcript",
  instagram: "instagram/media/transcript",
};

// See the video transcription recipe, transcript shapes differ per platform.
// This adapter covers the two used here; extend for the others as needed.
function extractText(
  platform: Platform,
  data: Record<string, unknown>,
): string {
  if (platform === "tiktok") return (data.transcript as string) ?? "";
  if (platform === "instagram") {
    const t = data.transcripts as Array<{ text?: string }> | undefined;
    return t?.[0]?.text ?? "";
  }
  return (data.transcript_only_text as string) ?? "";
}

async function getTranscript(platform: Platform, postUrl: string) {
  const url = new URL(`${BASE}/${TRANSCRIPT_PATH[platform]}`);
  url.searchParams.set("url", postUrl);
  const res = await fetch(url, { headers: { "x-api-key": KEY! } });
  const json = (await res.json()) as {
    success: boolean;
    data?: Record<string, unknown>;
    error?: { message: string };
  };
  if (!json.success) {
    console.warn(`skipped ${postUrl}: ${json.error?.message}`);
    return null;
  }
  return { text: extractText(platform, json.data!) };
}

// ── Step 3: Pipeline ──────────────────────────────────────────────────────
const items = await search(topic);

const tiktoks = items
  .filter((i) => i.source === "tiktok")
  .slice(0, POSTS_PER_SOURCE);
const instagrams = items
  .filter((i) => i.source === "instagram")
  .slice(0, POSTS_PER_SOURCE);

const transcripts = await Promise.all([
  ...tiktoks.map(async (item) => ({
    title: item.title,
    url: item.url,
    transcript: await getTranscript("tiktok", item.url),
  })),
  ...instagrams.map(async (item) => ({
    title: item.title,
    url: item.url,
    transcript: await getTranscript("instagram", item.url),
  })),
]);

const corpus = transcripts
  .filter((t) => t.transcript !== null && t.transcript.text.length > 0)
  .map((t) => `# ${t.title}\n${t.url}\n\n${t.transcript!.text}`)
  .join("\n\n---\n\n");

console.log(`corpus length: ${corpus.length} chars`);
console.log(corpus.slice(0, 500), "…");

// The final `corpus` string after Step 3, markdown-formatted, ready for an LLM:
// # I tried Ozempic for 90 days, here's what happened
// https://www.tiktok.com/@drmike/video/7314...
//
// So I want to talk about what nobody tells you about Ozempic...
//
// ---
//
// # The Ozempic side effect doctors won't mention
// https://www.instagram.com/reel/CzA1234abcd/
//
// Three weeks in I started experiencing what they call...
//
// ---
//
// (8 more transcripts concatenated)