Production Checklist

const BASE_URL = "https://www.socialcrawl.dev";
const BASE_BACKOFF_MS = 500;
const MAX_BACKOFF_MS = 8_000;

const TRANSIENT_ERROR_TYPES = new Set([
  "IDEMPOTENCY_IN_PROGRESS",
  "RATE_LIMITED",
  "CONCURRENCY_LIMIT",
  "UPSTREAM_ERROR",
  "SERVICE_UNAVAILABLE",
  "INTERNAL_ERROR",
]);
const TRANSIENT_HTTP_STATUSES = new Set([429, 500, 502, 504]);

interface SocialCrawlEnvelope<T> {
  success: boolean;
  data?: T;
  error?: {
    type: string;
    message: string;
    status: number;
  };
  request_id: string;
  credits_used: number;
  credits_remaining: number | null;
  cached?: boolean;
  idempotent_replay?: true;
}

interface RequestOptions {
  endpoint: `/v1/${string}`;
  query?: Record<string, string>;
  attemptTimeoutMs: number;
  maxRetries?: number;
  onAttempt?: (event: AttemptLog) => void;
}

interface AttemptDetails {
  status: number | null;
  requestId: string | null;
  errorType: string | null;
  creditsUsed: number | null;
  cacheState: string | null;
}

interface AttemptLog extends AttemptDetails {
  endpoint: string;
  attempt: number;
  latencyMs: number;
}

interface RequestFailure {
  endpoint: string;
  status: number | null;
  errorType: string;
  requestId: string | null;
  creditsUsed: number | null;
  cacheState: string | null;
  attemptCount: number;
}

class SocialCrawlRequestError extends Error {
  readonly details: RequestFailure;

  constructor(message: string, details: RequestFailure) {
    super(message);
    this.name = "SocialCrawlRequestError";
    this.details = details;
  }
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null;
}

function isEnvelope(value: unknown): value is SocialCrawlEnvelope<unknown> {
  return (
    isRecord(value) &&
    typeof value.success === "boolean" &&
    typeof value.request_id === "string" &&
    typeof value.credits_used === "number"
  );
}

function parseRetryAfter(value: string | null): number | undefined {
  if (value === null) return undefined;
  const seconds = Number(value);
  return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1_000 : undefined;
}

function isRetryable(
  errorType: string,
  retryAfterMs: number | undefined,
): boolean {
  if (!TRANSIENT_ERROR_TYPES.has(errorType)) return false;

  // A withdrawn endpoint returns this type without Retry-After and is permanent.
  if (errorType === "SERVICE_UNAVAILABLE" && retryAfterMs === undefined) {
    return false;
  }

  return true;
}

function isRetryableHttpFailure(
  status: number,
  retryAfterMs: number | undefined,
): boolean {
  return (
    TRANSIENT_HTTP_STATUSES.has(status) ||
    (status === 503 && retryAfterMs !== undefined)
  );
}

const sleep = (milliseconds: number) =>
  new Promise<void>((resolve) => setTimeout(resolve, milliseconds));

function backoffMs(attempt: number, retryAfterMs = 0): number {
  const exponentialCap = Math.min(
    MAX_BACKOFF_MS,
    BASE_BACKOFF_MS * 2 ** attempt,
  );
  return Math.max(retryAfterMs, Math.random() * exponentialCap);
}

async function requestSocialCrawl<T>({
  endpoint,
  query = {},
  attemptTimeoutMs,
  maxRetries = 3,
  onAttempt,
}: RequestOptions): Promise<SocialCrawlEnvelope<T>> {
  const apiKey = process.env.SOCIALCRAWL_API_KEY;
  if (!apiKey) throw new Error("SOCIALCRAWL_API_KEY is not configured");

  const url = new URL(endpoint, BASE_URL);
  for (const [name, value] of Object.entries(query)) {
    url.searchParams.set(name, value);
  }

  // Generate once per logical request. Every retry below reuses this value.
  const idempotencyKey = crypto.randomUUID();

  for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
    const startedAt = performance.now();
    const emitAttempt = (details: AttemptDetails) => {
      try {
        onAttempt?.({
          endpoint,
          attempt: attempt + 1,
          latencyMs: Math.round(performance.now() - startedAt),
          ...details,
        });
      } catch {
        // Observability must not change the request outcome.
      }
    };

    const attemptSignal = AbortSignal.timeout(attemptTimeoutMs);
    let response: Response;
    try {
      response = await fetch(url, {
        headers: {
          Accept: "application/json",
          "x-api-key": apiKey,
          "Idempotency-Key": idempotencyKey,
        },
        signal: attemptSignal,
      });
    } catch (cause) {
      const isTimeout =
        attemptSignal.aborted ||
        (cause instanceof DOMException && cause.name === "TimeoutError");
      const details: AttemptDetails = {
        status: null,
        errorType: isTimeout ? "CLIENT_TIMEOUT" : "NETWORK_ERROR",
        requestId: null,
        creditsUsed: null,
        cacheState: null,
      };
      emitAttempt(details);

      if (attempt < maxRetries) {
        await sleep(backoffMs(attempt));
        continue;
      }

      throw new SocialCrawlRequestError(
        isTimeout ? "SocialCrawl request timed out" : "Network request failed",
        {
          endpoint,
          ...details,
          errorType: details.errorType ?? "NETWORK_ERROR",
          attemptCount: attempt + 1,
        },
      );
    }

    const headerRequestId = response.headers.get("x-request-id");
    const retryAfterMs = parseRetryAfter(response.headers.get("retry-after"));
    let body: unknown;
    try {
      body = await response.json();
    } catch {
      const isTimeout = attemptSignal.aborted;
      const details: AttemptDetails = {
        status: response.status,
        errorType: isTimeout ? "CLIENT_TIMEOUT" : "INVALID_RESPONSE",
        requestId: headerRequestId,
        creditsUsed: null,
        cacheState: response.headers.get("x-cache"),
      };
      emitAttempt(details);

      if (
        attempt < maxRetries &&
        (isTimeout || isRetryableHttpFailure(response.status, retryAfterMs))
      ) {
        await sleep(backoffMs(attempt, retryAfterMs));
        continue;
      }

      throw new SocialCrawlRequestError(
        isTimeout
          ? "SocialCrawl response timed out"
          : "Response was not valid JSON",
        {
          endpoint,
          ...details,
          errorType: details.errorType ?? "INVALID_RESPONSE",
          attemptCount: attempt + 1,
        },
      );
    }

    if (!isEnvelope(body)) {
      const partialBody = isRecord(body) ? body : {};
      const partialError = isRecord(partialBody.error) ? partialBody.error : {};
      const details: AttemptDetails = {
        status: response.status,
        errorType:
          typeof partialError.type === "string"
            ? partialError.type
            : "INVALID_RESPONSE",
        requestId:
          typeof partialBody.request_id === "string"
            ? partialBody.request_id
            : headerRequestId,
        creditsUsed:
          typeof partialBody.credits_used === "number"
            ? partialBody.credits_used
            : null,
        cacheState: response.headers.get("x-cache"),
      };
      emitAttempt(details);

      if (
        attempt < maxRetries &&
        isRetryableHttpFailure(response.status, retryAfterMs)
      ) {
        await sleep(backoffMs(attempt, retryAfterMs));
        continue;
      }

      throw new SocialCrawlRequestError("Response envelope was invalid", {
        endpoint,
        ...details,
        errorType: details.errorType ?? "INVALID_RESPONSE",
        attemptCount: attempt + 1,
      });
    }

    const details: AttemptDetails = {
      status: response.status,
      errorType: body.error?.type ?? null,
      requestId: body.request_id,
      creditsUsed: body.credits_used,
      cacheState: response.headers.get("x-cache"),
    };
    emitAttempt(details);

    if (response.ok && body.success) {
      return body as SocialCrawlEnvelope<T>;
    }

    const errorType = body.error?.type ?? "UNKNOWN_ERROR";
    const canRetry = isRetryable(errorType, retryAfterMs);

    if (!canRetry || attempt === maxRetries) {
      throw new SocialCrawlRequestError(body.error?.message ?? errorType, {
        endpoint,
        status: response.status,
        errorType,
        requestId: body.request_id,
        creditsUsed: body.credits_used,
        cacheState: response.headers.get("x-cache"),
        attemptCount: attempt + 1,
      });
    }

    await sleep(backoffMs(attempt, retryAfterMs));
  }

  throw new Error("Unreachable retry state");
}

const profile = await requestSocialCrawl<unknown>({
  endpoint: "/v1/tiktok/profile",
  query: { handle: "charlidamelio" },
  attemptTimeoutMs: 20_000,
  // The callback receives only this allowlisted metadata, never secrets or params.
  onAttempt: (event) => console.info("socialcrawl_attempt", event),
});

console.log({
  requestId: profile.request_id,
  creditsUsed: profile.credits_used,
  cached: profile.cached,
});