LangChain + carbon-llm — tracker chaque LLM call via un BaseCallbackHandler

.env.local

CARBON_LLM_API_KEY=test_xxxxxxxxxxxxxxxxxxxx
CARBON_LLM_BASE_URL=https://carbon-llm.com
CARBON_LLM_DEFAULT_TENANT_ID=default-tenant

lib/carbon-llm-handler.ts (langchain.js)

import { BaseCallbackHandler } from "@langchain/core/callbacks/base"
import type { LLMResult } from "@langchain/core/outputs"

const BASE_URL = process.env.CARBON_LLM_BASE_URL ?? "https://carbon-llm.com"
const API_KEY = process.env.CARBON_LLM_API_KEY
const DEFAULT_TENANT = process.env.CARBON_LLM_DEFAULT_TENANT_ID ?? "default"

export class CarbonLlmCallbackHandler extends BaseCallbackHandler {
  name = "carbon_llm_callback_handler"

  constructor(private opts: { tenantId?: string } = {}) {
    super()
  }

  async handleLLMEnd(output: LLMResult): Promise<void> {
    if (!API_KEY) return // dev local sans clé : no-op silencieux

    const usage = output.llmOutput?.tokenUsage as
      | { promptTokens?: number; completionTokens?: number }
      | undefined

    // Le model name est dans llmOutput selon les providers
    const model =
      (output.llmOutput?.modelName as string | undefined) ??
      (output.generations?.[0]?.[0]?.generationInfo?.model_name as string | undefined) ??
      "unknown"

    if (!usage) return

    // Fire-and-forget — pas d'await, pas de blocage chain
    fetch(`${BASE_URL}/api/v1/track`, {
      method: "POST",
      headers: {
        Authorization: `Bearer ${API_KEY}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify({
        model,
        prompt_tokens: usage.promptTokens ?? 0,
        completion_tokens: usage.completionTokens ?? 0,
        tenant_id: this.opts.tenantId ?? DEFAULT_TENANT,
        ingestion_source: "langchain_js",
      }),
    }).catch((e) => console.error("[carbon-llm] track failed:", e))
  }
}

Exemple langchain.js

import { ChatOpenAI } from "@langchain/openai"
import { CarbonLlmCallbackHandler } from "@/lib/carbon-llm-handler"

// Option 1 — par chain, avec tenant_id du customer courant
const result = await new ChatOpenAI({ model: "gpt-4o" }).invoke(
  "Hello world",
  { callbacks: [new CarbonLlmCallbackHandler({ tenantId: user.organizationId })] },
)

// Option 2 — global (toutes les chains de la requête)
import { CallbackManager } from "@langchain/core/callbacks/manager"
const cm = CallbackManager.fromHandlers({ handleLLMEnd: () => null })
cm.addHandler(new CarbonLlmCallbackHandler({ tenantId: user.organizationId }))

// Option 3 — LCEL : les Runnables héritent
const chain = prompt.pipe(new ChatOpenAI({ model: "gpt-4o" })).pipe(parser)
const out = await chain.invoke({ q: "..." }, {
  callbacks: [new CarbonLlmCallbackHandler({ tenantId: user.organizationId })],
})

carbon_llm_handler.py

import os
import httpx
from typing import Any, Dict, List, Optional
from langchain_core.callbacks import BaseCallbackHandler
from langchain_core.outputs import LLMResult

BASE_URL = os.environ.get("CARBON_LLM_BASE_URL", "https://carbon-llm.com")
API_KEY = os.environ.get("CARBON_LLM_API_KEY")
DEFAULT_TENANT = os.environ.get("CARBON_LLM_DEFAULT_TENANT_ID", "default")

# Pool client httpx réutilisé — évite de recréer une connexion par event
_client: Optional[httpx.AsyncClient] = None

def _get_client() -> httpx.AsyncClient:
    global _client
    if _client is None:
        _client = httpx.AsyncClient(timeout=5.0)
    return _client


class CarbonLlmHandler(BaseCallbackHandler):
    """Track LangChain LLM calls to carbon-llm. Async, fire-and-forget."""

    def __init__(self, tenant_id: Optional[str] = None) -> None:
        self.tenant_id = tenant_id or DEFAULT_TENANT

    async def on_llm_end(
        self,
        response: LLMResult,
        **kwargs: Any,
    ) -> None:
        if not API_KEY:
            return

        token_usage = (response.llm_output or {}).get("token_usage", {})
        model = (response.llm_output or {}).get("model_name", "unknown")
        prompt_tokens = token_usage.get("prompt_tokens", 0)
        completion_tokens = token_usage.get("completion_tokens", 0)

        if prompt_tokens == 0 and completion_tokens == 0:
            return

        # Fire-and-forget — log on error mais ne pas bloquer la chain
        try:
            await _get_client().post(
                f"{BASE_URL}/api/v1/track",
                headers={"Authorization": f"Bearer {API_KEY}"},
                json={
                    "model": model,
                    "prompt_tokens": prompt_tokens,
                    "completion_tokens": completion_tokens,
                    "tenant_id": self.tenant_id,
                    "ingestion_source": "langchain_python",
                },
            )
        except Exception as e:
            print(f"[carbon-llm] track failed: {e}")

Exemple LangChain Python

from langchain_openai import ChatOpenAI
from carbon_llm_handler import CarbonLlmHandler

# Par chain
handler = CarbonLlmHandler(tenant_id=user.organization_id)
llm = ChatOpenAI(model="gpt-4o")
result = await llm.ainvoke("Hello", config={"callbacks": [handler]})

# LCEL — les Runnables héritent
chain = prompt | ChatOpenAI(model="gpt-4o") | parser
out = await chain.ainvoke({"q": "..."}, config={"callbacks": [handler]})