From df4ca0bf7688faae9f3b5e7002471ef020699a81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=20Jurmanovi=C4=87?= Date: Wed, 12 Aug 2026 23:15:42 +0200 Subject: [PATCH] Track cached prompt tokens through to Langfuse OpenAI's chat completions response includes prompt_tokens_details. cached_tokens when prompt caching kicks in - previously dropped entirely. Now extracted in LlmClient and forwarded to Langfuse as the flat cache_read_input_tokens usage key. Used the flat key rather than OpenAI's nested prompt_tokens_details shape: the installed @langfuse/tracing SDK's usageDetails type is Record | OpenAiUsage (a 3-field camelCase shape, no _details nesting) even though the wire schema in @langfuse/core supports nesting - a gap in the SDK's public types, not something worth fighting with a cast. Verified: full suite (82/82) passing, tsc/eslint clean. --- src/core/langfuse/LangfuseTracer.ts | 9 +++++++++ src/core/llm/LlmClient.ts | 8 +++++++- src/core/llm/Tracer.ts | 7 ++++++- test/core/langfuse/LangfuseTracer.test.ts | 21 +++++++++++++++++++++ test/core/llm/LlmClient.test.ts | 22 ++++++++++++++++++++++ 5 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/core/langfuse/LangfuseTracer.ts b/src/core/langfuse/LangfuseTracer.ts index 881be79..5f8510f 100644 --- a/src/core/langfuse/LangfuseTracer.ts +++ b/src/core/langfuse/LangfuseTracer.ts @@ -16,6 +16,14 @@ import type { * OpenAI's native snake_case schema (prompt_tokens/completion_tokens/...); * our own ChatResult.usage uses camelCase for our internal contract, so it * needs translating at this boundary or Langfuse silently shows zero usage. + * + * Cached tokens use the flat schema's cache_read_input_tokens key rather + * than OpenAI's nested prompt_tokens_details.cached_tokens: the installed + * @langfuse/tracing SDK's usageDetails type is `Record | + * OpenAiUsage` (a 3-field camelCase shape with no _details nesting) even + * though the wire schema (@langfuse/core's OpenAiCompletionUsageSchema) + * supports the nested form — keeping everything as flat numbers avoids + * fighting that gap in the SDK's public types. */ function toUsageDetails(usage: GenerationOutput['usage']): Record | undefined { if (!usage) return undefined; @@ -23,6 +31,7 @@ function toUsageDetails(usage: GenerationOutput['usage']): Record 0 ? details : undefined; } diff --git a/src/core/llm/LlmClient.ts b/src/core/llm/LlmClient.ts index 03616da..f346a6d 100644 --- a/src/core/llm/LlmClient.ts +++ b/src/core/llm/LlmClient.ts @@ -13,7 +13,12 @@ export interface ChatOptions { export interface ChatResult { content: string; - usage?: { promptTokens?: number; completionTokens?: number; totalTokens?: number }; + usage?: { + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + cachedTokens?: number; + }; } export class LlmClient { @@ -43,6 +48,7 @@ export class LlmClient { promptTokens: response.usage.prompt_tokens, completionTokens: response.usage.completion_tokens, totalTokens: response.usage.total_tokens, + cachedTokens: response.usage.prompt_tokens_details?.cached_tokens, } : undefined, }; diff --git a/src/core/llm/Tracer.ts b/src/core/llm/Tracer.ts index 31868a4..1ebae57 100644 --- a/src/core/llm/Tracer.ts +++ b/src/core/llm/Tracer.ts @@ -7,7 +7,12 @@ export interface GenerationInput { export interface GenerationOutput { output: unknown; - usage?: { promptTokens?: number; completionTokens?: number; totalTokens?: number }; + usage?: { + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; + cachedTokens?: number; + }; metadata?: Record; } diff --git a/test/core/langfuse/LangfuseTracer.test.ts b/test/core/langfuse/LangfuseTracer.test.ts index 0396941..78024f6 100644 --- a/test/core/langfuse/LangfuseTracer.test.ts +++ b/test/core/langfuse/LangfuseTracer.test.ts @@ -78,6 +78,27 @@ describe('LangfuseTracer', () => { expect(generationEnd).toHaveBeenCalledTimes(1); }); + it('maps cachedTokens to the flat cache_read_input_tokens key', () => { + const tracer = new LangfuseTracer({ publicKey: 'pk', secretKey: 'sk', baseUrl: 'http://lf' }); + const handle = tracer.startTrace('my-pipeline'); + const gen = handle.startGeneration({ name: 'step-1', model: 'gpt-4o-mini', input: ['hi'] }); + gen.end({ + output: 'done', + usage: { promptTokens: 100, completionTokens: 20, totalTokens: 120, cachedTokens: 80 }, + }); + + expect(generationUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + usageDetails: { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + cache_read_input_tokens: 80, + }, + }) + ); + }); + it('omits usageDetails entirely when no usage is reported', () => { const tracer = new LangfuseTracer({ publicKey: 'pk', secretKey: 'sk', baseUrl: 'http://lf' }); const handle = tracer.startTrace('my-pipeline'); diff --git a/test/core/llm/LlmClient.test.ts b/test/core/llm/LlmClient.test.ts index 6c73017..7854cd4 100644 --- a/test/core/llm/LlmClient.test.ts +++ b/test/core/llm/LlmClient.test.ts @@ -32,6 +32,28 @@ describe('LlmClient', () => { }); }); + it('extracts cachedTokens from prompt_tokens_details when present', async () => { + createMock.mockResolvedValue({ + choices: [{ message: { content: 'hello' } }], + usage: { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + prompt_tokens_details: { cached_tokens: 80 }, + }, + }); + + const client = new LlmClient({ baseUrl: 'http://localhost:11434/v1' }); + const result = await client.chat([{ role: 'user', content: 'hi' }], { model: 'gpt-4o-mini' }); + + expect(result.usage).toEqual({ + promptTokens: 100, + completionTokens: 20, + totalTokens: 120, + cachedTokens: 80, + }); + }); + it('throws when the response has no choices', async () => { createMock.mockResolvedValue({ choices: [] });