draft

Author: Andrei Bratu

index.ts

@@ -36,7 +36,7 @@ export interface FlowLogResponse {
     source?: string;
     /** Any additional metadata to record. */
     metadata?: Record<string, unknown>;
-    /** Status of the Trace. When a Trace is marked as `complete`, no more Logs can be added to it. Monitoring Evaluators will only run on completed Traces. */
+    /** Status of a Log. Set to `incomplete` if you intend to update the Log and want the File's monitoring Evaluators to wait until you mark it as `complete`. If not provided, observability will pick up the Log as soon as possible; switching from specified to unspecified is undefined behavior. */
     logStatus?: Humanloop.LogStatus;
     /** Unique identifier for the Datapoint that this Log is derived from. This can be used by Humanloop to associate Logs to Evaluations. If provided, Humanloop will automatically associate this Log to Evaluations that require a Log for this Datapoint-Version pair. */
     sourceDatapointId?: string;

src/api/types/FlowLogResponse.ts

@@ -1,97 +1,56 @@
-import hash from "stable-hash";
+import * as contextApi from "@opentelemetry/api";
+import {
+    HUMANLOOP_CONTEXT_EVALUATION,
+    HUMANLOOP_CONTEXT_PROMPT,
+    HUMANLOOP_CONTEXT_TRACE_ID,
+} from "otel/constants";
+
+export function getTraceId(): string | undefined {
+    const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_TRACE_ID);
+    const value = contextApi.context.active().getValue(key);
+    return (value || undefined) as string | undefined;
+}
 
-import { FlowLogRequest, PromptLogRequest } from "../api";
-import { DatapointResponse } from "../api";
-import { Humanloop } from "../index";
-import { Version } from "./types";
+export function setTraceId(flowLogId: string): contextApi.Context {
+    const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_TRACE_ID);
+    return contextApi.context.active().setValue(key, flowLogId);
+}
 
-type EvaluationContextState = {
-    fileId?: string;
-    path?: string;
-    uploadCallback: (logId: string, datapoint: DatapointResponse) => void;
-    evaluatedVersion?: Version;
+export type PromptContext = {
+    path: string;
+    template?: string;
 };
 
-type EvaluationContextKey = {
-    inputs: Record<string, unknown> | undefined;
-    messages: Humanloop.ChatMessage[] | undefined;
-};
+export function setPromptContext(promptContext: PromptContext): contextApi.Context {
+    const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_PROMPT);
+    return contextApi.context.active().setValue(key, promptContext);
+}
 
-type EvaluationContextValue = {
-    runId: string;
+export function getPromptContext(): PromptContext | undefined {
+    const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_PROMPT);
+    return (contextApi.context.active().getValue(key) || undefined) as
+        | PromptContext
+        | undefined;
+}
+
+export type EvaluationContext = {
     sourceDatapointId: string;
-    uploadCallback: (logId: string) => void;
+    runId: string;
+    callback: (log_id: string) => void;
+    fileId: string;
+    path: string;
 };
 
-class EvaluationContext {
-    private state?: EvaluationContextState;
-    private static instance: EvaluationContext;
-    private inputMappings: Map<string, EvaluationContextValue[]> = new Map();
-
-    private constructor() {}
-
-    public static getInstance(): EvaluationContext {
-        if (!EvaluationContext.instance) {
-            EvaluationContext.instance = new EvaluationContext();
-        }
-        return EvaluationContext.instance;
-    }
-
-    public setState(state: EvaluationContextState): void {
-        this.state = state;
-    }
-
-    public getState(): Omit<EvaluationContextState, "uploadCallback"> | undefined {
-        return this.state === undefined
-            ? this.state
-            : {
-                  fileId: this.state.fileId,
-                  path: this.state.path,
-                  evaluatedVersion: this.state.evaluatedVersion,
-              };
-    }
-
-    public addDatapoint(datapoint: DatapointResponse, runId: string): void {
-        if (this.state === undefined) {
-            throw new Error("EvaluationContext state is not set");
-        }
-        const key = hash({ inputs: datapoint.inputs, messages: datapoint.messages });
-
-        if (!this.inputMappings.has(key)) {
-            this.inputMappings.set(key, []);
-        }
-        this.inputMappings.get(key)!.push({
-            runId,
-            sourceDatapointId: datapoint.id,
-            uploadCallback: (logId: string) =>
-                this.state!.uploadCallback(logId, datapoint),
-        });
-    }
-
-    public getDatapoint(key: EvaluationContextKey): EvaluationContextValue {
-        if (key.inputs !== undefined && "inputs" in key.inputs) {
-            key = { ...key, inputs: key.inputs.inputs as Record<string, unknown> };
-        }
-        const mappings = this.inputMappings.get(hash(key));
-        if (!mappings || mappings.length === 0) {
-            throw new Error(
-                `No input mappings found for: ${JSON.stringify(key)}. Try using peekDatapoint() first.`,
-            );
-        }
-        return mappings.pop()!;
-    }
-
-    public peekDatapoint(key: EvaluationContextKey): boolean {
-        const mappings = this.inputMappings.get(hash(key));
-        return mappings !== undefined && mappings.length > 0;
-    }
-
-    public isEvaluatedFile(args: FlowLogRequest | PromptLogRequest) {
-        return (
-            this.state &&
-            (this.state.fileId === args.id || this.state.path === args.path)
-        );
-    }
+export function setEvaluationContext(
+    evaluationContext: EvaluationContext,
+): contextApi.Context {
+    const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_EVALUATION);
+    return contextApi.context.active().setValue(key, evaluationContext);
 }
 
-export const evaluationContext = EvaluationContext.getInstance();
+export function getEvaluationContext(): EvaluationContext | undefined {
+    const key = contextApi.createContextKey(HUMANLOOP_CONTEXT_EVALUATION);
+    return (contextApi.context.active().getValue(key) || undefined) as
+        | EvaluationContext
+        | undefined;
+}