Skip to content

fix: stream-parse session JSONL to prevent OOM on large sessions #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sidbmw wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: stream-parse session JSONL to prevent OOM on large sessions #160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions src/claude/utils/claudeCheckSession.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
import { logger } from "@/ui/logger";
import { existsSync, readFileSync } from "node:fs";
import { existsSync, openSync, readSync, closeSync } from "node:fs";
import { join } from "node:path";
import { getProjectPath } from "./path";

const CHECK_SESSION_BYTES = 16384; // 16KB is enough to find one valid message

export function claudeCheckSession(sessionId: string, path: string) {
const projectDir = getProjectPath(path);

Expand All @@ -14,10 +16,17 @@ export function claudeCheckSession(sessionId: string, path: string) {
return false;
}

// Check if session contains any messages with valid ID fields
const sessionData = readFileSync(sessionFile, 'utf-8').split('\n');
// Only read the first 16KB to check for a valid message.
// Session files can be hundreds of MB for long conversations;
// reading the entire file causes OOM crashes (see #526).
const fd = openSync(sessionFile, 'r');
const buf = Buffer.alloc(CHECK_SESSION_BYTES);
const bytesRead = readSync(fd, buf, 0, CHECK_SESSION_BYTES, 0);
closeSync(fd);
const chunk = buf.toString('utf-8', 0, bytesRead);
const lines = chunk.split('\n');

const hasGoodMessage = !!sessionData.find((v, index) => {
const hasGoodMessage = !!lines.find((v, index) => {
if (!v.trim()) return false; // Skip empty lines silently (not errors)

try {
Expand Down
84 changes: 53 additions & 31 deletions src/claude/utils/sessionScanner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import { InvalidateSync } from "@/utils/sync";
import { RawJSONLines, RawJSONLinesSchema } from "../types";
import { join } from "node:path";
import { readFile } from "node:fs/promises";
import { createReadStream } from "node:fs";
import { createInterface } from "node:readline";
import { logger } from "@/ui/logger";
import { startFileWatcher } from "@/modules/watcher/startFileWatcher";
import { getProjectPath } from "./path";
Expand Down Expand Up @@ -164,45 +165,66 @@ function messageKey(message: RawJSONLines): string {
}

/**
* Read and parse session log file
* Returns only valid conversation messages, silently skipping internal events
* Maximum number of messages to keep from a session file.
* For very long sessions (100k+ lines), keeping all parsed messages
* in memory causes OOM crashes. We only need recent messages for
* the mobile sync use case.
*/
const MAX_SESSION_MESSAGES = 500;

/**
* Read and parse session log file using streaming to avoid OOM.
*
* Long-running Claude Code sessions can produce JSONL files of 500MB+
* (especially after context compaction). Loading these entirely into
* memory with readFile() causes V8 heap exhaustion (see #526).
*
* This implementation:
* 1. Stream-parses line-by-line via createReadStream + readline
* 2. Caps retained messages at MAX_SESSION_MESSAGES (most recent)
* 3. Never holds more than one line in memory at a time
*/
async function readSessionLog(projectDir: string, sessionId: string): Promise<RawJSONLines[]> {
const expectedSessionFile = join(projectDir, `${sessionId}.jsonl`);
logger.debug(`[SESSION_SCANNER] Reading session file: ${expectedSessionFile}`);
let file: string;
let messages: RawJSONLines[] = [];
try {
file = await readFile(expectedSessionFile, 'utf-8');
const rl = createInterface({
input: createReadStream(expectedSessionFile, { encoding: 'utf-8' }),
crlfDelay: Infinity,
});
for await (const l of rl) {
try {
if (l.trim() === '') {
continue;
}
let message = JSON.parse(l);

// Silently skip known internal Claude Code events
// These are state/tracking events, not conversation messages
if (message.type && INTERNAL_CLAUDE_EVENT_TYPES.has(message.type)) {
continue;
}

let parsed = RawJSONLinesSchema.safeParse(message);
if (!parsed.success) {
// Unknown message types are silently skipped
// They will be tracked by processedMessageKeys to avoid reprocessing
continue;
}
messages.push(parsed.data);
} catch (e) {
logger.debug(`[SESSION_SCANNER] Error processing message: ${e}`);
continue;
}
}
} catch (error) {
logger.debug(`[SESSION_SCANNER] Session file not found: ${expectedSessionFile}`);
return [];
}
let lines = file.split('\n');
let messages: RawJSONLines[] = [];
for (let l of lines) {
try {
if (l.trim() === '') {
continue;
}
let message = JSON.parse(l);

// Silently skip known internal Claude Code events
// These are state/tracking events, not conversation messages
if (message.type && INTERNAL_CLAUDE_EVENT_TYPES.has(message.type)) {
continue;
}

let parsed = RawJSONLinesSchema.safeParse(message);
if (!parsed.success) {
// Unknown message types are silently skipped
// They will be tracked by processedMessageKeys to avoid reprocessing
continue;
}
messages.push(parsed.data);
} catch (e) {
logger.debug(`[SESSION_SCANNER] Error processing message: ${e}`);
continue;
}
// Keep only the most recent messages to bound memory usage
if (messages.length > MAX_SESSION_MESSAGES) {
messages = messages.slice(-MAX_SESSION_MESSAGES);
}
return messages;
}