def commit_checkpoint(repo, session, message: str) -> str:
    """Snapshot the working tree as a git commit. The commit SHA is the checkpoint id."""
    repo.index.add("*")
    sha = repo.index.commit(
        message,
        author=AgentAuthor(session.id),
    ).hexsha
    session.record_checkpoint(sha, kind="git", step=session.current_step)
    return sha

def rollback(repo, session, to_sha: str) -> None:
    repo.git.reset("--hard", to_sha)   # destructive — loses uncommitted work
    session.truncate_after(to_sha)

from langgraph.checkpoint.postgres import PostgresSaver

checkpointer = PostgresSaver.from_conn_string(DB_URL)
graph = builder.compile(checkpointer=checkpointer)

# Resume from the latest checkpoint of a thread:
config = {"configurable": {"thread_id": session_id}}
result = graph.invoke(None, config)   # None input = "resume where we left off"

interface Checkpoint {
  session_id: string;
  task: string;
  completed_steps: Step[];
  pending_steps: Step[];
  decisions: Decision[];
  artifacts: string[];          // file paths produced
  written_at: string;           // ISO timestamp
}

async function writeCheckpoint(path: string, cp: Checkpoint): Promise<void> {
  // See 8.2 for the atomic-write contract.
  await atomicWriteJSON(path + ".tmp", path, cp);
}

-- Per-turn state, append-only.
CREATE TABLE agent_turns (
  turn_id     BIGSERIAL PRIMARY KEY,
  session_id  UUID NOT NULL REFERENCES agent_sessions(session_id),
  turn_number INT  NOT NULL,
  tool_name   TEXT,
  input_hash  TEXT,
  output_hash TEXT,
  payload     JSONB NOT NULL,         -- full structured turn
  written_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
  UNIQUE (session_id, turn_number)
);

-- Resumption reads: SELECT payload FROM agent_turns
--   WHERE session_id = $1 ORDER BY turn_number;

import { promises as fs } from "fs";

async function atomicWriteJSON(tmpPath: string, finalPath: string, state: unknown): Promise<void> {
  const data = JSON.stringify(state);
  // 1. Write the full payload to a temp file. A crash here leaves only the temp
  //    file, which the next run can ignore or garbage-collect.
  await fs.writeFile(tmpPath, data, { mode: 0o600 });
  // 2. fsync to flush kernel buffers to disk before the rename. Without this,
  //    a power loss after the rename can still leave an empty file on disk.
  const fh = await fs.open(tmpPath, "r");
  await fh.sync();           // fsync the temp file
  await fh.close();
  // 3. The rename is atomic on POSIX. This is the commit point.
  await fs.rename(tmpPath, finalPath);
}

def resume_session(handoff_path: str) -> Session:
    """Continuation-agent entry point. Reads handoff + checkpoint, orients, resumes."""
    handoff = read_handoff(handoff_path)           # human-legible — Module 4.2
    checkpoint = load_checkpoint(handoff.checkpoint_path)   # lossless — 8.1/8.2
    session = Session.restore(checkpoint)
    # Orient: what is the single highest-priority next action?
    session.focus_on(handoff.next_step)
    if handoff.blockers:
        session.surface_blockers_to_human(handoff.blockers)
    return session

# Module 8 — State, Checkpointing, and Multi-Session Design

**Course**: Master Course · **Module**: 8 · **Duration**: 45 min · **Prerequisites**: Modules 1–7

> *Stateless agents are prototypes.*

---

## Learning Objectives

1. Compare the 5 checkpointing patterns (git, LangGraph super-steps, external file, DB-backed, none).
2. Design crash-safe, atomic checkpoint writes.
3. Implement the multi-session handoff architecture (extends Module 4.2).

---

# 8.1 — Checkpointing Patterns

*What is serialized, when, and how is it restored? The choice is the second-most-consequential state decision in a harness, after the loop architecture.*

## The five patterns

Every stateful harness answers two questions: *what gets serialized*, and *when*. The answer determines whether the agent can resume after a crash, after a budget exhaustion (Module 7.4), after a context-window overflow (Module 3), or across deliberate multi-session boundaries (Module 4.2). There are five patterns in production use, each with a different cost/gain profile.

| Pattern | Mechanism | Examples |
| --- | --- | --- |
| **Git commits as checkpoints** | Each step committed; rollback via git reset | Claude Code |
| **LangGraph super-steps** | State serialized at each graph node boundary | LangGraph |
| **External state file** | Agent writes progress file; new session reads it | Aider, custom |
| **DB-backed state** | Full agent state in PostgreSQL/Redis per session | Enterprise |
| **No checkpointing** | Stateless; each session starts fresh | Pi, minimal harnesses |

The patterns differ on four axes, and the choice among them is a rubric-level decision (Module 0.3). State the decision: *this harness uses [pattern] because [use case demands X], trading [cost] for [gain].*

## Pattern 1: Git commits as checkpoints

Each meaningful step is a git commit. Rollback is `git reset`. Branching is `git branch`. Diffing two sessions is `git log`. The agent's version-control system *is* the checkpoint store — there is no separate state infrastructure.

This is Claude Code's model, and it is the right choice when the task is *code-centric*: the artifact is a repository, the steps are edits, and the resumption unit is "the working tree at the last commit." The checkpoint is free (you wanted version control anyway), human-readable (any engineer can read a git log), and rollback is native.

```python
def commit_checkpoint(repo, session, message: str) -> str:
    """Snapshot the working tree as a git commit. The commit SHA is the checkpoint id."""
    repo.index.add("*")
    sha = repo.index.commit(
        message,
        author=AgentAuthor(session.id),
    ).hexsha
    session.record_checkpoint(sha, kind="git", step=session.current_step)
    return sha

def rollback(repo, session, to_sha: str) -> None:
    repo.git.reset("--hard", to_sha)   # destructive — loses uncommitted work
    session.truncate_after(to_sha)
```

**Cost**: only works for code tasks. A non-code agent (research, analysis, customer support) has nothing to commit. The git history also gets polluted with agent checkpoints unless they are written to a separate branch.

## Pattern 2: LangGraph super-steps

LangGraph serializes state at each node boundary — what it calls a **super-step**. The graph transitions from node to node; at each transition, the full state object is written to a checkpointer (in-memory, SQLite, or Postgres). Resumption replays from any saved super-step.

```python
from langgraph.checkpoint.postgres import PostgresSaver

checkpointer = PostgresSaver.from_conn_string(DB_URL)
graph = builder.compile(checkpointer=checkpointer)

# Resume from the latest checkpoint of a thread:
config = {"configurable": {"thread_id": session_id}}
result = graph.invoke(None, config)   # None input = "resume where we left off"
```

**Gain**: fine-grained and resumable at any node. The `thread_id` is the resumption key; the checkpointer stores every intermediate state. This is what powers LangGraph's `interrupt()` (Module 6.2) — the human-in-the-loop primitive is built on the same checkpoint store.

**Cost**: requires LangGraph, and the state schema must be stable across resumptions. A schema change between session start and resume breaks deserialization. Enterprise deployments pin the schema version; long-lived sessions can outlive a schema migration, which is itself a failure mode.

## Pattern 3: External state file

The agent writes its own progress file (JSON, JSONL, Markdown) at meaningful boundaries. A new session reads it. This is Aider's model and the simplest possible checkpoint — no infrastructure beyond the filesystem.

```typescript
interface Checkpoint {
  session_id: string;
  task: string;
  completed_steps: Step[];
  pending_steps: Step[];
  decisions: Decision[];
  artifacts: string[];          // file paths produced
  written_at: string;           // ISO timestamp
}

async function writeCheckpoint(path: string, cp: Checkpoint): Promise<void> {
  // See 8.2 for the atomic-write contract.
  await atomicWriteJSON(path + ".tmp", path, cp);
}
```

**Gain**: simple, works anywhere, human-readable. The file is inspectable — you can open it in a text editor and see exactly what the agent thinks its state is.

**Cost**: the file is **not atomic by default** (a crash mid-write leaves a truncated file — see 8.2 for the fix), there is no native rollback (you must implement versioning yourself, often by appending rather than overwriting), and the file can drift from reality if the agent forgets to write it after a side effect.

## Pattern 4: DB-backed state

The full agent state — messages, tool calls, decisions, artifacts, telemetry — is stored in a database (Postgres for durability, Redis for speed) keyed by session id. This is the enterprise pattern: durable, queryable, multi-session, multi-tenant.

```sql
-- Per-turn state, append-only.
CREATE TABLE agent_turns (
  turn_id     BIGSERIAL PRIMARY KEY,
  session_id  UUID NOT NULL REFERENCES agent_sessions(session_id),
  turn_number INT  NOT NULL,
  tool_name   TEXT,
  input_hash  TEXT,
  output_hash TEXT,
  payload     JSONB NOT NULL,         -- full structured turn
  written_at  TIMESTAMPTZ NOT NULL DEFAULT now(),
  UNIQUE (session_id, turn_number)
);

-- Resumption reads: SELECT payload FROM agent_turns
--   WHERE session_id = $1 ORDER BY turn_number;
```

**Gain**: durable, queryable ("show me every session that called `send_email` last week"), supports multi-session analytics, and plays well with multi-tenant isolation (Module 4.3's per-tenant scoping, Fleet Module F06).

**Cost**: infrastructure. A database to operate, schema migrations to run, backups to test. The schema-migration problem from Pattern 2 is now a database migration, which is harder.

## Pattern 5: No checkpointing

Stateless. Each session starts fresh. Pi's default mode. This is not always wrong — for short, single-session tasks (a one-shot refactor, a single research query) checkpointing is overhead with no payoff.

**Cost**: cannot resume. Every interruption (crash, budget exhaustion, context overflow) is a full restart from step zero. The compounding-math problem from Module 7.1 applies: restarting a 50-step task because of a crash at step 49 is catastrophic. For any task longer than a handful of steps, "no checkpointing" is a defect.

## Tradeoffs summary

| Pattern | Gains | Costs |
| --- | --- | --- |
| Git commits | free version control; human-readable; rollback native | only for code tasks; git repo required |
| LangGraph super-steps | fine-grained; resumable at any node; powers `interrupt()` | requires LangGraph; state schema must be stable |
| External state file | simple; works anywhere | not atomic by default; no native rollback; can drift |
| DB-backed | durable; queryable; multi-session; multi-tenant | infrastructure; schema migrations |
| None | simplest code | cannot resume; every interruption = restart |

---

# 8.2 — Atomic, Crash-Safe Writes

*A checkpoint write must be atomic: either it fully completes, or it doesn't happen. A half-written checkpoint is worse than no checkpoint.*

## The atomicity contract

A checkpoint write must be **atomic**: either it fully completes, or it doesn't happen. A half-written checkpoint (the harness crashed mid-write) is worse than no checkpoint — resumption from a corrupt checkpoint produces undefined behavior. The agent resumes from a state that never existed, with half-applied side effects, and the resulting chaos is much harder to debug than a clean "no checkpoint, restart from zero."

The pattern: write to a temp file, then `rename` (atomic on POSIX filesystems). The rename is the commit point — before it, the old checkpoint is intact; after it, the new one is. A crash at any point leaves either the old or the new, never a mix.

```typescript
import { promises as fs } from "fs";

async function atomicWriteJSON(tmpPath: string, finalPath: string, state: unknown): Promise<void> {
  const data = JSON.stringify(state);
  // 1. Write the full payload to a temp file. A crash here leaves only the temp
  //    file, which the next run can ignore or garbage-collect.
  await fs.writeFile(tmpPath, data, { mode: 0o600 });
  // 2. fsync to flush kernel buffers to disk before the rename. Without this,
  //    a power loss after the rename can still leave an empty file on disk.
  const fh = await fs.open(tmpPath, "r");
  await fh.sync();           // fsync the temp file
  await fh.close();
  // 3. The rename is atomic on POSIX. This is the commit point.
  await fs.rename(tmpPath, finalPath);
}
```

Three subtleties in that code are worth flagging, because they are the difference between "atomic in theory" and "atomic in production."

**Subtlety 1: `fsync` before `rename`.** A bare `rename` is atomic *in the filesystem namespace*, but the file's data may still be in the kernel page cache. A power loss after the rename but before the data reaches disk can leave an empty or partial file at the destination. The `fsync` flushes the temp file's contents before the rename, so when the rename commits, the data is durable. This is the same lesson databases learned decades ago; it applies to agent checkpoints.

**Subtlety 2: rename is atomic on the same filesystem only.** `rename(2)` is atomic when source and destination are on the same filesystem. If the temp file is on `/tmp` (often tmpfs) and the destination is on `/data` (a real disk), the rename degrades to copy+unlink — *not* atomic. The temp file must live on the same filesystem as the destination.

**Subtlety 3: on Windows, `rename` fails if the destination exists.** POSIX `rename` overwrites atomically; Windows `MoveFileEx` does not, unless you pass `MOVEFILE_REPLACE_EXISTING`. Node's `fs.rename` handles this, but raw syscalls do not — a portability trap.

## The crash-recovery contract

The point of atomic writes is the crash-recovery contract they enable:

> **After a crash, the harness resumes from the last valid checkpoint with no duplicated side effects.**

Every step of the recovery is a property you can test:

1. **Last-valid property**: there exists a well-formed checkpoint to resume from. Guaranteed by atomicity — the worst case is the pre-step checkpoint, never a corrupt one.
2. **No-duplicate property**: side effects already applied are not re-applied. Guaranteed by idempotency (Module 7.2) — the agent re-runs the step, and the step is safe to re-run.
3. **Loss-bounded property**: at most one step of work is lost. Guaranteed by checkpoint frequency — you checkpoint after every meaningful step, so a crash costs at most one re-run.

The Module 8 lab verifies all three properties by killing the process mid-task with `SIGKILL` and asserting the resumed session matches expectations.

## Checkpoint frequency

How often to checkpoint is a tension between durability (checkpoint often — lose less on crash) and cost (every checkpoint is a write, and a synchronous checkpoint stalls the loop). The spectrum:

- **Every tool call** (maximal durability). The agent never loses more than one tool call. Cost: a write per turn; latency on long tasks.
- **Every meaningful step** (the typical default). A "meaningful step" is a logical unit — a plan step, a sub-task. Cost: writes are batched; one step of loss on crash.
- **On budget/context thresholds** (minimal). Checkpoint only when the session is about to be interrupted anyway. Cost: a crash between thresholds loses everything since the last one.

The right frequency depends on the cost of re-running a step. A step that costs $0.01 of tokens can be re-run freely; a step that called a paid API or sent an email cannot. **Checkpoint at least as often as the cost of re-running the intervening work becomes unacceptable.**

---

# 8.3 — Multi-Session Architectures

*The initializer + continuation pattern, formalized. Extends Module 4.2 with checkpointing as the substrate.*

## The handoff architecture

The multi-session pattern from Module 4.2, restated with checkpointing as the substrate:

- **Initializer agent** runs once; sets up environment; commits initial state (git checkpoint); writes handoff.
- **Continuation agents** read git log + handoff + checkpoint; orient; resume from highest-priority incomplete task.
- Each session writes a checkpoint before ending; the next session reads it.

The checkpoint is the technical substrate; the handoff file is the human-legible summary. **Both are needed**: the checkpoint for lossless resume, the handoff for the continuation agent's orientation. A checkpoint alone gives the continuation agent the *state* but not the *intent* — it does not know which pending task is highest priority, or which decisions are settled versus re-openable. The handoff carries that.

```python
def resume_session(handoff_path: str) -> Session:
    """Continuation-agent entry point. Reads handoff + checkpoint, orients, resumes."""
    handoff = read_handoff(handoff_path)           # human-legible — Module 4.2
    checkpoint = load_checkpoint(handoff.checkpoint_path)   # lossless — 8.1/8.2
    session = Session.restore(checkpoint)
    # Orient: what is the single highest-priority next action?
    session.focus_on(handoff.next_step)
    if handoff.blockers:
        session.surface_blockers_to_human(handoff.blockers)
    return session
```

## The session tree

When an agent branches — tries an approach, backs out, tries another — the natural representation is not a linear log but a **tree of sessions**. Each node is a checkpoint; branching creates a child; resumption walks a root-to-leaf path. This is the architecture Tau uses (DD-21) and the generalization of git's branch model applied to agent state.

> **Read this in real code: Tau's session tree (DD-21).** Tau implements checkpointing as an append-only JSONL session tree where state is reconstructed by replay. Each entry is typed and discriminated (`MessageEntry`, `CompactionEntry`, `BranchSummaryEntry`, `LeafEntry`). `SessionState.from_entries()` replays the entry list to derive current context — this is **event-sourcing** applied to agent sessions. Branching works by walking a root-to-leaf path; a `BranchSummaryEntry` injects branch context without carrying the full branch. The on-disk record is never rewritten. See `src/tau_agent/session/` — `jsonl.py`, `tree.py`, `memory.py`.

Why event-sourcing rather than state-snapshotting? Three reasons.

**1. Auditable history.** Every entry is on disk forever. You can answer "what did the agent believe at turn 14?" by replaying entries 1–14. A state-snapshot design can answer only "what does it believe now?"

**2. Non-destructive compaction.** Module 3's compaction — summarize old history — is destructive in a state-snapshot design (the raw history is gone). In an event-sourced design, a `CompactionEntry` *references* the entries it replaces; replay can run with or without the compaction applied. You get the token savings of compaction without losing the original history.

**3. Cheap branching.** A new branch is just a new leaf pointing at an existing node — no copy of the parent state. Backtracking to try a different approach is O(1), not O(state-size).

The cost is complexity: replaying a long session is slower than loading a single snapshot, so production systems cache the derived state alongside the event log. The tradeoff is the same one databases made when they adopted write-ahead logs: durability and auditability win, and the derived-state cache is an optimization on top.

## Crash recovery scenarios

The session tree plus atomic writes gives clean answers to the failure modes Module 7 catalogues:

| Scenario | Without checkpointing | With 8.1–8.2 checkpointing |
| --- | --- | --- |
| Process killed (SIGKILL) mid-step | All work lost; restart from zero | Resume from last checkpoint; at most one step re-run |
| Budget exhausted mid-task | Agent halts; in-progress state discarded | Checkpoint saved; handoff written; continuation resumes |
| Context window overflows | Compaction (Module 3) or crash | Checkpoint + new session continues with compacted context |
| User interrupts for approval | State lost if loop doesn't handle it | `interrupt()` (Module 6.2) checkpoints and waits |
| Tool side-effect applied twice after retry | Duplicate commit / email / charge | Idempotency (7.2) makes re-run safe |

The unifying property: **checkpointing converts catastrophic interruptions into bounded re-runs.** The worst case is never "start over from zero" — it is "redo the last step." That property is what separates a production agent from a prototype.

## Session diffing

Compare two sessions to find behavioral drift — useful for regression testing after a model or prompt change. Did the new prompt make the agent faster? More reliable? More error-prone? Session diffing answers by comparing checkpointed states across sessions.

With git checkpoints, this is `git log` and `git diff` across branches. With event-sourced JSONL (Tau), it is a diff of the entry streams — which tool calls differ, where the branches diverge, which decisions reversed. With a DB-backed store, it is a SQL query joining two `session_id`s on `turn_number` and returning differing rows. The checkpoint format determines what diffing is possible; choose a format that makes the diffs you need computable.

---

## Anti-Patterns

### The non-atomic write
`fs.writeFile(path, data)` directly, no temp+rename. A crash mid-write corrupts the checkpoint. Cure: temp-file + fsync + rename (8.2).

### The cross-filesystem rename
Temp file on `/tmp`, destination on `/data`. The rename silently degrades to copy+unlink — not atomic. Cure: same-filesystem temp directory.

### The stale handoff
Handoff file written at session start but never updated. Continuation agent reads stale "next step" and does the wrong thing. Cure: write the handoff at every checkpoint, or at least at session end.

### The schema that drifts
DB-backed or event-sourced state whose schema changes between session start and resume. Deserialization fails. Cure: pin schema versions; migrate at session boundaries, not mid-session.

### State without idempotency
Checkpoint lets the agent resume — but the resumed step re-applies a side effect. Two commits, two emails. Cure: idempotency (Module 7.2) is the precondition for safe resumption.

---

## Key Terms

| Term | Definition |
| --- | --- |
| **Checkpoint** | Serialized agent state at a point in time; enables resume |
| **Super-step** | LangGraph's per-node checkpoint boundary |
| **Atomic write** | temp-file + fsync + rename; crash leaves old-or-new, never mixed |
| **Crash-recovery contract** | Resume from last valid checkpoint; no duplicated side effects; at most one step lost |
| **Session tree** | Branching tree of checkpoints; root-to-leaf path = one session's history |
| **Event sourcing** | Append-only event log; state derived by replay; non-destructive compaction |
| **Session diffing** | Comparing two sessions to find behavioral drift |
| **Handoff** | Human-legible resume summary (Module 4.2); checkpoint is the technical substrate |

---

## Lab Exercise

See `07-lab-spec.md`. Implement atomic checkpoint writes (temp + fsync + rename). Kill the process mid-task with `SIGKILL`; verify the harness resumes from the last checkpoint with no duplicated side effects. Build a 3-session task that pauses between sessions via checkpoints.

---

## References

1. **POSIX `rename(2)`** — the atomicity guarantee (same-filesystem only).
2. **PostgreSQL documentation** — write-ahead logging; the event-sourcing precedent for agent sessions.
3. **LangGraph documentation** — super-step checkpoints and the `thread_id` resumption model.
4. **HuggingFace Tau source** — the session-tree / event-sourced checkpointing reference (DD-21).
5. **Module 4.2** — the handoff pattern (this module adds the checkpoint substrate).
6. **Module 6.2** — `interrupt()` requires checkpointing (two views of the same infrastructure).
7. **Module 7.2** — idempotency as the precondition for safe crash recovery.
8. **Module 7.4** — budget-exhaustion recovery via checkpoint.
9. **Fleet Module F06** — per-tenant checkpoint isolation at fleet scale.