diff

Cell-level git diffs for notebooks

Notebook JSON is nearly unreadable in git diff: one changed cell shows up as a wall of escaped source, and metadata noise swamps the signal. The functions here diff notebooks at the cell level instead. Cells are matched by their id, so pairing is exact with no alignment heuristics, and each changed cell is reported as a small unified diff of its source. render_diff turns that into terminal-friendly output, and nbdev_diff_driver wires it into git diff itself (installed by nbdev-install-hooks).

import shutil, tempfile, random, io
from contextlib import redirect_stdout
from fastcore.test import *

random.seed(42)

Everything below works against a demo repo holding one committed notebook plus some uncommitted edits: a changed cell and an added cell.

td = Path(tempfile.mkdtemp(prefix='nbdiff_test_'))
g = Git(td)
g.init(b='main')
g('config', 'user.email', '[email protected]')
g('config', 'user.name', 'nbdev')
nb_path = td/'test.ipynb'
nb = new_nb(['x=1', 'y=2'])
write_nb(nb, nb_path)
g.add('test.ipynb')
g.commit(m='initial notebook')
nb.cells[0].source = 'x = 100'
nb.cells.append(mk_cell('z=3'))
write_nb(nb, nb_path)

---

source

read_nb_from_git

def read_nb_from_git(
    g:Git, # The git object
    path, # The path to the notebook (absolute or relative to git root)
    ref:NoneType=None, # The git ref to read from (e.g. HEAD); None for working dir
)->AttrDict: # The notebook; empty if `path` doesn't exist at `ref`

Read notebook from git ref (e.g. HEAD) at path, or working dir if ref is None

read_nb_from_git(g, 'test.ipynb', 'HEAD').cells

[{'cell_type': 'code',
  'execution_count': None,
  'id': '390c8c7d',
  'metadata': {},
  'outputs': [],
  'source': 'x=1',
  'idx_': 0,
  'lang_': 'python'},
 {'cell_type': 'code',
  'execution_count': None,
  'id': '7247342c',
  'metadata': {},
  'outputs': [],
  'source': 'y=2',
  'idx_': 1,
  'lang_': 'python'}]

A path that doesn’t exist at ref (such as a notebook not yet committed) comes back as an empty notebook, so a new file diffs as “all cells added”. Other git failures, such as a bad ref, still raise:

test_eq(read_nb_from_git(g, 'new.ipynb', 'HEAD').cells, [])
test_fail(lambda: read_nb_from_git(g, 'test.ipynb', 'nosuchref'))

---

source

nbs_pair

def nbs_pair(
    nb_path, # Path to the notebook
    ref_a:str='HEAD', # First git ref (None for working dir)
    ref_b:NoneType=None, # Second git ref (None for working dir)
    f:function=noop, # Function to call on contents
):

NBs at two refs; None means working dir. By default provides HEAD and working dir

a,b = nbs_pair(nb_path)
a

{'390c8c7d': {'cell_type': 'code',
  'execution_count': None,
  'id': '390c8c7d',
  'metadata': {},
  'outputs': [],
  'source': 'x=1',
  'idx_': 0,
  'lang_': 'python'},
 '7247342c': {'cell_type': 'code',
  'execution_count': None,
  'id': '7247342c',
  'metadata': {},
  'outputs': [],
  'source': 'y=2',
  'idx_': 1,
  'lang_': 'python'}}

---

source

changed_cells

def changed_cells(
    nb_path, ref_a:str='HEAD', # First git ref (None for working dir)
    ref_b:NoneType=None, # Second git ref (None for working dir)
    adds:bool=True, # Include cells in b but not in a
    changes:bool=True, # Include cells with different content
    dels:bool=False, # Include cells in a but not in b
    metadata:bool=False, # Consider cell metadata when comparing
    outputs:bool=False, # Consider cell outputs when comparing
):

Return set of cell IDs for changed/added/deleted cells between two refs

changed_cells(td/'test.ipynb')

{'390c8c7d', 'd8100f2f'}

---

source

source_diff

def source_diff(
    old_source, # Original source string
    new_source, # New source string
):

Return unified diff string for source change

print(source_diff('x = 1\ny=2', 'x = 100\ny=2'))

--- 
+++ 
@@ -1,2 +1,2 @@
-x = 1
+x = 100
 y=2

---

source

cell_diffs

def cell_diffs(
    nb_path, ref_a:str='HEAD', # First git ref (None for working dir)
    ref_b:NoneType=None, # Second git ref (None for working dir)
    adds:bool=True, # Include cells in b but not in a
    changes:bool=True, # Include cells with different content
    dels:bool=False, # Include cells in a but not in b
    metadata:bool=False, # Consider cell metadata when comparing
    outputs:bool=False, # Consider cell outputs when comparing
):

{cell_id:diff} for changed/added/deleted cells between two refs

d = cell_diffs(td/'test.ipynb')
d

{'d8100f2f': '--- \n+++ \n@@ -0,0 +1 @@\n+z=3',
 '390c8c7d': '--- \n+++ \n@@ -1 +1 @@\n-x=1\n+x = 100'}

Rendering

cell_diffs gives us the data; for git diff we need something readable in a terminal. render_diff prints one section per changed cell, in notebook order, headed by the cell id and the kind of change. Cell sources can contain very long lines (big markdown cells, embedded data) which line-oriented diff output would dump in full, so every line is truncated to maxlen characters, with an ellipsis marking the cut. Colors are plain ANSI escapes keyed on each line’s leading diff character.

test_eq(_dline('x'*200, maxlen=10), 'x'*10+'…')
test_eq(_dline('-del', color=True), '\x1b[31m-del\x1b[0m')
test_eq(_dline(' context', color=True), ' context')
test_eq(_dline('# nb.ipynb', color=True), '\x1b[1;7m# nb.ipynb\x1b[0m')  # file header: bold+inverse
test_eq(_dline('## modified', color=True), '\x1b[1m## modified\x1b[0m')  # cell header: bold

Deleted cells aren’t in the new notebook, so to place them in the output each one goes right after its nearest surviving predecessor from the old notebook:

test_eq(_diff_order(dict.fromkeys('abcd'), dict.fromkeys('ace')), list('abcde'))
test_eq(_diff_order(dict.fromkeys('ab'), {}), list('ab'))
test_eq(_diff_order({}, dict.fromkeys('xy')), list('xy'))

---

source

render_diff

def render_diff(
    old, # `{id: source}` for the old version of the notebook
    new, # `{id: source}` for the new version
    maxlen:int=120, # Truncate diff lines to this width (falsy: no limit)
    color:bool=False, # Add ANSI colors?
):

Render cell-level changes between two notebooks as truncated unified diffs

Rendering our demo repo’s pending changes shows the modified cell and the added one, each under a header carrying the change kind and cell id:

a,b = nbs_pair(nb_path, f=_src)
print(render_diff(a, b))

## modified 390c8c7d:
@@ -1 +1 @@
-x=1
+x = 100

## added d8100f2f:
@@ -0,0 +1 @@
+z=3

r = render_diff(a, b)
assert '## modified 390c8c7d:' in r and '## added' in r
assert '\x1b[32m+x = 100\x1b[0m' in render_diff(a, b, color=True)
assert '## deleted' in render_diff({'gone':'x=9'} | a, b)

---

source

nb_diff

def nb_diff(
    nb_path, # Path to the notebook
    ref_a:str='HEAD', # First git ref
    ref_b:NoneType=None, # Second git ref; None for working dir
    maxlen:int=120, # Truncate diff lines to this width (falsy: no limit)
    color:bool=False, # Add ANSI colors?
):

Rendered cell diff for nb_path between two refs

Long lines get the ellipsis treatment. This matters more than it may look: nbdime, for example, computes character-level diffs, which go quadratic on multi-KB single lines and can hang git diff for minutes on programmatically-generated notebooks. Diffing and truncating at the line level makes cost independent of line length:

nb.cells[1].source = 'y = 2  # ' + ', '.join(f'note {i}' for i in range(30))
write_nb(nb, nb_path)
print(nb_diff(nb_path))

## modified 390c8c7d:
@@ -1 +1 @@
-x=1
+x = 100

## modified 7247342c:
@@ -1 +1 @@
-y=2
+y = 2  # note 0, note 1, note 2, note 3, note 4, note 5, note 6, note 7, note 8, note 9, note 10, note 11, note 12, not…

## added d8100f2f:
@@ -0,0 +1 @@
+z=3

assert all(len(l)<=121 for l in nb_diff(nb_path).splitlines())
assert any(l.endswith('…') for l in nb_diff(nb_path).splitlines())
assert all(len(l)>121 for l in nb_diff(nb_path, maxlen=0).splitlines() if l.startswith('+y'))

Git integration

nbdev_diff_driver is a git external diff driver: git calls it once per changed notebook, passing the repo path plus temp files holding the pre- and post-change versions (/dev/null when the file was added or deleted). nbdev-install-hooks registers it, after which plain git diff shows cell-level output for notebooks; pass --diff false there to install the merge driver only. When git detects a rename it appends two extra arguments, and for an unmerged path it passes just the repo path.

---

source

nbdev_diff_driver

def nbdev_diff_driver(
    path:str, # Repo path of the notebook being diffed
    old_file:Annotated=None, # Pre-change file (git temp file, or /dev/null)
    old_hex:Annotated=None, # Pre-change blob hash
    old_mode:Annotated=None, # Pre-change file mode
    new_file:Annotated=None, # Post-change file
    new_hex:Annotated=None, # Post-change blob hash
    new_mode:Annotated=None, # Post-change file mode
    rename_to:Annotated=None, # New repo path, when git detected a rename
    similarity:Annotated=None, # Similarity score, when git detected a rename
    maxlen:int=120, # Truncate diff lines to this width (0 for no limit)
):

Git external diff driver for notebooks; installed by nbdev-install-hooks

We can call the driver directly the same way git does, with the old version extracted to a file:

old_p = td/'old.ipynb'
old_p.write_text(g.show('HEAD:test.ipynb'))
nbdev_diff_driver('test.ipynb', str(old_p), '', '', str(nb_path), '', '')

# test.ipynb
## modified 390c8c7d:
@@ -1 +1 @@
-x=1
+x = 100

## modified 7247342c:
@@ -1 +1 @@
-y=2
+y = 2  # note 0, note 1, note 2, note 3, note 4, note 5, note 6, note 7, note 8, note 9, note 10, note 11, note 12, not…

## added d8100f2f:
@@ -0,0 +1 @@
+z=3

When cell sources are unchanged (e.g. only outputs or metadata differ), the driver prints nothing at all, not even the path header:

s = io.StringIO()
with redirect_stdout(s): nbdev_diff_driver('test.ipynb', str(old_p), '', '', str(old_p), '', '')
test_eq(s.getvalue(), '')

nbdev-diff is the standalone version, like a cell-level git diff for notebooks: it shows changes between two refs (or a ref and the working directory) for one notebook, a directory, or the whole project.

---

source

nbdev_diff

def nbdev_diff(
    path:str=None, # Notebook or directory (default: project's notebooks folder)
    ref_a:str='HEAD', # First git ref
    ref_b:str=None, # Second git ref (default: working directory)
    maxlen:int=120, # Truncate diff lines to this width (0 for no limit)
    color:bool_arg=None, # Add ANSI colors? (default: only if stdout is a tty)
):

Cell-level diffs for changed notebooks between two git refs

with working_directory(td): nbdev_diff('test.ipynb', color=False)

# test.ipynb
## modified 390c8c7d:
@@ -1 +1 @@
-x=1
+x = 100

## modified 7247342c:
@@ -1 +1 @@
-y=2
+y = 2  # note 0, note 1, note 2, note 3, note 4, note 5, note 6, note 7, note 8, note 9, note 10, note 11, note 12, not…

## added d8100f2f:
@@ -0,0 +1 @@
+z=3

g.add('test.ipynb')
g.commit(m='update notebook')
assert not changed_cells(td/'test.ipynb')
assert not cell_diffs(td/'test.ipynb')

shutil.rmtree(td)