diff

Cell-level git diffs for notebooks

Notebook JSON is nearly unreadable in git diff: one changed cell shows up as a wall of escaped source, and metadata noise swamps the signal. The functions here diff notebooks at the cell level instead. Cells are matched by their id, so pairing is exact with no alignment heuristics, and each changed cell is reported as a small unified diff of its source. render_diff turns that into terminal-friendly output, and nbdev_diff_driver wires it into git diff itself (installed by nbdev-install-hooks).

import tempfile, random, io
from contextlib import redirect_stdout
from fastcore.test import *

random.seed(42)

Everything below works against a demo repo holding one committed notebook plus some uncommitted edits:

tmp = tempfile.TemporaryDirectory(prefix='nbdiff_test_')
td = Path(tmp.name)
g = Git(td)
g.init(b='main')
g('config', 'user.email', '[email protected]')
g('config', 'user.name', 'nbdev')

''

Its one committed notebook has five one-line cells, the first tagged #| export:

nb_path = td/'test.ipynb'
nb = new_nb(['#| export\nx=1', 'y=2', 'a=3', 'b=4', 'c=5'])
write_nb(nb, nb_path)
g.add('test.ipynb')
g.commit(m='initial notebook')

'[main (root-commit) 3c848ff] initial notebook\n 1 file changed, 58 insertions(+)\n create mode 100644 test.ipynb'

The uncommitted edits change the first cell’s source and append a cell carrying a stored output and a meta-form eval: false directive:

nb.cells[0].source = '#| export\nx = 100'
out = dict(output_type='execute_result', data={'text/plain':'101'}, metadata={}, execution_count=1)
nb.cells.append(mk_cell('x+1', outputs=[out], metadata={'nbdev': {'eval': 'false'}}))
write_nb(nb, nb_path)

---

source

read_nb_from_git

def read_nb_from_git(
    g:fastgit.core.Git, # The git object
    path, # The path to the notebook (absolute or relative to git root)
    ref:NoneType=None, # The git ref to read from (e.g. HEAD); None for working dir
)->fastcore.basics.AttrDict: # The notebook; empty if `path` doesn't exist at `ref`

Read notebook from git ref (e.g. HEAD) at path, or working dir if ref is None

read_nb_from_git(g, 'test.ipynb', 'HEAD').cells

[{'cell_type': 'code',
  'execution_count': None,
  'id': '390c8c7d',
  'metadata': {},
  'outputs': [],
  'source': '#| export\nx=1',
  'idx_': 0,
  'lang_': 'python'},
 {'cell_type': 'code',
  'execution_count': None,
  'id': '7247342c',
  'metadata': {},
  'outputs': [],
  'source': 'y=2',
  'idx_': 1,
  'lang_': 'python'},
 {'cell_type': 'code',
  'execution_count': None,
  'id': 'd8100f2f',
  'metadata': {},
  'outputs': [],
  'source': 'a=3',
  'idx_': 2,
  'lang_': 'python'},
 {'cell_type': 'code',
  'execution_count': None,
  'id': '6f770d65',
  'metadata': {},
  'outputs': [],
  'source': 'b=4',
  'idx_': 3,
  'lang_': 'python'},
 {'cell_type': 'code',
  'execution_count': None,
  'id': 'd670e58e',
  'metadata': {},
  'outputs': [],
  'source': 'c=5',
  'idx_': 4,
  'lang_': 'python'}]

A path that doesn’t exist at ref (such as a notebook not yet committed) comes back as an empty notebook, so a new file diffs as “all cells added”. Other git failures, such as a bad ref, still raise:

test_eq(read_nb_from_git(g, 'new.ipynb', 'HEAD').cells, [])
test_fail(lambda: read_nb_from_git(g, 'test.ipynb', 'nosuchref'))

Cells are keyed by id. Notebooks saved before nbformat 4.5 have no cell ids, so their cells get positional c<n> keys instead, which still pair correctly as long as no cells were reordered:

nb0 = new_nb(['a', 'b'])
for c in nb0.cells: c.pop('id')
test_eq(_srcdict(nb0, _src), {'c0': 'a', 'c1': 'b'})

---

source

nbs_pair

def nbs_pair(
    nb_path, # Path to the notebook
    ref_a:str='HEAD', # First git ref (None for working dir)
    ref_b:NoneType=None, # Second git ref (None for working dir)
    f:function=noop, # Function to call on contents
):

NBs at two refs; None means working dir. By default provides HEAD and working dir

By default nbs_pair compares HEAD with the working directory, returning each version as {id: cell}. f maps the cells, so passing _src shows just the edited version’s sources, without their directive lines:

a,b = nbs_pair(nb_path)
nbs_pair(nb_path, f=_src)[1]

{'390c8c7d': 'x = 100',
 '7247342c': 'y=2',
 'd8100f2f': 'a=3',
 '6f770d65': 'b=4',
 'd670e58e': 'c=5',
 '0351d8ae': 'x+1'}

---

source

changed_cells

def changed_cells(
    nb_path, *, ref_a:str='HEAD', # First git ref (None for working dir)
    ref_b:NoneType=None, # Second git ref (None for working dir)
    adds:bool=True, # Include cells in b but not in a
    changes:bool=True, # Include cells with different content
    dels:bool=False, # Include cells in a but not in b
    metadata:bool=False, # Consider cell metadata when comparing
    outputs:bool=False, # Consider cell outputs when comparing
):

Return set of cell IDs for changed/added/deleted cells between two refs

Added and modified cells are reported by default. Deleted cells are opt-in via dels, since they have no content in the new notebook to look at, and metadata and outputs widen what counts as a change. The demo repo’s two edits:

changed_cells(td/'test.ipynb')

{'0351d8ae', '390c8c7d'}

Each kind can be switched off; changes=False leaves just the added cell:

test_eq(changed_cells(td/'test.ipynb', changes=False), {nb.cells[-1].id})

---

source

source_diff

def source_diff(
    old_source, # Original source string
    new_source, # New source string
):

Return unified diff string for source change

unified_diff starts with ---/+++ file-header lines, empty here since cells have no filenames, which is why render_diff drops the first two lines of each diff:

print(source_diff('x = 1\ny=2', 'x = 100\ny=2'))

--- 
+++ 
@@ -1,2 +1,2 @@
-x = 1
+x = 100
 y=2

---

source

cell_diffs

def cell_diffs(
    nb_path, *, ref_a:str='HEAD', # First git ref (None for working dir)
    ref_b:NoneType=None, # Second git ref (None for working dir)
    adds:bool=True, # Include cells in b but not in a
    changes:bool=True, # Include cells with different content
    dels:bool=False, # Include cells in a but not in b
    metadata:bool=False, # Consider cell metadata when comparing
    outputs:bool=False, # Consider cell outputs when comparing
):

{cell_id:diff} for changed/added/deleted cells between two refs

cell_diffs pairs each changed id with its source diff, so the modified cell shows its old and new line, and the added cell only new lines:

d = cell_diffs(td/'test.ipynb')
d

{'0351d8ae': '--- \n+++ \n@@ -0,0 +1 @@\n+x+1',
 '390c8c7d': '--- \n+++ \n@@ -1,2 +1,2 @@\n #| export\n-x=1\n+x = 100'}

Rendering

render_diff combines cell headers, source changes, context cells, and stored outputs into a terminal display. Similar changed lines use word-level highlighting. The examples below develop these parts before rendering the demo notebook.

_dline truncates one line to maxlen, ending a cut line with … and the number of characters cut in brackets. It also colors a line by its leading marker: deletions red, additions green, hunk markers cyan, skipped-cell markers blue, cell headers bold, and the # path file header bold and inverse. Context lines stay plain:

test_eq(_dline('x'*200, maxlen=10), 'xxxx…[196]')
test_eq(_dline('-del', color=True), '\x1b[31m-del\x1b[0m')
test_eq(_dline(' context', color=True), ' context')
test_eq(_dline('# nb.ipynb', color=True), '\x1b[1;7m# nb.ipynb\x1b[0m')
test_eq(_dline('## modified', color=True), '\x1b[1m## modified\x1b[0m')
test_eq(_dline('## … 2 cells', color=True), '\x1b[34m## … 2 cells\x1b[0m')

A deleted line and the added line in the same position merge into one ! line when their word similarity reaches MIN_RATIO. Similarity uses SequenceMatcher on \w+ runs, excluding whitespace and punctuation. The displayed diff retains all three: word runs, whitespace runs, and individual punctuation characters.

Without color, deletions use [-…-] and additions use {+…+}. With color, deletions are red and struck out, and additions are green:

w = _word_diff('x=1', 'x = 100')
test_eq(_wline(w), '!x{+ +}=[-1-]{+ 100+}')
test_eq(_wline(w, color=True), '!x\x1b[32m \x1b[0m=\x1b[31;9m1\x1b[0m\x1b[32m 100\x1b[0m')
test_eq(_wline(w, maxlen=6, color=True), '!x…[7]')
w

[(' ', 'x'), ('+', ' '), (' ', '='), ('-', '1'), ('+', ' 100')]

Lines too different to merge stay a -/+ pair, and lines beyond maxlen are never merged, since a merged line longer than the display would show little of either version:

assert _word_diff('import os', 'from pathlib import Path') is None
assert _word_diff('from fasttransport.errors import APIError', 'from fastspec.spec import OpSpec') is None
assert _word_diff('x=1', 'x = 100', maxlen=5) is None

_refine applies this to a whole diff, pairing each run of - lines with the + run that follows it position by position. Pairs that don’t merge keep the plain diff’s shape, a block of - lines followed by a block of + lines, with any unpaired leftovers joining that block:

lines = source_diff('x=1\ny=2\nz=3\nk=4', 'x = 100\ny=2\nw=5\nq=6').splitlines()[2:]
r = '\n'.join(_dline(l) for l in _refine(lines, MAXLEN))
assert '-z=3\n-k=4\n+w=5\n+q=6' in r
PrettyString(r)

@@ -1,4 +1,4 @@
!x{+ +}=[-1-]{+ 100+}
 y=2
-z=3
-k=4
+w=5
+q=6

Deleted cells aren’t in the new notebook, so to place them in the output each one goes right after its nearest surviving predecessor from the old notebook. A notebook that is entirely deleted or entirely new keeps its own order:

test_eq(_diff_order(dict.fromkeys('abcd'), dict.fromkeys('ace')), list('abcde'))
test_eq(_diff_order(dict.fromkeys('ab'), {}), list('ab'))
test_eq(_diff_order({}, dict.fromkeys('xy')), list('xy'))

---

source

render_diff

def render_diff(
    old, # `{id: cell}` for the old version of the notebook
    new, # `{id: cell}` for the new version
    maxlen:int=180, # Truncate diff lines to this width (falsy: no limit)
    color:bool=False, # Add ANSI colors?
    context:int=1, # Unchanged cells to show either side of each change
    show_out:bool=True, # Show the stored outputs of changed and added cells?
):

Render cell-level changes between two notebooks as truncated unified diffs, with context cells and outputs

Each changed cell has a header with its change kind, id, and merged nbdev directives. Moving a directive between source and metadata makes no difference. Changing a directive produces a header even when the source is unchanged. Directive lines are omitted from the source diff.

The edit to x becomes one ! line. Unchanged neighbours provide context, and a marker counts the cells skipped between hunks. The added cell’s stored output follows its diff with a | prefix. Stored output is capped at OUT_MAXLEN characters:

ids = list(b)
r = render_diff(a, b)
assert f'## modified {ids[0]} [export]:\n@@ -1 +1 @@\n!x{{+ +}}=[-1-]{{+ 100+}}' in r
assert f'## added {ids[-1]} [eval=false]:' in r
test_eq(render_diff({'d': mk_cell('#| export\nz=1')}, {'d': mk_cell('z=1', metadata={'nbdev': {'export': 'true'}})}), '')
mixed = mk_cell('#| export\nz=1', metadata={'nbdev': {'eval': 'false'}})
test_eq(render_diff({'d': mk_cell('z=1')}, {'d': mixed}), '## modified d [export eval=false]:')
assert f'## {ids[1]}:\n y=2' in r and f'## {ids[4]}:' in r and ids[2] not in r
assert '## … 2 cells' in r and '| 101' in r
assert '\x1b[34m## … 2 cells\x1b[0m' in render_diff(a, b, color=True)
PrettyString(r)

## modified 390c8c7d [export]:
@@ -1 +1 @@
!x{+ +}=[-1-]{+ 100+}

## 7247342c:
 y=2

## … 2 cells

## d670e58e:
 c=5

## added 0351d8ae [eval=false]:
@@ -0,0 +1 @@
+x+1
| 101

Set color=True to use ANSI colors instead of word-diff markers. Deleted cells retain their change label. Set context=0 and show_out=False to show only changed source:

rc = render_diff(a, b, color=True)
assert '\x1b[31;9m1\x1b[0m' in rc and '\x1b[32m+x+1\x1b[0m' in rc
assert '## deleted' in render_diff({'gone': mk_cell('x=9')} | a, b)
r0 = render_diff(a, b, context=0, show_out=False)
assert ids[1] not in r0 and '| 101' not in r0
PrettyString(r0)

## modified 390c8c7d [export]:
@@ -1 +1 @@
!x{+ +}=[-1-]{+ 100+}

## … 4 cells

## added 0351d8ae [eval=false]:
@@ -0,0 +1 @@
+x+1

---

source

nb_diff

def nb_diff(
    nb_path, # Path to the notebook
    ref_a:str='HEAD', # First git ref
    ref_b:NoneType=None, # Second git ref; None for working dir
    maxlen:int=180, # Truncate diff lines to this width (falsy: no limit)
    color:bool=False, # Add ANSI colors?
    context:int=1, # Unchanged cells to show either side of each change
    show_out:bool=True, # Show the stored outputs of changed and added cells?
):

Rendered cell diff for nb_path between two refs

Long lines are truncated at maxlen and marked with an ellipsis and the number of characters cut. They skip word-level matching, but reading and comparing their source still depends on line length. Set maxlen=0 to show complete lines:

nb.cells[1].source = 'y = 2  # ' + ', '.join(f'note {i}' for i in range(30))
write_nb(nb, nb_path)
long_diff = nb_diff(nb_path)
assert all(len(l)<=MAXLEN for l in long_diff.splitlines())
assert any(re.search(r'…\[\d+\]$', l) for l in long_diff.splitlines())
assert nb.cells[1].source in nb_diff(nb_path, maxlen=0)
PrettyString(long_diff)

## modified 390c8c7d [export]:
@@ -1 +1 @@
!x{+ +}=[-1-]{+ 100+}

## modified 7247342c:
@@ -1 +1 @@
-y=2
+y = 2  # note 0, note 1, note 2, note 3, note 4, note 5, note 6, note 7, note 8, note 9, note 10, note 11, note 12, note 13, note 14, note 15, note 16, note 17, note 18, not…[268]

## d8100f2f:
 a=3

## … 1 cells

## d670e58e:
 c=5

## added 0351d8ae [eval=false]:
@@ -0,0 +1 @@
+x+1
| 101

Git integration

nbdev_diff_driver is a git external diff driver: git calls it once per changed notebook, passing the repo path plus temp files holding the pre- and post-change versions (/dev/null when the file was added or deleted). nbdev-install-hooks registers it, after which plain git diff shows cell-level output for notebooks; pass --diff false there to install the merge driver only. When git detects a rename it appends two extra arguments, and for an unmerged path it passes just the repo path.

---

source

nbdev_diff_driver

def nbdev_diff_driver(
    path:str, # Repo path of the notebook being diffed
    old_file:str=None, # Pre-change file (git temp file, or /dev/null)
    old_hex:str=None, # Pre-change blob hash
    old_mode:str=None, # Pre-change file mode
    new_file:str=None, # Post-change file
    new_hex:str=None, # Post-change blob hash
    new_mode:str=None, # Post-change file mode
    rename_to:str=None, # New repo path, when git detected a rename
    similarity:str=None, # Similarity score, when git detected a rename
    maxlen:int=180, # Truncate diff lines to this width (0 for no limit)
    context:int=1, # Unchanged cells to show either side of each change
    show_out:bool=True, # Show the stored outputs of changed and added cells?
):

Git external diff driver for notebooks; installed by nbdev-install-hooks

We can call the driver directly the same way git does, with the old version extracted to a file:

old_p = td/'old.ipynb'
old_p.write_text(g.show('HEAD:test.ipynb'))
nbdev_diff_driver('test.ipynb', str(old_p), '', '', str(nb_path), '', '')

# test.ipynb
## modified 390c8c7d [export]:
@@ -1 +1 @@
!x{+ +}=[-1-]{+ 100+}

## modified 7247342c:
@@ -1 +1 @@
-y=2
+y = 2  # note 0, note 1, note 2, note 3, note 4, note 5, note 6, note 7, note 8, note 9, note 10, note 11, note 12, note 13, note 14, note 15, note 16, note 17, note 18, not…[268]

## d8100f2f:
 a=3

## … 1 cells

## d670e58e:
 c=5

## added 0351d8ae [eval=false]:
@@ -0,0 +1 @@
+x+1
| 101

When sources and directives are unchanged, the driver prints nothing, including the path header. Stored outputs and other metadata do not count as changes:

s = io.StringIO()
with redirect_stdout(s): nbdev_diff_driver('test.ipynb', str(old_p), '', '', str(old_p), '', '')
test_eq(s.getvalue(), '')

---

source

nbdev_diff

def nbdev_diff(
    path:str=None, # Notebook or directory (default: project's notebooks folder)
    ref_a:str='HEAD', # First git ref
    ref_b:str=None, # Second git ref (default: working directory)
    maxlen:int=180, # Truncate diff lines to this width (0 for no limit)
    color:<function bool_arg at 0x7f2b0a845120>=None, # Add ANSI colors? (default: only if stdout is a tty)
    context:int=1, # Unchanged cells to show either side of each change
    show_out:bool=True, # Show the stored outputs of changed and added cells?
):

Cell-level diffs for changed notebooks between two git refs

nbdev-diff is the standalone version, like a cell-level git diff for notebooks: it shows changes between two refs (or a ref and the working directory) for one notebook, a directory, or the whole project. Run from the demo repo, it renders the same pending changes, headed by the notebook path:

with working_directory(td): nbdev_diff('test.ipynb', color=False)

# test.ipynb
## modified 390c8c7d [export]:
@@ -1 +1 @@
!x{+ +}=[-1-]{+ 100+}

## modified 7247342c:
@@ -1 +1 @@
-y=2
+y = 2  # note 0, note 1, note 2, note 3, note 4, note 5, note 6, note 7, note 8, note 9, note 10, note 11, note 12, note 13, note 14, note 15, note 16, note 17, note 18, not…[268]

## d8100f2f:
 a=3

## … 1 cells

## d670e58e:
 c=5

## added 0351d8ae [eval=false]:
@@ -0,0 +1 @@
+x+1
| 101

g.add('test.ipynb')
g.commit(m='update notebook')
assert not changed_cells(td/'test.ipynb')
assert not cell_diffs(td/'test.ipynb')

tmp.cleanup()