""" activations.py: the "everything else" line. How much scratch memory does ONE step need? Same tiny factory as Part 5 (GPT-2-small shape, random weights, NumPy float32, CPU). We push a prompt through in one go, then again in chunks of 512 tokens (each chunk writes its notes to the notebook and reads all earlier notes), and record the peak extra memory each step allocates on top of the model and the notebook (tracemalloc). """ import tracemalloc, time import numpy as np D, LAYERS, HEADS, D_FF, VOCAB = 768, 12, 12, 3072, 256 HD = D // HEADS rng = np.random.default_rng(0) w = lambda *s: (rng.standard_normal(s) * 0.02).astype(np.float32) EMB, POS = w(VOCAB, D), w(4096, D) L = [dict(qkv=w(3 * D, D), out=w(D, D), up=w(D_FF, D), down=w(D, D_FF)) for _ in range(LAYERS)] HEAD = w(VOCAB, D) def norm(x): return (x - x.mean(0)) / np.sqrt(x.var(0) + 1e-5) def gelu(x): return 0.5 * x * (1 + np.tanh(0.7978845608 * (x + 0.044715 * x * x * x))) def softmax(a): a = np.exp(a - a.max(-1, keepdims=True)) return a / a.sum(-1, keepdims=True) def heads(m): return m.reshape(HEADS, HD, -1).transpose(0, 2, 1) def run(tokens, start, cache=None): t = len(tokens) x = (EMB[tokens] + POS[start:start + t]).T for i, p in enumerate(L): q, k, v = (heads(m) for m in np.split(p["qkv"] @ norm(x), 3)) if cache is not None: cache["K"][i, :, start:start + t] = k cache["V"][i, :, start:start + t] = v k, v = cache["K"][i, :, :start + t], cache["V"][i, :, :start + t] att = q @ k.transpose(0, 2, 1) / HD ** 0.5 # (heads, t, everything so far) mask = np.triu(np.ones((t, k.shape[1]), bool), k=k.shape[1] - t + 1) att = softmax(np.where(mask, -1e9, att)) x = x + p["out"] @ (att @ v).transpose(0, 2, 1).reshape(D, t) x = x + p["down"] @ gelu(p["up"] @ norm(x)) return HEAD @ norm(x)[:, -1] def new_cache(n): return dict(K=np.zeros((LAYERS, HEADS, n, HD), np.float32), V=np.zeros((LAYERS, HEADS, n, HD), np.float32)) def peak(fn): tracemalloc.start(); tracemalloc.reset_peak() t0 = time.perf_counter(); fn(); dt = time.perf_counter() - t0 _, pk = tracemalloc.get_traced_memory(); tracemalloc.stop() return pk, dt MB = 1e6 model = sum(a.nbytes for p in L for a in p.values()) print(f"weight tables on the 12 floors: {model/MB:.0f} MB, notebook: {2*LAYERS*HEADS*HD*4/1024:.0f} KB per token\n") print("peak scratch memory for ONE step (on top of the model and the notebook)") print(f" {'prompt':>7} {'notebook':>9} {'all at once':>12} {'in chunks of 512':>17}") CHUNK = 512 for n in [512, 1024, 2048, 4096]: prompt = list(rng.integers(0, VOCAB, n)) kv = 2 * LAYERS * HEADS * HD * 4 * n whole, t1 = peak(lambda: run(prompt, 0)) c = new_cache(n) def chunked(): for s in range(0, n, CHUNK): run(prompt[s:s + CHUNK], s, c) part, t2 = peak(chunked) print(f" {n:>7} {kv/MB:>6.0f} MB {whole/MB:>9.0f} MB {part/MB:>14.0f} MB")