""" gpu_memory.py: how much GPU memory does an LLM server need? The worksheet, one line each: room for notebooks = GPU memory - model tables - everything else tokens that fit = room for notebooks / notebook bytes per token users that fit = tokens that fit / tokens per chat model tables = parameters x bytes per number notebook per token = 2 (K and V) x floors x notebook heads x head size x bytes per number everything else = the runtime, scratch space for the current step, and a safety margin. Rule of thumb used here: 2 GB + 5% of the card, per GPU (6 GB on an 80 GB card, the same allowance as Part 5). Measure yours: vLLM prints the KV cache memory it got when it starts. Model shapes are from each model's published config.json. 1 GB = 1e9 bytes, as in Part 5. """ GB = 1e9 def everything_else(gpu_gb): # per GPU return 2 * GB + 0.05 * gpu_gb * GB # name params floors notebook heads head size MODELS = { "Llama 3.1 8B": (8.03e9, 32, 8, 128), "Qwen2.5 7B": (7.62e9, 28, 4, 128), "Mixtral 8x7B": (46.7e9, 32, 8, 128), "Llama 3.1 70B": (70.6e9, 80, 8, 128), } GPUS = {"L4": 24, "L40S": 48, "H100": 80, "H200": 141} def tables(model, nbytes): return MODELS[model][0] * nbytes def per_token(model, nbytes): _, floors, kv_heads, head = MODELS[model] return 2 * floors * kv_heads * head * nbytes def users(model, gpu_gb, n_gpus=1, w_bytes=2, kv_bytes=2, ctx=8192): """Tables and notebooks are split evenly across n_gpus (tensor parallel); every GPU keeps its own 'everything else'.""" room = n_gpus * gpu_gb * GB - tables(model, w_bytes) - n_gpus * everything_else(gpu_gb) if room <= 0: return 0.0, room return room / per_token(model, kv_bytes) / ctx, room # ---- 1) The worksheet, filled in once ----------------------------------- m, g = "Llama 3.1 8B", "H100" u, room = users(m, GPUS[g]) print(f"1) worksheet: {m}, BF16, one {g}, 8K-token chats") print(f" GPU memory {GPUS[g]*GB/GB:7.2f} GB") print(f" - model tables 8.03B x 2 bytes {tables(m, 2)/GB:7.2f} GB") print(f" - everything else 2 GB + 5% {everything_else(GPUS[g])/GB:7.2f} GB") print(f" = room for notebooks {room/GB:7.2f} GB") print(f" / notebook per token {per_token(m, 2)/1024:7.0f} KB -> {room/per_token(m, 2)/1e3:,.0f}K tokens") print(f" / tokens per chat 8,192 -> {u:.0f} users") # ---- 2) Model tables by number size --------------------------------------- print("\n2) model tables (GB) by bytes per number") print(f" {'model':<16}{'BF16 (2 B)':>11}{'FP8 (1 B)':>11}{'INT4 (0.5 B)':>14}{'notebook/token':>16}") for name in MODELS: print(f" {name:<16}{tables(name, 2)/GB:>11.1f}{tables(name, 1)/GB:>11.1f}" f"{tables(name, 0.5)/GB:>14.1f}{per_token(name, 2)/1024:>13.0f} KB") # ---- 3) Same model, different GPUs and chat lengths --------------------- print("\n3) Llama 3.1 8B in BF16: users that fit on ONE GPU") ctxs = [4096, 8192, 32768, 131072] print(f" {'GPU':<11}{'room':>9}" + "".join(f"{c//1024:>8}K" for c in ctxs)) for gpu, gb in GPUS.items(): _, room = users(m, gb) row = "".join(f"{users(m, gb, ctx=c)[0]:>9.1f}" for c in ctxs) print(f" {gpu:<11}{room/GB:>6.1f} GB{row}") # ---- 4) A big model across several GPUs ---------------------------------- m70 = "Llama 3.1 70B" print(f"\n4) {m70} on H100s, 8K-token chats") print(f" {'GPUs':<6}{'tables per GPU':>16}{'room (all GPUs)':>17}{'users BF16':>12}{'users FP8 tables':>18}") for n in [1, 2, 4, 8]: ub, rb = users(m70, 80, n, w_bytes=2) uf, _ = users(m70, 80, n, w_bytes=1) rb_s = f"{rb/GB:.1f} GB" if rb > 0 else "doesn't fit" print(f" {n:<6}{tables(m70, 2)/n/GB:>13.1f} GB{rb_s:>17}{ub:>12.1f}{uf:>18.1f}") # ---- 5) Plan backwards: what do I need for N users? --------------------- print("\n5) smallest setup for 50 users at 8K (BF16 notebooks)") for name, w in [("Llama 3.1 8B", 2), ("Qwen2.5 7B", 2), ("Mixtral 8x7B", 2), ("Llama 3.1 70B", 2), ("Llama 3.1 70B", 1)]: tag = "BF16" if w == 2 else "FP8" for gpu in ["L4", "H100"]: for n in [1, 2, 4, 8]: if users(name, GPUS[gpu], n, w_bytes=w)[0] >= 50: print(f" {name + ' ' + tag + ' tables':<28} {gpu:<5}: {n} GPU{'s' if n > 1 else ''}" f" ({users(name, GPUS[gpu], n, w_bytes=w)[0]:.0f} users fit)") break else: print(f" {name + ' ' + tag + ' tables':<28} {gpu:<5}: more than 8") # ---- 6) Four small GPUs: one shared copy, or four copies? ----------------- shared, _ = users(m, GPUS["L4"], 4) one, _ = users(m, GPUS["L4"], 1) print("\n6) Llama 3.1 8B BF16 on four L4s, 8K-token chats") print(f" one copy split across all 4 (tensor parallel): {shared:5.1f} users") print(f" four separate copies, one per GPU: {4*one:5.1f} users ({one:.1f} each)") # ---- 7) Check the "everything else" line on your own server --------------- print("\n7) Llama 3.1 8B BF16, one H100, 8K chats, with different 'everything else' allowances") for ee in [6, 10]: room = 80 * GB - tables(m, 2) - ee * GB print(f" everything else {ee:>2} GB -> room {room/GB:5.2f} GB -> {room/per_token(m, 2)/8192:4.1f} users")