What Deep Learning Actually Costs

import pandas as pd

rows = []
for name, parameters, tokens in [
        ('a small classifier', 5e6, 1e7),
        ('BERT-base', 110e6, 2.5e11),
        ('a 1B model', 1e9, 2e11),
        ('a 7B model, Chinchilla-optimal', 7e9, 1.4e11),
        ('a 70B model', 70e9, 1.4e12)]:
    flops = 6 * parameters * tokens
    rows.append({'model': name,
                 'parameters': f'{parameters:,.0f}',
                 'training tokens': f'{tokens:,.0f}',
                 'training FLOPs': f'{flops:.2e}'})
print(pd.DataFrame(rows).to_string(index=False))
print()
print('Those exponents are the whole story. A small classifier is')
print('3e14 FLOPs - a laptop afternoon. A 70B model is 5.9e23,')
print('which is two BILLION times more.')
print()
print('Deep learning costs are not on a continuum you can walk')
print('along. They are in bands separated by factors of a')
print('thousand, and knowing which band your problem is in is')
print('most of the planning.')

import pandas as pd

# Peak numbers from the vendor, and the fraction a real training
# run achieves. 35-50% is normal and hard-won; above 50% is good.
ACCELERATORS = [
    ('one laptop CPU core', 50e9, 0.60, 0.0),
    ('a consumer GPU (RTX-class)', 80e12, 0.35, 0.40),
    ('one A100 80GB', 312e12, 0.45, 2.50),
    ('one H100', 989e12, 0.45, 5.00),
    ('8 x H100 node', 7912e12, 0.40, 40.00),
]
rows = []
for name, peak, efficiency, hourly in ACCELERATORS:
    effective = peak * efficiency
    rows.append({'hardware': name,
                 'peak FLOP/s': f'{peak:.2e}',
                 'realistic': f'{effective:.2e}',
                 'usd/hour': f'{hourly:.2f}'})
print(pd.DataFrame(rows).to_string(index=False))
print()


def cost_of(flops, name):
    for hardware, peak, efficiency, hourly in ACCELERATORS:
        if hardware != name:
            continue
        seconds = flops / (peak * efficiency)
        return seconds / 3600, seconds / 3600 * hourly
    raise ValueError(name)


print('training cost of a 1-billion-parameter model on 200B tokens:')
flops = 6 * 1e9 * 2e11
for name, _, _, _ in ACCELERATORS[1:]:
    hours, dollars = cost_of(flops, name)
    print(f'  {name:<28} {hours:12,.0f} GPU-hours  ${dollars:12,.0f}')
print()
print('Note the last two rows. Eight H100s cost the same per FLOP')
print('as one, because the price scales with the hardware - what')
print('you buy with a bigger node is TIME, not money. A run that')
print('takes 42 days on one H100 takes five on eight of them for')
print('the same bill, minus whatever the communication overhead')
print('eats.')

import pandas as pd

H100_EFFECTIVE = 989e12 * 0.40
H100_HOURLY = 5.00

rows = []
for name, parameters, tokens in [
        ('fine-tune 7B with LoRA', 7e9, 5e7),
        ('full fine-tune 7B', 7e9, 1e9),
        ('train 7B from scratch', 7e9, 1.4e12),
        ('train 70B from scratch', 70e9, 1.4e12),
        ('train 400B from scratch', 400e9, 15e12)]:
    flops = 6 * parameters * tokens
    hours = flops / H100_EFFECTIVE / 3600
    rows.append({'job': name,
                 'FLOPs': f'{flops:.1e}',
                 'H100-hours': f'{hours:,.0f}',
                 'cost at $5/hr': f'${hours * H100_HOURLY:,.0f}',
                 'wall clock on 64 H100s': f'{hours / 64 / 24:,.1f} days'})
print(pd.DataFrame(rows).to_string(index=False))
print()
print('A LoRA fine-tune is $7 and about an hour. A full fine-tune')
print('is $147. Training 7B from scratch is $206,000 and 27 days')
print('on 64 H100s - and that is the SUCCESSFUL run, not the')
print('three failed attempts before it. A 400B model is $126')
print('million and would take 45 years on those 64 cards, which')
print('is why the organisations that do it own tens of thousands.')
print()
print('This is why the industry shape is what it is: a handful of')
print('organisations train base models, and everyone else adapts')
print('them. The gap between the third row and the first is four')
print('orders of magnitude, and no amount of cleverness closes it.')

import pandas as pd

rows = []
for name, parameters in [('125M', 125e6), ('1.3B', 1.3e9),
                         ('7B', 7e9), ('70B', 70e9)]:
    weights_fp32 = parameters * 4
    gradients = parameters * 4
    adam_state = parameters * 8          # two moments, fp32
    total = weights_fp32 + gradients + adam_state
    rows.append({'model': name,
                 'weights': f'{weights_fp32 / 1024 ** 3:8.1f} GiB',
                 'gradients': f'{gradients / 1024 ** 3:8.1f} GiB',
                 'Adam state': f'{adam_state / 1024 ** 3:8.1f} GiB',
                 'total, before activations': f'{total / 1024 ** 3:8.1f} GiB',
                 'fits in 80 GiB': 'yes' if total < 80 * 1024 ** 3 else 'NO'})
print(pd.DataFrame(rows).to_string(index=False))
print()
print('Adam costs twice what the weights do, because it keeps two')
print('moment estimates per parameter. With the gradients that is')
print('FOUR times the model, before a single activation is stored.')
print()
print('A 7B model needs 104 GiB to train with AdamW in fp32 and')
print('does not fit on an 80 GiB card. This is the single most')
print('common surprise in the field, and it is why every one of')
print('the following exists:')
print()
for technique, saving in [
        ('mixed precision (bf16 weights, fp32 master)',
         'cuts about 25% off the total'),
        ('gradient checkpointing',
         'trades activation memory for 30% more compute'),
        ('8-bit optimiser states',
         'removes three quarters of the Adam state'),
        ('ZeRO / FSDP sharding',
         'divides all three terms by the device count'),
        ('LoRA',
         'removes gradients and optimiser state almost entirely'),
        ('SGD with momentum instead of Adam',
         'halves the optimiser state')]:
    print(f'  {technique:<44} {saving}')

import pandas as pd

# Roughly: activations per layer per token, for a transformer
# storing what the backward pass needs.
layers, dimension = 32, 4096
per_token_per_layer = dimension * 16 * 2        # bf16

rows = []
for batch, length in [(1, 512), (1, 4096), (4, 4096),
                      (1, 32768), (8, 8192)]:
    activations = batch * length * layers * per_token_per_layer
    rows.append({'batch': batch, 'sequence length': length,
                 'tokens in flight': f'{batch * length:,}',
                 'activation memory': f'{activations / 1024 ** 3:8.1f} GiB'})
print(pd.DataFrame(rows).to_string(index=False))
print()
print('Same model throughout; only the batch and the context')
print('length change. One sequence of 512 tokens needs 2 GiB of')
print('activations; one of 32,768 needs 128 GiB, on top of the')
print('104 GiB the weights and optimiser already took.')
print()
print('Gradient checkpointing is the standard answer: store the')
print('activations at a few layer boundaries only and recompute')
print('the rest during the backward pass. It cuts activation')
print('memory by roughly the square root of the layer count and')
print('costs about 30% more compute - which is a good trade when')
print('the alternative is not running at all.')

import pandas as pd

# A 7B model serving a chat product.
PARAMETERS = 7e9
H100_EFFECTIVE = 989e12 * 0.30        # inference achieves less than training
H100_HOURLY = 5.00
TOKENS_PER_REQUEST = 600

flops_per_token = 2 * PARAMETERS
tokens_per_second = H100_EFFECTIVE / flops_per_token
print(f'one H100, 7B model: {tokens_per_second:,.0f} tokens/second '
      f'at full batch')
print(f'= {tokens_per_second / TOKENS_PER_REQUEST:,.0f} requests/second')
print()

requests_per_second = tokens_per_second / TOKENS_PER_REQUEST
cost_per_request = H100_HOURLY / 3600 / requests_per_second
print(f'cost per request: ${cost_per_request:.8f}')
print(f'cost per 1,000 requests: ${cost_per_request * 1000:.5f}')
print()

TRAINING_COST = 206000
rows = []
for daily in [1e4, 1e6, 1e8, 1e9]:
    annual = daily * 365 * cost_per_request
    rows.append({'requests/day': f'{daily:,.0f}',
                 'annual inference': f'${annual:,.0f}',
                 'daily inference': f'${daily * cost_per_request:,.0f}',
                 'days to equal the training run':
                     f'{TRAINING_COST / (daily * cost_per_request):,.0f}'})
print(f'against a one-off training cost of ${TRAINING_COST:,}')
print(pd.DataFrame(rows).to_string(index=False))
print()
print('Read the last column. At ten thousand requests a day,')
print('inference is a rounding error and the training run')
print('dominates for five centuries. At a hundred million a day it')
print('overtakes the whole training budget in 52 days - and then')
print('does it again every 52 days, forever.')
print()
print('That crossover is the number to find for your own product,')
print('because it decides where the engineering effort goes. Below')
print('it, optimise training. Above it, a week spent making the')
print('model four times cheaper to serve saves more than the')
print('entire training budget, repeatedly. That is why')
print('distillation and quantisation are production disciplines')
print('rather than research curiosities.')

import pandas as pd

# Cost per thousand requests, for the ladder of options.
H100_HOURLY = 5.00
H100_EFFECTIVE = 989e12 * 0.30
TOKENS = 600

rows = []
for name, parameters, speedup in [
        ('70B, fp16', 70e9, 1.0),
        ('70B, int8', 70e9, 2.0),
        ('7B, fp16', 7e9, 1.0),
        ('7B, int8', 7e9, 2.0),
        ('1B distilled, int8', 1e9, 2.0),
        ('a logistic regression', 1e4, 1.0)]:
    effective = H100_EFFECTIVE * speedup
    tokens_per_second = effective / (2 * parameters)
    requests_per_second = tokens_per_second / TOKENS
    per_thousand = H100_HOURLY / 3600 / requests_per_second * 1000
    rows.append({'model': name,
                 'requests/s per H100': f'{requests_per_second:,.1f}',
                 'per million requests': per_thousand * 1000})
table = pd.DataFrame(rows)
baseline = table.loc[0, 'per million requests']
table['relative to 70B fp16'] = [
    'baseline' if value == baseline else f'{baseline / value:,.0f}x cheaper'
    for value in table['per million requests']]
table['per million requests'] = [
    f'${value:,.2f}' if value >= 0.01 else f'${value:.6f}'
    for value in table['per million requests']]
print(table.to_string(index=False))
print()
print('Each step down is roughly an order of magnitude. A million')
print('requests cost $393 on a 70B model in fp16 and $2.81 on a')
print('distilled 1B in int8 - 140 times cheaper - and the only')
print('question is whether the quality still clears the bar, which')
print('is a measurement rather than an opinion.')
print()
print('The last row is there for perspective: 50,000 times')
print('cheaper again than the distilled model. If a logistic')
print('regression solves your problem, the serving cost is not a')
print('consideration at all, and no amount of optimising a')
print('transformer will get you near it.')

import pandas as pd

# Grid intensity varies by more than an order of magnitude, which
# is the largest single factor in a run's emissions.
GRIDS = [('Sweden / hydro', 25), ('France / nuclear', 60),
         ('US average', 390), ('Germany', 380),
         ('India / coal-heavy', 710)]

H100_WATTS = 700
PUE = 1.2                     # data-centre overhead
hours = 24000                 # a 7B training run on 64 GPUs for ~16 days

energy_kwh = H100_WATTS * PUE * hours / 1000
print(f'{hours:,} GPU-hours at {H100_WATTS} W and a PUE of {PUE}')
print(f'= {energy_kwh:,.0f} kWh')
print()
rows = []
for grid, intensity in GRIDS:
    kilograms = energy_kwh * intensity / 1000
    rows.append({'grid': grid, 'g CO2 per kWh': intensity,
                 'emissions': f'{kilograms:,.0f} kg CO2',
                 'equivalent flights, London-New York':
                     f'{kilograms / 1000:,.1f}'})
print(pd.DataFrame(rows).to_string(index=False))
print()
print('The same run emits 28 times more on a coal-heavy grid than')
print('on a hydroelectric one. Choosing the region is a larger')
print('lever than any optimisation in the training code, costs')
print('nothing, and takes one line of configuration.')
print()
print('The honest framing: one training run is a few flights. The')
print('aggregate across an industry is not negligible, and the')
print('inference total now exceeds the training total by a wide')
print('margin - which is another reason the cheap-to-serve model')
print('is the responsible choice as well as the profitable one.')

import pandas as pd

rows = [
    ('1. What is the baseline worth?',
     'logistic regression, or a rule',
     'free - and it sometimes ends the project here'),
    ('2. What accuracy does the product need?',
     'a number, agreed in writing',
     'decides which band you are in'),
    ('3. Can a pretrained model be adapted?',
     'feature extraction first',
     'hours, not weeks'),
    ('4. What will serving cost at scale?',
     'cost per 1,000 x expected volume',
     'the number that kills projects post-launch'),
    ('5. How much does a failed run cost?',
     'budget for three attempts',
     'the first run never works'),
    ('6. What is the retraining cadence?',
     'drift rate decides it',
     'a recurring cost, not a one-off'),
]
print(pd.DataFrame(rows, columns=['question', 'how to answer it',
                                  'why it matters']).to_string(index=False))
print()
print('Step 5 is the one left out of every plan. A training run')
print('fails on a bad learning rate, a corrupted shard or a')
print('silent preprocessing bug, and you find out after four')
print('days. Budget three runs and be pleased if you need two.')

import pandas as pd

# A step is data loading plus compute. If they are sequential,
# the GPU idles for the loading.
rows = []
for load_ms, compute_ms in [(5, 100), (50, 100), (100, 100), (200, 100)]:
    sequential = load_ms + compute_ms
    overlapped = max(load_ms, compute_ms)
    rows.append({'data load (ms)': load_ms,
                 'GPU compute (ms)': compute_ms,
                 'step, sequential': sequential,
                 'GPU utilisation': f'{compute_ms / sequential:.0%}',
                 'step, with prefetch': overlapped,
                 'speed-up from overlapping': f'{sequential / overlapped:.2f}x'})
print(pd.DataFrame(rows).to_string(index=False))
print()
print('Read the utilisation column. At 100 ms of loading against')
print('100 ms of compute the GPU is idle half the time, and')
print('overlapping the two - more worker processes, prefetch,')
print('pinned memory - gives a clean 2x with no change to the')
print('model at all.')
print()
print('The last row is the one to notice. At 200 ms of loading the')
print('GPU is idle two thirds of the time, and overlapping only')
print('gets 1.5x, because the loader is now the ceiling: the step')
print('cannot go below 200 ms however fast the GPU is. Past that')
print('point a faster GPU buys literally nothing, and the work is')
print('in the data pipeline.')
print()
print('That is almost always the first thing to fix, and it is')
print('almost never where people look. Check GPU utilisation')
print('before optimising anything else; if it is not above 80%,')
print('the model is not your problem.')

# Fill these in for your own project.
PARAMETERS = 7e9
TRAINING_TOKENS = 1e9                 # a fine-tune, not from scratch
REQUESTS_PER_DAY = 50000
TOKENS_PER_REQUEST = 500
GPU_HOURLY = 5.00
GPU_EFFECTIVE_FLOPS = 989e12 * 0.35

training_flops = 6 * PARAMETERS * TRAINING_TOKENS
training_hours = training_flops / GPU_EFFECTIVE_FLOPS / 3600
training_cost = training_hours * GPU_HOURLY

tokens_per_second = GPU_EFFECTIVE_FLOPS / (2 * PARAMETERS)
daily_tokens = REQUESTS_PER_DAY * TOKENS_PER_REQUEST
daily_gpu_hours = daily_tokens / tokens_per_second / 3600
daily_cost = daily_gpu_hours * GPU_HOURLY

print(f'TRAINING')
print(f'  {training_flops:.2e} FLOPs')
print(f'  {training_hours:,.1f} GPU-hours')
print(f'  ${training_cost:,.2f}  (budget 3x for failed runs: '
      f'${training_cost * 3:,.2f})')
print()
print(f'SERVING')
print(f'  {tokens_per_second:,.0f} tokens/second per GPU')
print(f'  {daily_gpu_hours:,.2f} GPU-hours/day')
print(f'  ${daily_cost:,.2f}/day   ${daily_cost * 365:,.2f}/year')
print()
print(f'inference passes training after '
      f'{training_cost / daily_cost:,.1f} days')
print(f'cost per request: ${daily_cost / REQUESTS_PER_DAY:.6f}')
print()
print('Three sanity checks on whatever you get:')
print(' - if the cost per request exceeds what the request earns,')
print('   the project is finished before it starts;')
print(' - if inference passes training inside a month, optimise')
print('   serving and not training;')
print(' - if the training cost is under a hundred dollars, stop')
print('   planning and run it - the estimate costs more than the')
print('   experiment.')