A network with more parameters than training examples can memorise the training set exactly and learn nothing. Regularisation is every technique that stops it, and the first thing to understand is that overfitting is not a bug in the network - it is the network doing precisely what you asked, which was to minimise the training loss.
Seeing it happen
Overfitting is a gap between two curves, so the only way to know you have it is to plot both.
import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
rng = np.random.default_rng(0)
# Deliberately hard: 400 examples, 60 features, only 8 of them real.
X, y = make_classification(n_samples=400, n_features=60, n_informative=8,
n_redundant=0, n_repeated=0, class_sep=0.7,
flip_y=0.05, random_state=0)
y = y.reshape(-1, 1).astype(float)
X = (X - X.mean(axis=0)) / X.std(axis=0)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.4, random_state=0)
def sigmoid(z):
return 1 / (1 + np.exp(-np.clip(z, -60, 60)))
def loss_of(predicted, actual):
p = np.clip(predicted, 1e-9, 1 - 1e-9)
return float(-np.mean(actual * np.log(p) + (1 - actual) * np.log(1 - p)))
def train(epochs=400, width=256, rate=0.05, weight_decay=0.0,
dropout=0.0, seed=0, record=None):
generator = np.random.default_rng(seed)
W1 = generator.normal(0, np.sqrt(2 / X_tr.shape[1]), (X_tr.shape[1], width))
b1 = np.zeros(width)
W2 = generator.normal(0, np.sqrt(2 / width), (width, 1))
b2 = np.zeros(1)
history = []
for epoch in range(1, epochs + 1):
z1 = X_tr @ W1 + b1
a1 = np.maximum(0, z1)
if dropout > 0:
mask = (generator.uniform(0, 1, a1.shape) > dropout) / (1 - dropout)
a1_dropped = a1 * mask
else:
mask = None
a1_dropped = a1
a2 = sigmoid(a1_dropped @ W2 + b2)
delta2 = (a2 - y_tr) / len(X_tr)
gW2 = a1_dropped.T @ delta2 + weight_decay * W2
gb2 = delta2.sum(axis=0)
delta1 = (delta2 @ W2.T)
if mask is not None:
delta1 = delta1 * mask
delta1 = delta1 * (z1 > 0)
gW1 = X_tr.T @ delta1 + weight_decay * W1
gb1 = delta1.sum(axis=0)
W1 -= rate * gW1
b1 -= rate * gb1
W2 -= rate * gW2
b2 -= rate * gb2
if record is not None and (epoch in record):
train_p = sigmoid(np.maximum(0, X_tr @ W1 + b1) @ W2 + b2)
test_p = sigmoid(np.maximum(0, X_te @ W1 + b1) @ W2 + b2)
history.append({'epoch': epoch,
'train loss': round(loss_of(train_p, y_tr), 4),
'test loss': round(loss_of(test_p, y_te), 4),
'train acc': round(float(((train_p > 0.5) == y_tr).mean()), 3),
'test acc': round(float(((test_p > 0.5) == y_te).mean()), 3)})
train_p = sigmoid(np.maximum(0, X_tr @ W1 + b1) @ W2 + b2)
test_p = sigmoid(np.maximum(0, X_te @ W1 + b1) @ W2 + b2)
return {
'train loss': round(loss_of(train_p, y_tr), 4),
'test loss': round(loss_of(test_p, y_te), 4),
'train acc': round(float(((train_p > 0.5) == y_tr).mean()), 3),
'test acc': round(float(((test_p > 0.5) == y_te).mean()), 3),
'weight norm': round(float(np.sqrt((W1 ** 2).sum() + (W2 ** 2).sum())), 1),
'history': pd.DataFrame(history),
}
marks = {1, 25, 50, 100, 200, 400, 800, 1600}
result = train(epochs=1600, record=marks)
print(f'{len(X_tr)} training examples, '
f'{X_tr.shape[1] * 256 + 256 + 256 + 1:,} parameters')
print(result['history'].to_string(index=False))
print()
print('The training loss falls to almost nothing and the training')
print('accuracy reaches 1.000: the network has memorised 240 rows.')
print('The test loss stopped falling long before that and then')
print('turned upwards. The epoch where the two curves part is the')
print('only thing in this table worth knowing.')
Open this example in the editor and change it. Nothing you type is saved.
The shape to recognise: training loss down, validation loss up. Not "validation loss higher than training loss" - that is normal and expected - but rising.
Early stopping
The cheapest regularisation is to stop at the turn. It costs nothing and is almost always worth having.
import numpy as np
# A validation curve with the usual shape: down, flat, up, with
# noise on top so the decision is not trivial.
curve = np.array([0.690, 0.610, 0.550, 0.500, 0.470, 0.452, 0.458,
0.455, 0.449, 0.430, 0.419, 0.424, 0.431, 0.448,
0.465, 0.490, 0.520])
def early_stop(curve, patience):
best, best_epoch, waited = float('inf'), 0, 0
for epoch, value in enumerate(curve, 1):
if value < best:
best, best_epoch, waited = value, epoch, 0
else:
waited += 1
if waited >= patience:
return epoch, best_epoch, best
return len(curve), best_epoch, best
print('patience stopped at best epoch best loss')
for patience in [1, 3, 5, 10]:
stopped, best_epoch, best = early_stop(curve, patience)
print(f'{patience:8d} {stopped:10d} {best_epoch:10d} {best:9.3f}')
print()
print('true minimum: epoch', int(curve.argmin() + 1), 'at', curve.min())
print()
print('A patience of 1 quits at epoch 7, on a 0.006 wobble, and')
print('keeps 0.452 - it never sees the real minimum of 0.419 four')
print('epochs later. Patience 3 and above find it. Patience 10')
print('spends a few extra epochs and loses nothing, because what')
print('you keep is the best checkpoint rather than the last one.')
print()
print('That is the part people get wrong: early stopping means')
print('RESTORING the best weights. Stopping and keeping wherever')
print('you happened to stop throws away the whole benefit, and')
print('every framework makes it a flag you have to set.')
Open this example in the editor and change it. Nothing you type is saved.
Weight decay
Add a penalty proportional to the size of the weights, and the optimiser has to justify every large weight with a reduction in loss. Large weights are how a network builds a sharp, wiggly decision boundary, so penalising them produces a smoother one.
import numpy as np
import pandas as pd
# L1 and L2 on the same weight vector, and what each does to it.
weights = np.array([3.0, 0.4, -0.1, 0.02, -2.5, 0.001])
print('L2 penalty (sum of squares) :', round(float((weights ** 2).sum()), 4))
print('L1 penalty (sum of absolute):', round(float(np.abs(weights).sum()), 4))
print()
print('The gradient of each penalty is what matters:')
def row(values):
return ' '.join(f'{v:8.3f}' for v in values)
print(' weight ', row(weights))
print(' L2 gradient ', row(2 * weights), ' - proportional to w')
print(' L1 gradient ', row(np.sign(weights)), ' - constant')
print()
print('L2 pushes a weight of 3.0 a hundred and fifty times harder')
print('than a weight of 0.02, so it shrinks everything towards')
print('zero and leaves nothing exactly at zero. L1 pushes both the')
print('same, so small weights reach zero and stay - which is why')
print('L1 produces sparsity and L2 produces smoothness.')
Open this example in the editor and change it. Nothing you type is saved.
Then the measurement, on the overfitting network from the first example.
import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
X, y = make_classification(n_samples=400, n_features=60, n_informative=8,
n_redundant=0, class_sep=0.7, flip_y=0.05,
random_state=0)
y = y.reshape(-1, 1).astype(float)
X = (X - X.mean(axis=0)) / X.std(axis=0)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.4, random_state=0)
def sigmoid(z):
return 1 / (1 + np.exp(-np.clip(z, -60, 60)))
def loss_of(p, actual):
p = np.clip(p, 1e-9, 1 - 1e-9)
return float(-np.mean(actual * np.log(p) + (1 - actual) * np.log(1 - p)))
def train(weight_decay=0.0, dropout=0.0, epochs=1600, width=256,
rate=0.05, seed=0):
generator = np.random.default_rng(seed)
W1 = generator.normal(0, np.sqrt(2 / 60), (60, width))
b1 = np.zeros(width)
W2 = generator.normal(0, np.sqrt(2 / width), (width, 1))
b2 = np.zeros(1)
for _ in range(epochs):
z1 = X_tr @ W1 + b1
a1 = np.maximum(0, z1)
if dropout > 0:
mask = (generator.uniform(0, 1, a1.shape) > dropout) / (1 - dropout)
else:
mask = np.ones_like(a1)
a1d = a1 * mask
a2 = sigmoid(a1d @ W2 + b2)
delta2 = (a2 - y_tr) / len(X_tr)
gW2 = a1d.T @ delta2 + weight_decay * W2
delta1 = ((delta2 @ W2.T) * mask) * (z1 > 0)
gW1 = X_tr.T @ delta1 + weight_decay * W1
W1 -= rate * gW1
b1 -= rate * delta1.sum(axis=0)
W2 -= rate * gW2
b2 -= rate * delta2.sum(axis=0)
train_p = sigmoid(np.maximum(0, X_tr @ W1 + b1) @ W2 + b2)
test_p = sigmoid(np.maximum(0, X_te @ W1 + b1) @ W2 + b2)
return {'train loss': round(loss_of(train_p, y_tr), 4),
'test loss': round(loss_of(test_p, y_te), 4),
'train acc': round(float(((train_p > 0.5) == y_tr).mean()), 3),
'test acc': round(float(((test_p > 0.5) == y_te).mean()), 3),
'weight norm': round(float(np.sqrt((W1 ** 2).sum() + (W2 ** 2).sum())), 1)}
rows = []
for decay in [0.0, 0.001, 0.01, 0.1, 1.0, 10.0]:
row = {'weight decay': decay}
row.update(train(weight_decay=decay))
rows.append(row)
print(pd.DataFrame(rows).to_string(index=False))
print()
print('Read the weight norm and the train/test gap down the table.')
print('At zero decay the network memorises - perfect training')
print('accuracy, a weight norm of 23, and the worst test loss in')
print('the table. Decay shrinks the norm and the test loss nearly')
print('halves, from 1.02 to 0.56.')
print()
print('Two honest observations. First, the test ACCURACY barely')
print('moves - 0.700 to 0.713 - while the loss improves by a')
print('factor of two. Decay mostly stopped the network being')
print('confidently wrong, which the loss sees and accuracy cannot.')
print('Second, past 0.1 it goes too far: at 1.0 and above the')
print('weights are crushed to zero, training accuracy collapses to')
print('0.512, and the model predicts the base rate for everyone.')
print('Falling TRAINING accuracy is the signature of too much')
print('regularisation, as against too little.')
Open this example in the editor and change it. Nothing you type is saved.
Dropout
During training, zero a random fraction of the activations in a layer. The network cannot rely on any single unit, so it has to spread the representation out.
import numpy as np
rng = np.random.default_rng(1)
activations = np.array([[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]])
p = 0.5
# Inverted dropout: zero half, and divide the survivors by (1-p) so
# the expected sum is unchanged. This is what every framework does,
# and it is why inference needs no rescaling at all.
mask = (rng.uniform(0, 1, activations.shape) > p) / (1 - p)
print('activations ', activations.ravel())
print('mask ', mask.ravel())
print('after dropout ', (activations * mask).ravel())
print()
print('sum before ', round(float(activations.sum()), 2))
print('sum after ', round(float((activations * mask).sum()), 2))
# One draw is noisy; the expectation is what matters.
draws = np.array([(activations * ((rng.uniform(0, 1, activations.shape) > p)
/ (1 - p))).sum()
for _ in range(20000)])
print(f'average over 20,000 draws {draws.mean():.3f}'
f' (target {activations.sum():.1f})')
print()
print('Each individual draw is wrong by a lot. The average is')
print('right, which is the whole trick: training sees an unbiased')
print('but noisy version, inference sees the exact value with')
print('dropout switched off.')
Open this example in the editor and change it. Nothing you type is saved.
Three things about dropout that are not obvious:
- It must be off at inference. In PyTorch that is
model.eval(). Leaving it on makes every prediction random, and because the average is still correct, the bug shows up as unstable predictions rather than wrong ones - which is much harder to notice. - It fights batch normalisation. Both inject noise, both change behaviour between train and eval, and together they tend to be worse than either. Modern convolutional architectures use batch norm and little or no dropout.
- Rates differ by layer type. 0.5 on a wide fully-connected layer is standard. 0.1 to 0.3 is typical in a transformer. On a convolutional layer, ordinary dropout is usually a mistake - neighbouring pixels are correlated, so zeroing individual activations removes almost no information;
Dropout2d, which zeroes whole channels, is the correct form.
import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
X, y = make_classification(n_samples=400, n_features=60, n_informative=8,
n_redundant=0, class_sep=0.7, flip_y=0.05,
random_state=0)
y = y.reshape(-1, 1).astype(float)
X = (X - X.mean(axis=0)) / X.std(axis=0)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.4, random_state=0)
def sigmoid(z):
return 1 / (1 + np.exp(-np.clip(z, -60, 60)))
def loss_of(p, actual):
p = np.clip(p, 1e-9, 1 - 1e-9)
return float(-np.mean(actual * np.log(p) + (1 - actual) * np.log(1 - p)))
def train(dropout=0.0, weight_decay=0.0, epochs=1600, width=256, rate=0.05, seed=0):
generator = np.random.default_rng(seed)
W1 = generator.normal(0, np.sqrt(2 / 60), (60, width))
b1 = np.zeros(width)
W2 = generator.normal(0, np.sqrt(2 / width), (width, 1))
b2 = np.zeros(1)
best_loss, best_accuracy, best_epoch = float('inf'), 0.0, 0
for epoch in range(1, epochs + 1):
z1 = X_tr @ W1 + b1
a1 = np.maximum(0, z1)
mask = ((generator.uniform(0, 1, a1.shape) > dropout) / (1 - dropout)
if dropout > 0 else np.ones_like(a1))
a1d = a1 * mask
a2 = sigmoid(a1d @ W2 + b2)
delta2 = (a2 - y_tr) / len(X_tr)
gW2 = a1d.T @ delta2 + weight_decay * W2
delta1 = ((delta2 @ W2.T) * mask) * (z1 > 0)
W1 -= rate * (X_tr.T @ delta1 + weight_decay * W1)
b1 -= rate * delta1.sum(axis=0)
W2 -= rate * gW2
b2 -= rate * delta2.sum(axis=0)
# Dropout OFF for evaluation - this is what model.eval() does.
epoch_test = sigmoid(np.maximum(0, X_te @ W1 + b1) @ W2 + b2)
epoch_loss = loss_of(epoch_test, y_te)
if epoch_loss < best_loss:
best_loss = epoch_loss
best_accuracy = float(((epoch_test > 0.5) == y_te).mean())
best_epoch = epoch
train_p = sigmoid(np.maximum(0, X_tr @ W1 + b1) @ W2 + b2)
test_p = sigmoid(np.maximum(0, X_te @ W1 + b1) @ W2 + b2)
return {'train acc': round(float(((train_p > 0.5) == y_tr).mean()), 3),
'final test loss': round(loss_of(test_p, y_te), 4),
'final test acc': round(float(((test_p > 0.5) == y_te).mean()), 3),
'best test loss': round(best_loss, 4),
'best test acc': round(best_accuracy, 3),
'best epoch': best_epoch}
rows = []
for name, options in [('nothing', {}),
('dropout 0.5', {'dropout': 0.5}),
('width 16 instead of 256', {'width': 16}),
('weight decay 0.05', {'weight_decay': 0.05}),
('weight decay 0.1', {'weight_decay': 0.1}),
('dropout 0.5 + decay 0.05',
{'dropout': 0.5, 'weight_decay': 0.05})]:
row = {'technique': name}
row.update(train(**options))
rows.append(row)
table = pd.DataFrame(rows)
print(table.to_string(index=False))
print()
print('best final test loss:',
table.loc[table['final test loss'].idxmin(), 'technique'])
print('best loss at ANY epoch (i.e. with early stopping):',
table.loc[table['best test loss'].idxmin(), 'technique'])
print()
print('Three things this table says that a tutorial will not.')
print()
print('Early stopping alone is the biggest single win available:')
print('the unregularised run touches 0.611 at epoch 64 and then')
print('spends 1,500 more epochs climbing to 1.024. Keeping the')
print('checkpoint from epoch 64 costs nothing and recovers 40% of')
print('the loss.')
print()
print('Dropout does almost nothing here. It improves the best')
print('loss by 0.02 and makes the final loss worse. Dropout is a')
print('technique for wide layers with plenty of data; on 240')
print('tabular rows it is the wrong tool, and no amount of tuning')
print('the rate changes that.')
print()
print('A smaller network is not automatically a regularised one.')
print('Width 16 still reaches 1.000 training accuracy - 60')
print('features is enough to memorise 240 rows at any width in')
print('this range - so capacity was never the binding constraint.')
print()
print('Weight decay wins, and combining it with dropout adds')
print('nothing on top. Try the cheap ones first and measure, in')
print('that order.')
Open this example in the editor and change it. Nothing you type is saved.
Data augmentation, and why it beats the others
Every technique so far constrains the model. Augmentation changes the data, by generating new training examples that are different pixels but the same label. It is usually the single most effective regulariser available, because it adds information about what the label is invariant to - information the model has no other way to obtain.
import numpy as np
import pandas as pd
from scipy import ndimage
from sklearn.datasets import load_digits
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
digits = load_digits() # 8x8 grey-scale, 0-16
train_images, test_images, train_labels, test_labels = train_test_split(
digits.images, digits.target, test_size=0.4, random_state=0,
stratify=digits.target)
# Deliberately starved: 40 training images for ten classes.
train_images, train_labels = train_images[:40], train_labels[:40]
test_flat = test_images.reshape(len(test_images), -1) / 16.0
def rotate_by(angle):
def transform(image):
return ndimage.rotate(image, angle, reshape=False, order=1,
mode='constant', cval=0.0)
return transform
def identity(image):
return image
def mirror(image):
return image[:, ::-1]
def evaluate(transforms):
images, labels = [], []
for image, label in zip(train_images, train_labels):
for transform in transforms:
images.append(transform(image))
labels.append(label)
flat = np.array(images).reshape(len(images), -1) / 16.0
model = LogisticRegression(max_iter=4000).fit(flat, labels)
return len(flat), model.score(flat, labels), model.score(test_flat, test_labels)
rows = []
for name, transforms in [
('originals only', [identity]),
('+ rotations of 12 degrees', [identity, rotate_by(12), rotate_by(-12)]),
('+ rotations of 8 and 16', [identity, rotate_by(8), rotate_by(-8),
rotate_by(16), rotate_by(-16)]),
('+ horizontal mirrors', [identity, mirror])]:
examples, train_score, test_score = evaluate(transforms)
rows.append({'training set': name, 'examples': examples,
'train acc': round(train_score, 3),
'test acc': round(test_score, 3)})
print(pd.DataFrame(rows).to_string(index=False))
print()
print('Rotating the digits took the test accuracy from 0.720 to')
print('0.776 - five and a half points, from no new photographs.')
print('Mirroring them took it DOWN to 0.661, which is worse than')
print('having done nothing at all.')
print()
print('Same dataset, same model, same amount of extra data. The')
print('difference is that a rotated 7 is a 7 and a mirrored 7 is')
print('not. Augmentation adds the information that the label does')
print('not change under some transformation; choose the wrong')
print('transformation and you have added the opposite.')
Open this example in the editor and change it. Nothing you type is saved.
The rule is one question: would a human give this the same label? A horizontally flipped cat is a cat, so flipping is valid for animal photographs. A flipped 2 is not a 2. A rotated chest X-ray is not a realistic chest X-ray, so large rotations are invalid in medical imaging even though they are routine elsewhere. Getting this wrong teaches the model that two different things are the same, and the table above is what that costs.
Note also that the five extra rotations bought nothing over the first two - 0.775 against 0.776. Augmentation adds information about an invariance, and once the model has learned that rotation does not matter, further rotations are repetition.
The complete list, in the order to try them
- More data. Not a technique, but it dominates everything below and is sometimes available.
- Augmentation. The best regulariser when the invariances are known.
- Early stopping. Free, and nearly always correct.
- Weight decay. One number;
0.01with AdamW is a reasonable start. - A smaller model. Underrated. If 240 examples will not support 16,000 parameters, a smaller network is not a compromise, it is the right model.
- Dropout. Effective on fully-connected layers; mostly superseded by batch norm in vision.
- Label smoothing. Replace a target of 1.0 with 0.9. Stops the network becoming absurdly confident and improves calibration.
- Ensembling. Train five models, average their predictions. Reliably the best result, at five times the cost.
import numpy as np
truth = np.array([0.0, 0.0, 1.0, 0.0])
def cross_entropy(predicted, target):
p = np.clip(predicted, 1e-9, 1)
return float(-(target * np.log(p)).sum())
# What a confident network pushes towards under each target.
predicted = np.array([0.001, 0.001, 0.997, 0.001])
smoothed = truth * 0.9 + 0.1 / len(truth)
print('hard target ', truth)
print('smoothed target', smoothed.round(4))
print()
print(f'loss against hard target {cross_entropy(predicted, truth):.4f}')
print(f'loss against smoothed target {cross_entropy(predicted, smoothed):.4f}')
print()
# The mechanism is the gradient with respect to the LOGIT, which for
# softmax plus cross-entropy is simply (predicted - target).
print('p(true class) gradient, hard target gradient, smoothed')
for p in [0.5, 0.8, 0.9, 0.925, 0.95, 0.99, 0.9999]:
print(f'{p:13} {p - 1.0:21.4f} {p - 0.925:+18.4f}')
print()
print('Against a hard target the gradient is negative at every')
print('value of p: it keeps pushing the logit up forever, because')
print('p = 1 is never reached. That is how a network ends up')
print('99.99% confident about a blurry photograph.')
print()
print('Against a smoothed target it crosses zero at p = 0.925 and')
print('then turns POSITIVE, pushing the logit back down. The model')
print('has a confidence it is supposed to stop at, which is the')
print('whole of what label smoothing does.')
Open this example in the editor and change it. Nothing you type is saved.
Failure modes
| Symptom | Cause | Fix |
|---|---|---|
| Training accuracy 1.00, test accuracy poor | Classic overfitting | Augment, decay, early stop, smaller model |
| Both accuracies poor, training loss still falling slowly | Underfitting, or too much regularisation | Reduce decay and dropout; train longer; more capacity |
| Validation loss rises while validation accuracy holds | The model is getting more confident about its mistakes | Label smoothing; stop earlier |
| Validation better than training | Dropout and batch norm active in training, not in eval - usually fine | Compare like with like before worrying |
| Weight decay seems to do nothing | Applied only to the last layer, or the optimiser is Adam rather than AdamW | Check what the parameter group covers |
| Augmentation made it worse | An augmentation that changes the true label | Look at twenty augmented examples with their labels |
| Great offline, worse in production | The validation set leaked, or the data has shifted | Re-split by time or by group |
The fourth row deserves its own sentence, because it sends people chasing a non-existent bug: validation loss below training loss is normal. Training loss is measured with dropout on and averaged over the epoch while the weights were still improving; validation loss is measured after the epoch, with dropout off. Two different measurements of two different things.
Check your understanding
Regularisation should cost you training accuracy. If a technique improves the test score without reducing the training score, it is not regularising - it is fixing something else, and you have mislabelled what happened.
import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
X, y = make_classification(n_samples=300, n_features=80, n_informative=10,
n_redundant=0, class_sep=0.6, flip_y=0.1,
random_state=1)
X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=0.5, random_state=0)
rows = []
for C in [10.0, 1.0, 0.1, 0.01, 0.001, 0.0003, 0.0001]:
model = LogisticRegression(C=C, max_iter=8000).fit(X_tr, y_tr)
rows.append({'C (inverse strength)': C,
'penalty strength': round(1 / C, 4),
'train acc': round(model.score(X_tr, y_tr), 3),
'test acc': round(model.score(X_te, y_te), 3),
'weight norm': round(float(np.linalg.norm(model.coef_)), 2)})
table = pd.DataFrame(rows)
print(table.to_string(index=False))
print()
best = table.loc[table['test acc'].idxmax()]
print(f'best test accuracy {best["test acc"]} at penalty strength '
f'{best["penalty strength"]}')
print(f'its training accuracy is {best["train acc"]}, down from '
f'{table["train acc"].max()}')
print()
print('Training accuracy falls monotonically as the penalty rises,')
print('from 1.000 to 0.527. Test accuracy rises from 0.447 to')
print('0.720 and then falls back to 0.473 - an interior optimum,')
print('found by sweeping five decades of penalty strength.')
print()
print('The best model on the test set is NOT the best model on the')
print('training set, and it never will be. That is not a flaw in')
print('the experiment, it is the entire reason regularisation')
print('works: you are trading training accuracy, which you do not')
print('care about, for test accuracy, which you do.')
Open this example in the editor and change it. Nothing you type is saved.