What It Is Good At, and Where It Lies

# Six failure classes, each with the mechanism rather than a warning.
failures = [
    ('Specific facts not in the prompt',
     'Parametric memory is lossy compression; the model has no way to',
     'tell a recalled fact from a plausible one'),
    ('Arithmetic and counting',
     'Digits and characters are hidden inside tokens; there is no',
     'register to carry a value in'),
    ('Anything after the training cutoff',
     'The weights were frozen; the model cannot know that it does not know',
     ''),
    ('Its own confidence',
     'Fluency comes from the same distribution as content, so a guess',
     'is phrased exactly like a fact'),
    ('Long chains of exact reasoning',
     'Each token is sampled; a per-step error rate compounds',
     'multiplicatively'),
    ('Saying "I do not know"',
     'The training data contains very few examples of refusing, and',
     'preference tuning rewards answers'),
]
for title, line, more in failures:
    print(f'* {title}')
    print(f'    {line}')
    if more:
        print(f'    {more}')
    print()

# The compounding in the fifth row is the one with arithmetic.
print('a task needing N exact steps, at 98% accuracy per step:')
for steps in [1, 5, 10, 25, 50]:
    print(f'  {steps:3d} steps -> {0.98 ** steps:6.1%} chance of a correct answer')
print()
print('Ten steps at 98% is 82%. Fifty is 36%. This is why agents')
print('that chain many calls fail in ways that look random: the')
print('per-step accuracy is high and the end-to-end accuracy is')
print('not, and the only fix is fewer steps or a check after each')
print('one.')

import numpy as np
import torch
import torch.nn as nn

torch.manual_seed(0)
torch.set_num_threads(4)
rng = np.random.default_rng(0)

# Training data: a product catalogue. Each product has ONE price,
# and the model sees it many times.
PRODUCTS = {'widget': '12', 'gadget': '34', 'sprocket': '56',
            'flange': '78'}
lines = []
for _ in range(4000):
    name = list(PRODUCTS)[rng.integers(len(PRODUCTS))]
    lines.append(f'the {name} costs {PRODUCTS[name]} pounds .')
words = ' '.join(lines).split()
vocabulary = sorted(set(words) | {'doohickey'})   # a word with no price
to_id = {word: index for index, word in enumerate(vocabulary)}
data = torch.tensor([to_id[word] for word in words])
BLOCK = 8


class Model(nn.Module):
    def __init__(self, size, width=64):
        super().__init__()
        self.token = nn.Embedding(size, width)
        self.position = nn.Embedding(BLOCK, width)
        layer = nn.TransformerEncoderLayer(width, 4, 4 * width, dropout=0.0,
                                           activation='gelu', batch_first=True,
                                           norm_first=True)
        self.body = nn.TransformerEncoder(layer, 2, enable_nested_tensor=False)
        self.norm = nn.LayerNorm(width)
        self.head = nn.Linear(width, size)

    def forward(self, ids):
        length = ids.shape[1]
        hidden = self.token(ids) + self.position(torch.arange(length))
        mask = nn.Transformer.generate_square_subsequent_mask(length)
        return self.head(self.norm(
            self.body(hidden, mask=mask, is_causal=True)))


model = Model(len(vocabulary))
optimiser = torch.optim.AdamW(model.parameters(), lr=3e-3)
loss_fn = nn.CrossEntropyLoss()
for _ in range(700):
    start = torch.randint(0, len(data) - BLOCK - 1, (64,))
    inputs = torch.stack([data[i:i + BLOCK] for i in start])
    targets = torch.stack([data[i + 1:i + BLOCK + 1] for i in start])
    optimiser.zero_grad()
    loss_fn(model(inputs).reshape(-1, len(vocabulary)),
            targets.reshape(-1)).backward()
    optimiser.step()
model.eval()


def answer(prompt):
    ids = torch.tensor([[to_id[word] for word in prompt.split()]])
    with torch.no_grad():
        probabilities = torch.softmax(model(ids)[0, -1], dim=-1)
    best = int(probabilities.argmax())
    return vocabulary[best], float(probabilities[best])


print('prices in the training data:', PRODUCTS)
print()
print('asking about products it has seen:')
for name in PRODUCTS:
    token, confidence = answer(f'the {name} costs')
    correct = 'correct' if token == PRODUCTS[name] else 'WRONG'
    print(f'  "the {name} costs" -> {token!r} '
          f'(confidence {confidence:.3f})  {correct}')
print()
print('asking about a product it has NEVER seen:')
token, confidence = answer('the doohickey costs')
print(f'  "the doohickey costs" -> {token!r} (confidence {confidence:.3f})')
print()
print('There is no price for a doohickey anywhere in the training')
print('data. The model produced one anyway.')
print()
print('Notice what the confidence did. It fell from 0.999 to')
print('0.508 - so on a model this small, with a vocabulary of 24')
print('tokens, uncertainty IS visible in the number. That is the')
print('good case, and it is the case you do not get in')
print('production: with a vocabulary of 100,000 and a training')
print('set of trillions of tokens, the equivalent drop is small,')
print('noisy and indistinguishable from ordinary variation.')
print()
print('And 0.508 would pass most confidence thresholds anyone')
print('writes.')
print()
print('Look at why. The position after "costs" is always a price')
print('in the training data, so the distribution there puts all')
print('its mass on the four prices it knows. "No price exists" is')
print('not one of the options, because the vocabulary has no token')
print('for it and the data has no example of it.')
print()
distribution = torch.softmax(
    model(torch.tensor([[to_id[w] for w in 'the doohickey costs'.split()]]))[0, -1],
    dim=-1)
print('the full distribution after "the doohickey costs":')
for index in torch.topk(distribution, 6).indices:
    print(f'  {vocabulary[int(index)]:<12} {float(distribution[index]):.4f}')
print()
print('THIS is the detectable signature, rather than the top-1')
print('number: the mass is spread over the four prices the model')
print('knows, and nothing else gets more than 0.0003. The model')
print('is not uncertain about what KIND of thing goes here - it is')
print('certain it is one of these four prices and has no basis for')
print('choosing between them.')
print()
print('A real pipeline can see that shape in the log-probabilities')
print('and route the request to retrieval instead of answering.')
print()
print('That is hallucination with the lid off. Not a glitch, not a')
print('lie, and nothing a better model of this architecture fixes:')
print('the model was asked for the most likely token and it gave')
print('one.')

# Two prompts for the same question, differing only in whether
# "not found" is an available answer.
WITHOUT = """Answer the question.

Question: What does a doohickey cost?"""

WITH = """Answer the question using ONLY the catalogue below. If the
answer is not in the catalogue, reply exactly: NOT IN CATALOGUE.

Catalogue:
  widget    12 pounds
  gadget    34 pounds
  sprocket  56 pounds
  flange    78 pounds

Question: What does a doohickey cost?"""

for label, prompt in [('no escape hatch', WITHOUT), ('with one', WITH)]:
    print(f'--- {label} ---')
    print(prompt)
    print()

print('The second prompt does three things, and all three matter:')
print(' - it puts the facts in the context, so recall is not')
print('   needed;')
print(' - it bounds the source, so the model is not drawing on')
print('   parametric memory at all;')
print(' - it supplies an exact token for "absent", which gives the')
print('   distribution somewhere to put its mass other than a')
print('   number.')
print()
print('The third is the one people leave out, and it is the one')
print('that turns a confident wrong answer into a usable signal.')
print('A pipeline can branch on NOT IN CATALOGUE. It cannot branch')
print('on a plausible invented price.')

import numpy as np

rng = np.random.default_rng(3)

# A calibrated model: when it says 70%, it is right 70% of the time.
# An overconfident one: when it says 90%, it is right 70% of the time.
buckets = np.array([0.55, 0.65, 0.75, 0.85, 0.95])

print('what the model claims   how often it is right')
print(f'{"":>21}   {"calibrated":>12} {"overconfident":>14}')
calibrated_error = 0.0
overconfident_error = 0.0
for claimed in buckets:
    actual_calibrated = claimed
    # Overconfidence compresses the true range into the top end.
    actual_over = 0.5 + (claimed - 0.5) * 0.45
    calibrated_error += abs(claimed - actual_calibrated) / len(buckets)
    overconfident_error += abs(claimed - actual_over) / len(buckets)
    print(f'{claimed:>21.0%}   {actual_calibrated:>12.0%} {actual_over:>14.0%}')
print()
print(f'mean calibration error: calibrated {calibrated_error:.3f}, '
      f'overconfident {overconfident_error:.3f}')
print()
print('Preference tuning - the step that makes a model helpful and')
print('polite - systematically produces the right-hand column.')
print('Human raters prefer confident answers, so training rewards')
print('confidence, so the stated percentages drift upwards while')
print('accuracy does not.')
print()
print('Two things that are genuinely useful, and one that is not:')
print()
print(' USEFUL   token log-probabilities, where the API exposes')
print('          them. They are at least a measurement of')
print('          something inside the model.')
print(' USEFUL   self-consistency: ask five times at temperature')
print('          0.7 and see whether the answers agree. Agreement')
print('          correlates with correctness far better than any')
print('          stated confidence.')
print(' NOT      asking the model how confident it is. You are')
print('          sampling text about confidence, not reading a')
print('          measurement.')

import collections

# Five independent samples for each of four questions. One has a
# fact in the prompt, one is a judgement call, one is a fact the
# model does not have, one is arithmetic.
runs = {
    'in the prompt': ['30 days', '30 days', '30 days', '30 days', '30 days'],
    'a judgement call': ['probably yes', 'yes', 'it depends',
                         'likely', 'yes with caveats'],
    'a fact it lacks': ['42 pounds', '38 pounds', '45 pounds',
                        '40 pounds', '36 pounds'],
    'arithmetic': ['11638', '11638', '11638', '11648', '11638'],
}

print(f'{"question":<20} {"agreement":>10} {"distinct":>9}  reading')
for question, answers in runs.items():
    counts = collections.Counter(answers)
    agreement = counts.most_common(1)[0][1] / len(answers)
    if agreement == 1.0:
        reading = 'stable'
    elif agreement >= 0.6:
        reading = 'mostly settled; check the outlier'
    else:
        reading = 'unstable - look closer'
    print(f'{question:<20} {agreement:>10.0%} {len(counts):>9}  {reading}')
print()
print('Two rows come out at 20% agreement, and they mean entirely')
print('different things. That is the trap in raw self-consistency:')
print('it measures whether the STRINGS match, and the strings can')
print('differ while the answer does not.')
print()

# So normalise before comparing. Here, to the claim being made.
def normalise(answer):
    text = answer.lower()
    if any(word in text for word in ('yes', 'likely', 'probably')):
        return 'affirmative'
    if 'depends' in text or 'caveat' in text:
        return 'qualified'
    return text


print('the same five samples, compared on the CLAIM rather than')
print('the wording:')
print()
print(f'{"question":<20} {"agreement":>10} {"distinct":>9}  reading')
for question, answers in runs.items():
    raw = len(collections.Counter(answers))
    counts = collections.Counter(normalise(a) for a in answers)
    agreement = counts.most_common(1)[0][1] / len(answers)
    if agreement == 1.0:
        reading = 'unanimous'
    elif len(counts) < raw:
        reading = f'{raw} wordings collapsed to {len(counts)} claims'
    elif agreement >= 0.8:
        reading = 'one answer plus an outlier'
    else:
        reading = 'genuinely different answers'
    print(f'{question:<20} {agreement:>10.0%} {len(counts):>9}  {reading}')
print()
print('The judgement call jumps to 80%: five wordings collapse to')
print('two claims, and four of the five samples said yes. That is')
print('agreement, not uncertainty, and raw string matching called')
print('it the same as total ignorance.')
print()
print('The arithmetic row does NOT collapse - 11638 and 11648 are')
print('two different numbers, not two phrasings - so 80% there')
print('means four runs agreed and one got it wrong. Worth')
print('catching, and a different problem from the row below it.')
print()
print('The missing price stays at 20% under both measures. Five')
print('different numbers are five different answers however you')
print('normalise them.')
print()
print('The missing-price row is the signal worth building on. Five')
print('different prices for the same product is the model telling')
print('you it is guessing, and five calls is cheap against the')
print('cost of shipping a wrong price to a customer.')
print()
print('Note the first row carefully. Perfect agreement means the')
print('answer is STABLE, not that it is right. A model that')
print('reliably misreads the same document agrees with itself')
print('every time. Self-consistency detects uncertainty, not')
print('error.')

tasks = [
    ('Write a sonnet about databases', 'hard for a person', 'easy'),
    ('Count the letters in "antidisestablishmentarianism"',
     'trivial for a person', 'unreliable'),
    ('Summarise a 20-page contract', 'hard for a person', 'easy'),
    ('Multiply 847 by 1293', 'tedious for a person', 'unreliable'),
    ('Translate idiomatic Japanese to English',
     'needs years of study', 'good'),
    ('Say which of two numbers is larger: 9.11 or 9.9',
     'trivial for a person', 'historically wrong'),
    ('Write a working SQL query from a description',
     'needs training', 'good'),
    ('Reverse the word "stressed"', 'trivial for a person',
     'unreliable'),
    ('Explain a research paper to a teenager',
     'needs real understanding', 'good'),
]
width = max(len(task) for task, _, _ in tasks)
print(f'{"task":<{width}}  {"for a person":<24} {"for a model"}')
for task, human, model in tasks:
    print(f'{task:<{width}}  {human:<24} {model}')
print()
print('Every "unreliable" row is a tokenisation problem or an')
print('arithmetic problem, and every "easy" row is a')
print('language-production problem. The frontier is jagged because')
print('it follows the TRAINING OBJECTIVE, not human difficulty.')
print()
print('The 9.11 against 9.9 case is the clearest illustration.')
print('Version numbers and dates make "11" greater than "9" all')
print('over the training data, the digits are separate tokens, and')
print('there is no numeric comparison anywhere in the')
print('architecture - only a statistical association between')
print('digit strings.')
print()
print('Practical consequence: do not reason about what a model')
print('will be good at. Test the actual task. Intuition about')
print('difficulty is calibrated to humans and transfers badly.')

import collections

# A mocked-up evaluation over four task types, ten cases each.
# In production these are real model outputs; the structure of
# the analysis is the point.
results = {
    'extract a date from text': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
    'classify sentiment': [1, 1, 1, 1, 1, 1, 1, 0, 1, 1],
    'sum a column of numbers': [1, 1, 0, 1, 0, 1, 0, 1, 1, 0],
    'recall a company founding year': [1, 0, 0, 1, 0, 0, 1, 0, 0, 1],
}

print(f'{"task":<32} {"accuracy":>9} {"95% interval":>16}  verdict')
for task, scores in results.items():
    n = len(scores)
    correct = sum(scores)
    rate = correct / n
    # Wilson interval - honest about ten samples, which a plain
    # proportion is not.
    z = 1.96
    centre = (rate + z * z / (2 * n)) / (1 + z * z / n)
    spread = (z / (1 + z * z / n)) * (
        (rate * (1 - rate) / n + z * z / (4 * n * n)) ** 0.5)
    low, high = max(0.0, centre - spread), min(1.0, centre + spread)
    verdict = ('ship it' if low > 0.9 else
               'needs a tool or retrieval' if high < 0.8 else
               'too few samples to say')
    print(f'{task:<32} {rate:>9.0%} {f"[{low:.2f}, {high:.2f}]":>16}  {verdict}')
print()
print('Three things this table does that an accuracy number alone')
print('cannot.')
print()
print('It separates the tasks. The average over all forty cases is')
print('72%, which describes none of the four and would hide the')
print('fact that one of them is at 40%.')
print()
print('It shows the interval. 60% from ten samples spans [0.31,')
print('0.83] - wide enough that "60% accurate" is not yet a claim')
print('about anything. Even the row that scored 100% only')
print('establishes that the true rate is above 0.72. The verdict')
print('column says "too few samples to say" for three of the four,')
print('because that is the honest answer at this sample size, and')
print('a table that hid it would be worse than no table.')
print()
print('It points at the fix rather than the score. The two weak')
print('rows are arithmetic and recall, which are the two things')
print('this architecture cannot do - so the answer is a calculator')
print('and a database, not a better prompt and not a better')
print('model.')

# The same question, asked three ways. Only the framing changes.
QUERY = 'SELECT * FROM orders WHERE date > NOW()'

framings = [
    ('neutral', ['Is this SQL query correct?', '', QUERY]),
    ('user asserts it is right',
     ['I have checked this query and it is correct.',
      'Confirm for me.', '', QUERY]),
    ('user asserts it is wrong',
     ['This query is broken. Tell me what is wrong with it.',
      '', QUERY]),
]
for label, lines in framings:
    print(f'[{label}]')
    for line in lines:
        print('   ' + line)
    print()
print('All three ask about the same query. A preference-tuned')
print('model tends to agree with whichever premise the user')
print('supplied, because agreement was rewarded during training.')
print()
print('The query, for the record, returns only orders with a')
print('FUTURE date, which is almost certainly not what was')
print('wanted. The middle framing is the one most likely to get a')
print('confirmation that it is fine.')
print()
print('Two defences that work:')
print(' - ask neutrally, and never put your own conclusion in the')
print('   prompt;')
print(' - ask for the strongest argument AGAINST, in a separate')
print('   call, and read both answers.')
print()
print('This matters most in exactly the setting people reach for')
print('it: using a model to review your own work. The framing')
print('that feels natural - "here is my code, it looks fine to')
print('me" - is the framing that suppresses the review.')

# Four probes, ordered by how far outside the possible they sit.
probes = [
    ('A fact that exists', 'What is the boiling point of water?',
     'should be right, and is in the weights many times over'),
    ('A fact that is plausible but false',
     'What is the population of Thamesford-on-Sea?',
     'the name is plausible; a model with no escape hatch will '
     'produce a number'),
    ('A fact that cannot exist',
     'What was the verdict in Carpenter v. Mendoza, 1987?',
     'invented case; the shape of the question invites a verdict'),
    ('An impossible computation',
     'What is the last digit of pi?',
     'no answer exists; a well-tuned model says so'),
]
for label, question, note in probes:
    print(f'[{label}]')
    print(f'  Q: {question}')
    print(f'  -> {note}')
    print()
print('Run these four against whichever model you are using, then')
print('run them again with "If you do not know, say UNKNOWN."')
print('appended. The difference between the two sets of answers is')
print('the single most useful thing you can learn about a model')
print('before building on it.')
print()
print('What to look for:')
print(' - probe 2 and 3 answered with specifics and no hedge ->')
print('   you must supply retrieval and an escape hatch for every')
print('   factual question;')
print(' - probe 2 and 3 answered with UNKNOWN only when asked ->')
print('   the capability is there and the prompt has to request')
print('   it, which is cheap;')
print(' - probe 4 answered with a digit -> treat every numeric')
print('   answer from this model as unverified.')
print()
print('None of this is pessimism about the technology. It is the')
print('same discipline as knowing that floating-point addition is')
print('not associative: once you know where the edges are, you')
print('build inside them, and what you build is reliable.')