Math for AI cheat sheet

A scannable Math for AI reference: 13 short snippets across 9 topics, each linking back to the lesson it came from.

At a glance

Quick snippets

Vectors and matrices for ML

Vectors

import numpy as np

a = np.array([1.0, 2.0, 3.0])
b = np.array([4.0, 0.0, 1.0])

a @ b                      # 7.0  dot product = sum(a_i * b_i)
np.linalg.norm(a)          # 3.7417  Euclidean length
a + b                      # elementwise
2 * a                      # scalar broadcast
np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b))   # 0.5492  cosine similarity

Counting dimensions

# parameter count of a small dense network
def dense_params(n_in, n_out):
    return n_in * n_out + n_out     # weights + biases

dense_params(784, 128)    # 100480
dense_params(128, 10)     # 1290

# memory for a float32 embedding table
rows, dims = 50_000, 768
rows * dims * 4 / 1e6     # 153.6 MB just for the vectors

Full lesson: Vectors and matrices for ML →

Probability and distributions

Random variables

def softmax(logits):
    z = logits - logits.max(axis=-1, keepdims=True)   # stability
    e = np.exp(z)
    return e / e.sum(axis=-1, keepdims=True)

logits = np.array([2.0, 1.0, 0.1])
softmax(logits)     # [0.6590, 0.2424, 0.0986] — sums to 1

Conditional probability and likelihood

# P(A|B): among emails that contain "invoice", how many are spam?
b = mask_with_word & mask_spam
p_a_given_b = b.sum() / mask_with_word.sum()

# Bayes: flip a conditional using the base rate
# P(spam|word) = P(word|spam) * P(spam) / P(word)
p_word_given_spam = 0.60
p_spam = 0.20
p_word = 0.15
p_spam_given_word = p_word_given_spam * p_spam / p_word      # 0.80

Conditional probability and likelihood

# cross-entropy for one example: -log(probability of the true class)
probs = np.array([0.70, 0.20, 0.10])
true_index = 0
loss = -np.log(probs[true_index])            # 0.3567

# the same loss, written for a batch
def cross_entropy(probs, labels):
    picked = probs[np.arange(len(labels)), labels]
    return -np.log(picked + 1e-12).mean()

Full lesson: Probability and distributions →

Gradients and calculus intuition

Derivatives measure sensitivity

def f(x):
    return 3 * x ** 2          # df/dx = 6x

h = 1e-5
numeric = (f(2.0 + h) - f(2.0 - h)) / (2 * h)    # central difference
analytic = 6 * 2.0
numeric, analytic            # 12.000000000006, 12.0

Reading the loss curve

# clip the gradient norm so a single bad batch cannot destroy training
import numpy as np

def clip(grad, max_norm=1.0):
    norm = np.linalg.norm(grad)
    if norm > max_norm:
        grad = grad * (max_norm / norm)
    return grad

# and always compare against a trivial baseline before trusting a score
# e.g. predict the majority class, or the mean of y for regression

Full lesson: Gradients and calculus intuition →

Matrix decompositions and PCA

Eigenvalues and eigenvectors

import numpy as np

A = np.array([[2.0, 1.0],
              [1.0, 2.0]])

values, vectors = np.linalg.eigh(A)   # eigh assumes symmetric: use it for covariances
values                                # [1.0, 3.0]  ascending
vectors                               # columns are the eigenvectors, orthonormal

v = vectors[:, 0]
np.allclose(A @ v, values[0] * v)     # True by definition

Full lesson: Matrix decompositions and PCA →

Multivariable calculus for training

Curvature, Hessians and fragility

# the Hessian is the Jacobian of the gradient: it describes curvature
def loss(w):
    return w[0] ** 2 + 20 * w[1] ** 2        # a long narrow valley

H = np.diag([2.0, 40.0])
np.linalg.eigvalsh(H)                       # [2, 40] -> condition number 20

# the largest safe learning rate for gradient descent is 2 / L
L = np.linalg.eigvalsh(H).max()
2 / L                                       # 0.05: above this it oscillates and diverges

Full lesson: Multivariable calculus for training →

Optimisation and gradient descent variants

Schedules and diagnosing divergence

# linear warmup then cosine decay — the common transformer schedule
def lr_at(step, total, base_lr=1e-3, warmup=100):
    if step < warmup:
        return base_lr * step / max(1, warmup)
    progress = (step - warmup) / max(1, total - warmup)
    return 0.5 * base_lr * (1 + np.cos(np.pi * progress))

import matplotlib.pyplot as plt
steps = np.arange(2000)
plt.plot(steps, [lr_at(s, 2000) for s in steps])
plt.xlabel("step"); plt.ylabel("learning rate"); plt.yscale("log")

Full lesson: Optimisation and gradient descent variants →

Information theory for machine learning

Choosing the loss from the output

# the numerically stable way to combine log-softmax and negative log-likelihood
logits = np.array([[2.0, 1.0, 0.1]])
target = np.array([0])

def log_softmax(z):
    z = z - z.max(axis=-1, keepdims=True)
    return z - np.log(np.exp(z).sum(axis=-1, keepdims=True))

loss = -log_softmax(logits)[np.arange(len(target)), target].mean()
print(round(float(loss), 4))             # 0.4170

Full lesson: Information theory for machine learning →

Statistical estimation

Confidence intervals and bootstrap

def bootstrap_ci(values, statistic=np.mean, n=5000, alpha=0.05):
    rng = np.random.default_rng(0)
    values = np.asarray(values, float)
    stats = np.array([statistic(rng.choice(values, size=len(values), replace=True))
                      for _ in range(n)])
    lo, hi = np.quantile(stats, [alpha / 2, 1 - alpha / 2])
    return float(lo), float(hi)

accuracy = np.array([1, 1, 0, 1, 0] * 40)     # 60% on 200 examples
bootstrap_ci(accuracy, n=2000)                # roughly (0.53, 0.67)

Full lesson: Statistical estimation →

Sampling and Monte Carlo methods

Monte Carlo error and variance reduction

# a control variate: subtract a function with a known mean
def control_variate(y, x, known_mean):
    cov = np.cov(y, x)[0, 1]
    c = cov / np.var(x)
    return y - c * (x - known_mean)

# estimator of pi by counting points in a quarter circle
u = rng.random((1_000_000, 2))
inside = ((u ** 2).sum(axis=1) <= 1.0).astype(float)
pi_hat = 4 * inside.mean()
stderr = 4 * inside.std() / np.sqrt(len(inside))
pi_hat, stderr, np.pi

Full lesson: Sampling and Monte Carlo methods →

FAQ

Related cheat sheets

AI Basics AI Agents Machine Learning scikit-learn TensorFlow PyTorch

Last refreshed 2026-09-27.