Natural Language Processing cheat sheet

A scannable Natural Language Processing reference: 23 short snippets across 12 topics, each linking back to the lesson it came from.

At a glance

Quick snippets

Text preprocessing and tokens

Cleaning without destroying signal

import re
import unicodedata

def normalise(text):
    # NFKC folds full-width characters, ligatures and compatibility forms
    text = unicodedata.normalize("NFKC", text)
    text = re.sub(r"https?://\S+", " ", text)   # URLs are noise for most tasks
    text = re.sub(r"[ \t]+", " ", text)         # collapse horizontal whitespace
    return text.strip()

print(normalise("Cafe\u0301   costs  5 pounds"))   # "Cafe  costs 5 pounds"

From strings to tokens

from transformers import AutoTokenizer

tok = AutoTokenizer.from_pretrained("bert-base-uncased")
ids = tok("Tokenization isn't trivial.", add_special_tokens=True)["input_ids"]
print(tok.convert_ids_to_tokens(ids))
# ['[CLS]', 'token', '##ization', 'isn', "'", 't', 'trivial', '.', '[SEP]']

print(len(tok("unbelievable")["input_ids"]) - 2)   # pieces inside the special tokens

Full lesson: Text preprocessing and tokens →

Word and sentence embeddings

From sparse counts to dense vectors

import numpy as np

# Sparse: similarity is always zero until two documents share a literal term
one_hot = np.eye(3)                 # rows: cat, dog, car
print(float(one_hot[0] @ one_hot[1]))     # 0.0 - no notion of similarity

# Dense: distance encodes relatedness learned from co-occurrence
cat, dog, car = (np.array(v) for v in ([0.9, 0.1, 0.0], [0.8, 0.2, 0.1], [0.0, 0.9, 0.8]))

def cosine(a, b):
    return float(a @ b / (np.linalg.norm(a) * np.linalg.norm(b)))

… 2 more lines in the full lesson.

Using embeddings in practice

from sentence_transformers import SentenceTransformer
import numpy as np

model = SentenceTransformer("all-MiniLM-L6-v2")
docs = [
    "Reset your password from the account settings page.",
    "Billing is charged monthly on the signup date.",
    "Password reset links expire after one hour.",
]

# normalise_embeddings makes the dot product equal to cosine similarity
emb = model.encode(docs, normalize_embeddings=True)

… 5 more lines in the full lesson.

Full lesson: Word and sentence embeddings →

Transformers and fine-tuning basics

What self-attention computes

import torch
import torch.nn.functional as F

def self_attention(x, Wq, Wk, Wv):
    # x: (batch, seq_len, d_model)
    q, k, v = x @ Wq, x @ Wk, x @ Wv
    d_k = q.size(-1)
    scores = q @ k.transpose(-2, -1) / d_k ** 0.5   # scale keeps softmax stable
    weights = F.softmax(scores, dim=-1)             # each row sums to 1
    return weights @ v

x = torch.randn(2, 16, 64)

… 2 more lines in the full lesson.

When fine-tuning is worth it

from transformers import (
    AutoModelForSequenceClassification, AutoTokenizer, Trainer, TrainingArguments,
)

name = "distilbert-base-uncased"
tok = AutoTokenizer.from_pretrained(name)
model = AutoModelForSequenceClassification.from_pretrained(name, num_labels=2)

def encode(batch):
    return tok(batch["text"], truncation=True, max_length=256, padding="max_length")

train_ds, val_ds = train_raw.map(encode), val_raw.map(encode)

… 12 more lines in the full lesson.

Full lesson: Transformers and fine-tuning basics →

Classical text classification

Bag of words and TF-IDF

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.pipeline import Pipeline
from sklearn.linear_model import LogisticRegression

texts = [
    "the delivery was fast and the box was intact",
    "terrible service, my package arrived broken",
    "refund requested, item never shipped",
    "great quality, would order again",
]
labels = ["positive", "negative", "negative", "positive"]

… 13 more lines in the full lesson.

Naive Bayes versus linear models

from sklearn.naive_bayes import MultinomialNB, ComplementNB
from sklearn.svm import LinearSVC
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import cross_val_score

models = {
    "multinomial_nb": MultinomialNB(alpha=0.1),
    "complement_nb": ComplementNB(alpha=0.3),
    "logreg": LogisticRegression(max_iter=2000, C=4.0),
    "linear_svc": LinearSVC(C=1.0),
    "sgd_log": SGDClassifier(loss="log_loss", alpha=1e-5, max_iter=50),
}

… 7 more lines in the full lesson.

Inspecting what the model learned

import numpy as np

vec = pipeline.named_steps["tfidf"]
clf = pipeline.named_steps["clf"]
terms = np.array(vec.get_feature_names_out())

for class_index, class_name in enumerate(clf.classes_):
    weights = clf.coef_[class_index]
    top = np.argsort(weights)[-12:][::-1]
    print(class_name, list(terms[top]))

from sklearn.metrics import classification_report, confusion_matrix

… 3 more lines in the full lesson.

Full lesson: Classical text classification →

Sequence labelling: POS tagging and NER

Tagging schemes

tokens = ["Apple", "was", "founded", "in", "Cupertino", "by", "Steve", "Jobs"]

# BIO: B- begins a span, I- continues it, O is outside any span
bio_tags = ["B-ORG", "O", "O", "O", "B-LOC", "O", "B-PER", "I-PER"]

def extract_spans(tokens, tags):
    spans, current = [], None
    for token, tag in zip(tokens, tags):
        prefix, _, label = tag.partition("-")
        if prefix == "B":
            if current:
                spans.append(current)

… 13 more lines in the full lesson.

BiLSTM-CRF and transformer taggers

import torch
import torch.nn as nn

class BiLSTMTagger(nn.Module):
    def __init__(self, vocab_size, n_tags, dim=128, hidden=128, dropout=0.3):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, dim, padding_idx=0)
        self.lstm = nn.LSTM(dim, hidden, batch_first=True, bidirectional=True)
        self.dropout = nn.Dropout(dropout)
        self.classifier = nn.Linear(2 * hidden, n_tags)
        self.crf = None       # swap in torchcrf.CRF(n_tags) for structured decoding

… 9 more lines in the full lesson.

Entity-level evaluation

from seqeval.metrics import classification_report, f1_score

true_tags = [["B-ORG", "O", "O", "B-PER", "I-PER"]]
pred_tags = [["B-ORG", "O", "O", "B-PER", "O"]]      # missed "Jobs"

print(f1_score(true_tags, pred_tags))                # entity-level, not token-level
print(classification_report(true_tags, pred_tags, zero_division=0))

# the three error types, counted separately
def error_breakdown(gold_seqs, pred_seqs):
    from collections import Counter
    counts = Counter()

… 14 more lines in the full lesson.

Full lesson: Sequence labelling: POS tagging and NER →

Text similarity, clustering and topic modelling

Clustering

from sklearn.cluster import KMeans, AgglomerativeClustering
from sklearn.metrics import silhouette_score
from sentence_transformers import SentenceTransformer

encoder = SentenceTransformer("all-MiniLM-L6-v2")
X = encoder.encode(docs, normalize_embeddings=True)

# k-means needs k; sweep it and look for an elbow and a silhouette peak
for k in range(2, min(8, len(docs))):
    labels = KMeans(n_clusters=k, n_init=10, random_state=0).fit_predict(X)
    print(k, round(silhouette_score(X, labels), 3))

… 9 more lines in the full lesson.

Topic modelling

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation

counts = CountVectorizer(
    max_df=0.95, min_df=2, stop_words="english", ngram_range=(1, 2))
X_counts = counts.fit_transform(docs)

lda = LatentDirichletAllocation(
    n_components=5, max_iter=20, learning_method="batch", random_state=0)
lda.fit(X_counts)

vocab = counts.get_feature_names_out()

… 9 more lines in the full lesson.

Full lesson: Text similarity, clustering and topic modelling →

Semantic search and vector databases

Chunking decides the ceiling

from langchain_text_splitters import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=800,          # characters, not tokens: roughly 200 tokens
    chunk_overlap=120,       # about 15% overlap keeps a sentence from being cut
    separators=["

", "
", ". ", " ", ""],
    length_function=len,
)

… 9 more lines in the full lesson.

Hybrid search and reranking

from rank_bm25 import BM25Okapi

corpus_tokens = [c.lower().split() for c in chunks]
bm25 = BM25Okapi(corpus_tokens)

def hybrid(query, k=10, alpha=0.5):
    lexical = np.asarray(bm25.get_scores(query.lower().split()))
    lexical = lexical / (lexical.max() + 1e-9)

    q = encoder.encode([query], normalize_embeddings=True).astype("float32")
    dense_scores, dense_ids = index.search(q, len(chunks))
    dense = np.zeros(len(chunks))

… 15 more lines in the full lesson.

Full lesson: Semantic search and vector databases →

Summarisation

Abstractive summarisation

from transformers import pipeline

summariser = pipeline("summarization", model="facebook/bart-large-cnn", device=0)

def summarise(text, max_len=130, min_len=40):
    # BART has a 1024-token limit; split long inputs on paragraph boundaries
    result = summariser(text, max_length=max_len, min_length=min_len,
                        do_sample=False, truncation=True)
    return result[0]["summary_text"]

print(summarise(article[:4000]))

… 6 more lines in the full lesson.

Full lesson: Summarisation →

Question answering and reading comprehension

Extractive question answering

from transformers import pipeline

qa = pipeline("question-answering", model="deepset/roberta-base-squad2")

context = """The support plan costs 49 GBP per month and includes a four-hour
response target. Enterprise customers receive a one-hour response target and a
dedicated account manager."""
question = "What is the response time for enterprise customers?"

answer = qa(question=question, context=context)
print(answer)
# {'score': 0.94, 'start': 118, 'end': 124, 'answer': 'one-hour'}

… 5 more lines in the full lesson.

Making the system abstain

import re

def normalise(text):
    return re.sub(r"\s+", " ", text.lower()).strip()

def is_supported(answer, passage, threshold=0.6):
    """Cheap lexical check: does the answer appear in the passage at all?"""
    return normalise(answer) in normalise(passage)

def answer_with_abstention(question, passages, qa_pipe, min_extraction=0.1):
    best = {"answer": None, "confidence": 0.0, "reason": "no answer found"}
    for passage in passages:

… 15 more lines in the full lesson.

Full lesson: Question answering and reading comprehension →

Translation and multilingual pipelines

Choosing a translation model

from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM

# a bilingual model: one pair, usually higher quality
en_de = pipeline("translation_en_to_de", model="Helsinki-NLP/opus-mt-en-de")
print(en_de("The deployment failed because the database was unreachable.")[0]["translation_text"])

# a multilingual model: many pairs, one artefact to operate
tok = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")
model = AutoModelForSeq2SeqLM.from_pretrained("facebook/nllb-200-distilled-600M")

def translate(text, src="eng_Latn", tgt="fra_Latn", max_new_tokens=256):
    tok.src_lang = src

… 10 more lines in the full lesson.

Tokenisation and non-Latin scripts

from transformers import AutoTokenizer

tok = AutoTokenizer.from_pretrained("facebook/nllb-200-distilled-600M")

samples = {
    "english": "Deployment failed.",
    "german": "Die Bereitstellung ist fehlgeschlagen.",
    "japanese": "Deployment failed.",
    "arabic": "Deployment failed.",
}

for name, text in samples.items():

… 7 more lines in the full lesson.

Full lesson: Translation and multilingual pipelines →

Working with large language models for NLP tasks

Prompting or fine-tuning

# build the training set for a fine-tune from the examples you already labelled
import json

def to_chat_records(pairs):
    records = []
    for text, label in pairs:
        records.append({
            "messages": [
                {"role": "system", "content": SYSTEM},
                {"role": "user", "content": text},
                {"role": "assistant","content": json.dumps(label, ensure_ascii=False)},
            ]

… 9 more lines in the full lesson.

Full lesson: Working with large language models for NLP tasks →

Evaluating NLP systems

Automatic metrics and what they miss

import sacrebleu
from rouge_score import rouge_scorer
from bert_score import score as bertscore

hypotheses = ["the cat sat on the mat"]
references = [["the cat is sitting on the mat"]]

print(sacrebleu.corpus_bleu(hypotheses, references).score)
print(sacrebleu.corpus_chrf(hypotheses, references).score)

rouge = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"], use_stemmer=True)
print({k: round(v.fmeasure, 3) for k, v in rouge.score(references[0][0], hypotheses[0]).items()})

… 7 more lines in the full lesson.

Full lesson: Evaluating NLP systems →

FAQ

Related cheat sheets

AI Basics AI Agents Math for AI Machine Learning scikit-learn TensorFlow

Last refreshed 2026-09-27.