Sequence labelling: POS tagging and NER

BIO tagging, CRFs and BiLSTM taggers, transformer token classification, subword alignment, and entity-level evaluation that is actually correct.

Tagging schemes

tokens = ["Apple", "was", "founded", "in", "Cupertino", "by", "Steve", "Jobs"]

# BIO: B- begins a span, I- continues it, O is outside any span
bio_tags = ["B-ORG", "O", "O", "O", "B-LOC", "O", "B-PER", "I-PER"]

def extract_spans(tokens, tags):
    spans, current = [], None
    for token, tag in zip(tokens, tags):
        prefix, _, label = tag.partition("-")
        if prefix == "B":
            if current:
                spans.append(current)
            current = {"label": label, "start": len(spans), "tokens": [token]}
        elif prefix == "I" and current and current["label"] == label:
            current["tokens"].append(token)
        else:
            if current:
                spans.append(current)
            current = None
    if current:
        spans.append(current)
    return [(s["label"], " ".join(s["tokens"])) for s in spans]

print(extract_spans(tokens, bio_tags))
# [('ORG', 'Apple'), ('LOC', 'Cupertino'), ('PER', 'Steve Jobs')]

• BIO cannot represent two adjacent entities of the same type without a separating token; BIOES (with S- for a single-token span) removes that ambiguity and usually scores slightly better.
• An I- tag with no preceding B- of the same type is invalid. Your decoder should treat it as outside, not as the start of a span.
• The tag sequence is a structured prediction, so token accuracy is a poor metric: predicting all O gets about 85% token accuracy and finds zero entities.

BiLSTM-CRF and transformer taggers

import torch
import torch.nn as nn

class BiLSTMTagger(nn.Module):
    def __init__(self, vocab_size, n_tags, dim=128, hidden=128, dropout=0.3):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, dim, padding_idx=0)
        self.lstm = nn.LSTM(dim, hidden, batch_first=True, bidirectional=True)
        self.dropout = nn.Dropout(dropout)
        self.classifier = nn.Linear(2 * hidden, n_tags)
        self.crf = None       # swap in torchcrf.CRF(n_tags) for structured decoding

    def forward(self, token_ids, mask=None):
        x = self.dropout(self.embed(token_ids))
        if mask is not None:
            x = x * mask.unsqueeze(-1)
        out, _ = self.lstm(x)
        emissions = self.classifier(self.dropout(out))
        if self.crf is None:
            return emissions
        return self.crf.decode(emissions, mask=mask.bool())

A CRF layer learns transition scores between tags, so it never emits an I-PER directly after a B-LOC without paying a penalty. In practice it buys a fraction of a point over an independent per-token classifier, and much less than better embeddings do.

# the modern approach: a token-classification head over a pretrained encoder
from transformers import AutoTokenizer, AutoModelForTokenClassification
from transformers import DataCollatorForTokenClassification, TrainingArguments, Trainer

label_list = ["O", "B-PER", "I-PER", "B-ORG", "I-ORG", "B-LOC", "I-LOC"]
id2label = {i: l for i, l in enumerate(label_list)}
label2id = {l: i for i, l in id2label.items()}

tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased")
model = AutoModelForTokenClassification.from_pretrained(
    "distilbert-base-cased",
    num_labels=len(label_list), id2label=id2label, label2id=label2id)

def tokenize_and_align(examples):
    enc = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)
    aligned = []
    for i, labels in enumerate(examples["ner_tags"]):
        word_ids = enc.word_ids(batch_index=i)
        previous = None
        row = []
        for word_id in word_ids:
            if word_id is None:
                row.append(-100)                       # special tokens are ignored
            elif word_id != previous:
                row.append(labels[word_id])            # first subword carries the tag
            else:
                row.append(-100)                       # continuation subwords ignored
            previous = word_id
        aligned.append(row)
    enc["labels"] = aligned
    return enc

Entity-level evaluation

from seqeval.metrics import classification_report, f1_score

true_tags = [["B-ORG", "O", "O", "B-PER", "I-PER"]]
pred_tags = [["B-ORG", "O", "O", "B-PER", "O"]]      # missed "Jobs"

print(f1_score(true_tags, pred_tags))                # entity-level, not token-level
print(classification_report(true_tags, pred_tags, zero_division=0))

# the three error types, counted separately
def error_breakdown(gold_seqs, pred_seqs):
    from collections import Counter
    counts = Counter()
    for gold, pred in zip(gold_seqs, pred_seqs):
        g = set(extract_spans(gold, [t] * len(gold))) if False else None
        for g_tag, p_tag in zip(gold, pred):
            if g_tag == p_tag:
                continue
            if g_tag == "O":
                counts["spurious"] += 1
            elif p_tag == "O":
                counts["missed"] += 1
            else:
                counts["confused_label"] += 1
    return counts

print(error_breakdown(true_tags, pred_tags))

• Entity-level F1 requires an exact span match including boundaries. A span with the right label but one token too long counts as both a false positive and a false negative.
• Report per-entity-type scores. A single micro F1 hides a model that is excellent on person names and useless on organisations.
• Split by document, not by sentence. Sentences from the same document leak context and inflate the score.
• Boundary errors dominate in practice. Inspect whether misses are systematic (all long entities, all nested ones) before adding capacity.

FAQ

Related

Transformers and fine-tuning basics Evaluating NLP systems

Last refreshed 2026-09-18.