Parsing and consuming feeds in code

Three formats, one internal shape. Normalising early is what stops feed-handling code from spreading format checks through the whole application.

Libraries that do the normalising

import feedparser
from datetime import datetime, timezone

def normalise(url: str) -> list[dict]:
    feed = feedparser.parse(url)

    if feed.bozo and not feed.entries:
        # bozo is set for any parse problem; only fail when there is nothing usable
        raise ValueError("unparseable feed: " + str(feed.bozo_exception))

    out = []
    for e in feed.entries:
        published = e.get("published_parsed") or e.get("updated_parsed")
        when = (datetime(*published[:6], tzinfo=timezone.utc)
                if published else None)
        out.append({
            "id": e.get("id") or e.get("link") or e.get("title", ""),
            "title": (e.get("title") or "").strip(),
            "url": e.get("link", ""),
            "summary": (e.get("summary") or "").strip(),
            # content is a list of dicts with value and type
            "html": e.get("content", [{}])[0].get("value") if e.get("content") else "",
            "published": when,
            "author": e.get("author"),
            "tags": [t.get("term") for t in e.get("tags", [])],
        })
    return out

• feedparser never raises on bad XML; it sets bozo and returns whatever it could salvage. Check bozo_exception and decide whether the partial result is usable.
• Dates arrive in several forms. Use the parsed tuple, never the raw string, and treat a missing date as unknown rather than substituting the current time.
• Item identity is the first available of id, link, then title. If all three are missing the item has no identity and should be dropped.
• content is a list because Atom permits several content constructs; the first entry is the practical choice.
• Encoding issues are almost always a missing or wrong Content-Type charset; feedparser sniffs it, but check the raw bytes when text looks mangled.

Parsing hostile XML safely

import html
import lxml.html
import lxml.etree as etree

# 1. Bound the download before parsing anything
MAX_BYTES = 2 * 1024 * 1024
raw = fetch(url, max_bytes=MAX_BYTES, timeout=10)

# 2. Parse with entity expansion and network access disabled
parser = etree.XMLParser(resolve_entities=False, no_network=True,
                         load_dtd=False, huge_tree=False)
try:
    tree = etree.fromstring(raw, parser)
except etree.XMLSyntaxError:
    raise ValueError("feed is not well-formed XML")

# 3. Sanitise any HTML before it reaches a template
ALLOWED = {"p", "br", "a", "em", "strong", "ul", "ol", "li", "blockquote", "code", "img"}
CLEAN = lxml.html.clean.Cleaner(
    allow_tags=ALLOWED,
    remove_unknown_tags=False,
    safe_attrs_only=True,
    safe_attrs={"href", "src", "alt", "title"},
)

def safe_html(fragment: str) -> str:
    return CLEAN.clean_html(fragment)

FAQ

Related

Feed security: sanitising third-party content Aggregating feeds: OPML, merging and republishing

Last refreshed 2026-09-18.