diff --git a/datasets/amfv_datasets/scraping/__init__.py b/datasets/amfv_datasets/scraping/__init__.py index cd7cdb0..d5630df 100644 --- a/datasets/amfv_datasets/scraping/__init__.py +++ b/datasets/amfv_datasets/scraping/__init__.py @@ -27,6 +27,16 @@ scrape_guideline, scrape_nice, ) +from amfv_datasets.scraping.pubmed import ( + PubMedArticleRef, + PubMedFetchError, + PubMedListingPage, + build_pubmed_article_text, + list_pubmed_guidelines, + pubmed_ref_from_url, + scrape_pubmed, + scrape_pubmed_article, +) __all__ = [ "GuidanceRef", @@ -34,6 +44,9 @@ "LinkMode", "NiceFetchError", "OutputFormat", + "PubMedArticleRef", + "PubMedFetchError", + "PubMedListingPage", "ScrapeError", "ScrapeRun", "ScrapedDocument", @@ -41,6 +54,7 @@ "USER_AGENT", "absolute_unique_urls", "build_guideline_text", + "build_pubmed_article_text", "clean_text", "document_title", "default_client", @@ -48,7 +62,11 @@ "guidance_ref_from_url", "html_to_markdown", "list_published_guidance", + "list_pubmed_guidelines", + "pubmed_ref_from_url", "scrape_guideline", "scrape_listing_documents", "scrape_nice", + "scrape_pubmed", + "scrape_pubmed_article", ] diff --git a/datasets/amfv_datasets/scraping/base.py b/datasets/amfv_datasets/scraping/base.py index 0708ee8..3651cdb 100644 --- a/datasets/amfv_datasets/scraping/base.py +++ b/datasets/amfv_datasets/scraping/base.py @@ -74,7 +74,7 @@ def scrape_listing_documents[ListingItemT]( documents: int | None, client_factory: Callable[[], AbstractContextManager[httpx.Client]], list_page: Callable[[httpx.Client, int], Iterable[ListingItemT]], - scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument], + scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument | None], document_delay_seconds: float = 5.0, first_page_items: Iterable[ListingItemT] | None = None, ) -> Iterable[ScrapedDocument]: @@ -85,7 +85,8 @@ def scrape_listing_documents[ListingItemT]( fetched until a page returns no items (default: None). client_factory: Factory returning a context-managed HTTP client. list_page: Function that lists source-specific items for a page. - scrape_item: Function that scrapes one listed item into a document. + scrape_item: Function that scrapes one listed item into a document. It + may return None to skip a discovered item that is not a document. document_delay_seconds: Delay before scraping each document after the first one (default: 5.0). first_page_items: Already-fetched first listing page items. When set, @@ -99,6 +100,7 @@ def scrape_listing_documents[ListingItemT]( with client_factory() as client: page = 1 scraped = 0 + attempted = 0 page_items = list(first_page_items) if first_page_items is not None else None while documents is None or scraped < documents: if page_items is None: @@ -111,9 +113,13 @@ def scrape_listing_documents[ListingItemT]( for item in items: if documents is not None and scraped >= documents: break - if scraped and document_delay_seconds: + if attempted and document_delay_seconds: time.sleep(document_delay_seconds) - yield scrape_item(client, item) + document = scrape_item(client, item) + attempted += 1 + if document is None: + continue + yield document scraped += 1 page += 1 diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py index e8aece3..32cafb1 100644 --- a/datasets/amfv_datasets/scraping/cli.py +++ b/datasets/amfv_datasets/scraping/cli.py @@ -27,6 +27,7 @@ from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun from amfv_datasets.scraping.html import LinkMode from amfv_datasets.scraping.nice import scrape_nice +from amfv_datasets.scraping.pubmed import scrape_pubmed class ScraperSource(StrEnum): @@ -34,6 +35,7 @@ class ScraperSource(StrEnum): ALL = "all" NICE = "nice" + PUBMED = "pubmed" class OutputFormat(StrEnum): @@ -72,6 +74,8 @@ def scrape_documents( match selected_source: case ScraperSource.NICE: return scrape_nice(documents=documents, link_mode=link_mode, url=url) + case ScraperSource.PUBMED: + return scrape_pubmed(documents=documents, link_mode=link_mode, url=url) case ScraperSource.ALL: raise AssertionError("expanded source cannot be all") raise AssertionError(f"unsupported source: {source}") @@ -125,7 +129,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]: if source is ScraperSource.ALL: - return (ScraperSource.NICE,) + return (ScraperSource.NICE, ScraperSource.PUBMED) return (source,) diff --git a/datasets/amfv_datasets/scraping/pubmed.py b/datasets/amfv_datasets/scraping/pubmed.py new file mode 100644 index 0000000..5a1f411 --- /dev/null +++ b/datasets/amfv_datasets/scraping/pubmed.py @@ -0,0 +1,723 @@ +"""Scrape PubMed clinical practice guidelines into normalized markdown documents. + +PubMed indexes roughly 43k articles tagged with the `Guideline` publication +type, but most of them expose an abstract only and stay under publisher +copyright. This module deliberately scrapes the intersection with the PMC Open +Access subset instead: + + Guideline[pt] AND pubmed pmc open access[filter] AND English[la] + +That is about 3,000 guidelines, growing by a handful a week, and it is the only +slice where the full text is both retrievable and openly licensed. + +Licensing is recorded per document rather than claimed for the source, because +the Open Access Subset is not uniformly Creative Commons licensed. Censused +over all 2,999 records present on 2026-07-29: CC BY 44.5%, CC BY-NC 22.1%, +CC BY-NC-ND 18.4%, CC BY-NC-SA 2.1%, CC0 1.3%, and 11.7% carrying publisher +terms instead. By what that permits, 45.8% is unrestricted for derivative +works, 24.1% is non-commercial only, 18.4% asserts NoDerivatives, and 11.7% +has to be read case by case. + +Discovery and extraction both go through NCBI's E-utilities rather than the +rendered pubmed.ncbi.nlm.nih.gov pages. `esearch` paginates by numeric offset and +reports the result total, `esummary` resolves a whole batch of PMIDs to PMCIDs +and citation metadata in one request, and `efetch` returns JATS XML carrying the +article body, section structure and license. All three are documented, versioned +contracts that do not break when the website is restyled. + +Attribution: +Meditron's guideline scrapers (epfLLM/meditron, gap-replay/guidelines) cover a +dozen sources but have no PubMed scraper, so this module has no upstream port to +follow. It follows the conventions of `nice.py` in this package instead. +Source license: Apache License 2.0. +""" + +from __future__ import annotations + +import copy +import logging +import os +import re +import time +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any +from urllib.parse import urlparse + +import httpx +from lxml import etree + +from amfv_datasets.scraping.base import ( + ScrapedDocument, + ScrapeError, + ScrapeRun, + default_client, + scrape_listing_documents, +) +from amfv_datasets.scraping.html import LinkMode, clean_text, html_to_markdown + +BASE_URL = "https://pubmed.ncbi.nlm.nih.gov" +EUTILS_URL = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils" +PMC_ARTICLE_URL = "https://www.ncbi.nlm.nih.gov/pmc/articles" +# `Guideline` is a strict superset of `Practice Guideline`: searching for either +# returns exactly the `Guideline` count. The 352 extra records are clinical, not +# administrative — the 2025 Korean CPR guidelines and similar — so scoping to +# `Practice Guideline` would drop 352 records (11%) for nothing. +# +# `English[la]` drops 185 records. Most other sources in this package are +# English-only as a side effect of the URL they start from (CPS is a bilingual +# site scraped through its `/en/` routes; WHO publishes in six languages and is +# scraped through its English listing). PubMed's API returns every language, so +# the filter is explicit here to match. The excluded records are largely French +# CMAJ translations of English guidelines already in the corpus. +SEARCH_TERM = "Guideline[pt] AND pubmed pmc open access[filter] AND English[la]" +PUBMED_DATASET_NAME = "pubmed-webscrape" +PUBMED_DATASET_DISPLAY_NAME = "PubMed Webscrape" +NCBI_TOOL = "amfv" +# NCBI documents 3 requests/second without an API key and 10 with one, so unlike +# a scraped website this delay is a published allowance rather than a guess. Do +# not raise it to match the other scrapers "for consistency" — that turns a +# 20-minute run into a 4-hour one for no benefit. +DOCUMENT_DELAY_SECONDS = 0.4 +LISTING_PAGE_SIZE = 200 +# Enough of a non-CC license statement to tell "unrestricted re-use" apart from +# "all rights reserved" without carrying a wall of boilerplate per document. +_LICENSE_STATEMENT_CHARS = 400 +_FIGURE_CAPTION_CHARS = 300 +# NCBI answers a burst with 429 and is transiently unavailable often enough that +# a multi-thousand document run needs to ride both out. +_MAX_RETRIES = 4 +_RETRY_BACKOFF_SECONDS = 1.0 +_RETRYABLE_STATUS = frozenset({429, 500, 502, 503, 504}) + +logger = logging.getLogger(__name__) + +_PUBMED_PATH_RE = re.compile(r"^/(?P\d+)/?$") +_PMC_PATH_RE = re.compile(r"^/pmc/articles/(?PPMC\d+)/?$", re.IGNORECASE) +_CC_URL_RE = re.compile(r"https?://creativecommons\.org/\S+?(?=[\s\"<)]|$)") +# Unwrapping `xref` and dropping figures leaves the surrounding spacing behind, +# as in "(PRISMA flow diagram in Supplementary Fig. S1) ." — close it back up. +_ORPHAN_SPACE_RE = re.compile(r" +([,.;:)\]])") +_XLINK_HREF = "{http://www.w3.org/1999/xlink}href" +_ALI_LICENSE_REF = "{http://www.niso.org/schemas/ali/1.0/}license_ref" + +# JATS elements whose entire subtree is dropped. Reference lists and footnote +# groups are citation apparatus rather than clinical content. Figure payloads +# cannot be linked from the API response at all (see `_article_figures`), so the +# `` goes but its `