diff --git a/scripts/update_playwright_version.py b/scripts/update_playwright_version.py index 2aa1fd6455..35c40f4e1b 100755 --- a/scripts/update_playwright_version.py +++ b/scripts/update_playwright_version.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -"""Bump the Playwright version pinned by the project template's Dockerfile. +"""Bump the Playwright versions pinned by the project template's Dockerfile. -The template Dockerfile pins a single Playwright version (a Jinja `# % set playwright_version -= '...'` line) that selects the Apify base image tag and the in-image `playwright==` -pin. A version is only safe to pin once Apify has published the matching base image, so the -Apify Playwright base image's Docker Hub tags are the source of truth: this picks the highest -stable `-` tag for the Python version the template already uses, and rewrites -the pinned version line if it is newer. The Python version itself is never changed. +The template Dockerfile pins a shared Playwright version (a Jinja `# % set playwright_version = +'...'` line) that selects the standard Apify base image tags and their in-image +`playwright==` pins. A version is only safe to pin once Apify has published the matching +base image for every standard variant, so those images' Docker Hub tags are the source of truth: +this picks the highest stable `-` tag shared by those images, for the Python +version the template already uses, and rewrites the pinned version line if it is newer. The +Python version and Camoufox's separate compatibility pin are never changed. Single-purpose: run with no arguments from anywhere in the repository. """ @@ -21,51 +22,114 @@ DOCKERFILE = ( Path(__file__).resolve().parent.parent / 'src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile' ) -TAGS_URL = 'https://hub.docker.com/v2/repositories/apify/actor-python-playwright/tags?page_size=100' +TAGS_URL_TEMPLATE = 'https://hub.docker.com/v2/repositories/apify/{repository}/tags?page_size=100' +REQUEST_TIMEOUT_SECONDS = 30 +CAMOUFOX_VERSION_VARIABLE = 'camoufox_playwright_version' +CAMOUFOX_REPOSITORY = 'actor-python-playwright-camoufox' +SHARED_VERSION_VARIABLE = 'playwright_version' -# The pinned version line, e.g. `# % set playwright_version = '1.60.0'`. -VERSION_LINE = re.compile(r"(# % set playwright_version = ')([^']+)(')") +# The standard Apify image repositories selected by the automatically updated shared pin. Camoufox +# caps `playwright` below the latest release, so its explicit compatibility pin is excluded. +SHARED_VERSION_REPOSITORIES = ( + 'actor-python-playwright', + 'actor-python-playwright-chrome', + 'actor-python-playwright-firefox', + 'actor-python-playwright-webkit', +) +# A pinned version line, e.g. `# % set playwright_version = '1.60.0'`. +VERSION_LINE_TEMPLATE = r"(# % set {variable} = ')([^']+)(')" # The Python part of the base image tag, e.g. the `3.13` in `...:3.13-1.60.0`. PYTHON_PREFIX = re.compile(r'python-playwright[a-z-]*:(\d+\.\d+)-') -def fetch_tags() -> list[str]: - """Return all tag names of the Apify Playwright base image, following pagination.""" +def fetch_tags(repository: str) -> list[str]: + """Return all tag names of an Apify base image repository, following pagination.""" tags: list[str] = [] - url: str | None = TAGS_URL + url: str | None = TAGS_URL_TEMPLATE.format(repository=repository) while url: - with urllib.request.urlopen(url, timeout=30) as response: # noqa: S310 + with urllib.request.urlopen(url, timeout=REQUEST_TIMEOUT_SECONDS) as response: # noqa: S310 payload = json.load(response) tags.extend(result['name'] for result in payload['results']) url = payload['next'] return tags +def resolve_latest_version(repositories: tuple[str, ...], python_prefix: str) -> tuple[int, ...]: + """Return the highest stable version that every one of the given repositories publishes.""" + # Keep only stable `MAJOR.MINOR.PATCH` versions built for the template's current Python line. + tag_re = re.compile(rf'^{re.escape(python_prefix)}-(\d+\.\d+\.\d+)$') + published = { + repository: { + tuple(int(part) for part in match.group(1).split('.')) + for tag in fetch_tags(repository) + if (match := tag_re.match(tag)) + } + for repository in repositories + } + + without_stable_tag = [repository for repository, versions in published.items() if not versions] + if without_stable_tag: + images = ', '.join(f'apify/{repository}' for repository in without_stable_tag) + raise SystemExit(f'No stable {python_prefix}- tag is published by: {images}.') + + shared = set.intersection(*published.values()) + if not shared: + images = ', '.join(f'apify/{repository}' for repository in repositories) + raise SystemExit(f'No {python_prefix}- tag is shared by all of: {images}.') + return max(shared) + + +def validate_pinned_version(repository: str, python_prefix: str, pinned_version: str) -> None: + """Fail unless the repository publishes the exact pinned version for the Python line.""" + expected_tag = f'{python_prefix}-{pinned_version}' + if expected_tag not in fetch_tags(repository): + raise SystemExit(f'apify/{repository} does not publish the pinned tag {expected_tag}.') + + def main() -> None: - """Bump the pinned Playwright version in the template Dockerfile if a newer one is available.""" + """Bump the pinned Playwright versions in the template Dockerfile if newer ones are available.""" content = DOCKERFILE.read_text(encoding='utf-8') - version_match = VERSION_LINE.search(content) - if not version_match: - raise SystemExit(f'Pinned Playwright version line not found in {DOCKERFILE}.') - current = version_match.group(2) python_match = PYTHON_PREFIX.search(content) if not python_match: raise SystemExit(f'Python base image prefix not found in {DOCKERFILE}.') python_prefix = python_match.group(1) - # Keep only stable `MAJOR.MINOR.PATCH` versions built for the template's current Python line. - tag_re = re.compile(rf'^{re.escape(python_prefix)}-(\d+\.\d+\.\d+)$') - versions = [tuple(int(p) for p in m.group(1).split('.')) for tag in fetch_tags() if (m := tag_re.match(tag))] - if not versions: - raise SystemExit(f'No stable {python_prefix}- base image tags found.') - latest = max(versions) + shared_version_line = re.compile(VERSION_LINE_TEMPLATE.format(variable=SHARED_VERSION_VARIABLE)) + shared_version_match = shared_version_line.search(content) + if not shared_version_match: + raise SystemExit(f'Pinned {SHARED_VERSION_VARIABLE} line not found in {DOCKERFILE}.') + current = shared_version_match.group(2) + pinned = tuple(int(part) for part in current.split('.')) + + latest = resolve_latest_version(SHARED_VERSION_REPOSITORIES, python_prefix) latest_str = '.'.join(str(part) for part in latest) - if latest > tuple(int(part) for part in current.split('.')): - DOCKERFILE.write_text(VERSION_LINE.sub(rf'\g<1>{latest_str}\g<3>', content), encoding='utf-8') - print(f'Bumped Playwright version: {current} -> {latest_str}') + updated = content + if latest > pinned: + updated = shared_version_line.sub(rf'\g<1>{latest_str}\g<3>', content) + message = f'Bumped {SHARED_VERSION_VARIABLE}: {current} -> {latest_str}' + elif latest == pinned: + message = f'{SHARED_VERSION_VARIABLE} is already up to date ({current}).' else: - print(f'Playwright version is already up to date ({current}).') + # The pin is ahead of what the images publish, so the template would emit a `FROM` + # line for a tag that does not exist. Fail here rather than hours later in an e2e job. + images = ', '.join(f'apify/{repository}' for repository in SHARED_VERSION_REPOSITORIES) + raise SystemExit( + f'{SHARED_VERSION_VARIABLE} is pinned to {current}, but the newest tag published by all of ' + f'{images} is {latest_str}.' + ) + + camoufox_version_line = re.compile(VERSION_LINE_TEMPLATE.format(variable=CAMOUFOX_VERSION_VARIABLE)) + camoufox_version_match = camoufox_version_line.search(updated) + if not camoufox_version_match: + raise SystemExit(f'Pinned {CAMOUFOX_VERSION_VARIABLE} line not found in {DOCKERFILE}.') + validate_pinned_version(CAMOUFOX_REPOSITORY, python_prefix, camoufox_version_match.group(2)) + + # Only report the outcome once every validation has passed, so a failed run never prints a + # bump or an up-to-date message it did not actually persist. + if updated != content: + DOCKERFILE.write_text(updated, encoding='utf-8') + print(message) if __name__ == '__main__': diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile b/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile index 639e251387..0df452f063 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile @@ -6,10 +6,14 @@ `playwright` version resolved by the lockfile so the installed package matches the browser binaries shipped in the base image. #} # % set playwright_version = '1.60.0' +{# Camoufox ships its own Firefox build and caps the Playwright version it supports, so it keeps + an explicit compatible version instead of following the automatically updated shared pin. #} +# % set camoufox_playwright_version = '1.60.0' # % if cookiecutter.crawler_type == 'playwright' or cookiecutter.crawler_type.startswith('adaptive-') or cookiecutter.crawler_type == 'stagehand' # % set base_image = 'apify/actor-python-playwright:3.13-' ~ playwright_version # % elif cookiecutter.crawler_type == 'playwright-camoufox' +# % set playwright_version = camoufox_playwright_version # % set base_image = 'apify/actor-python-playwright-camoufox:3.13-' ~ playwright_version # % elif cookiecutter.crawler_type == 'playwright-chrome' # % set base_image = 'apify/actor-python-playwright-chrome:3.13-' ~ playwright_version diff --git a/tests/unit/scripts/test_update_playwright_version.py b/tests/unit/scripts/test_update_playwright_version.py new file mode 100644 index 0000000000..d2e90bf733 --- /dev/null +++ b/tests/unit/scripts/test_update_playwright_version.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import json +import re +from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse + +import pytest +from jinja2 import Environment + +from scripts import update_playwright_version + +if TYPE_CHECKING: + from collections.abc import Callable, Mapping + from pathlib import Path + +# Tags served by the fake Docker Hub, varying per repository so the tests can exercise which +# published version ends up shared across all four standard images. +TAGS_BY_REPOSITORY = { + 'actor-python-playwright': ['3.13-1.60.0', '3.13-1.61.0', '3.13-1.62.0'], + 'actor-python-playwright-chrome': ['3.13-1.60.0', '3.13-1.61.0'], + 'actor-python-playwright-firefox': ['3.13-1.60.0', '3.13-1.61.0', '3.13-1.62.0'], + 'actor-python-playwright-webkit': ['3.13-1.60.0', '3.13-1.61.0', '3.13-1.62.0'], + 'actor-python-playwright-camoufox': ['3.13-1.59.0', '3.13-1.60.0', '3.13-1.61.0'], +} +UNSTABLE_TAGS = ['3.13-beta', '3.13-latest'] +EXPECTED_SHARED_VERSION = '1.61.0' +EXPECTED_CAMOUFOX_VERSION = '1.60.0' +SHARED_VERSION_VARIABLE = 'playwright_version' +CAMOUFOX_VERSION_VARIABLE = 'camoufox_playwright_version' +CHROME_REPOSITORY = 'actor-python-playwright-chrome' +CAMOUFOX_REPOSITORY = 'actor-python-playwright-camoufox' +UNPUBLISHED_VERSION = '1.62.0' +# Position of the repository name in a Docker Hub tags path, `/v2/repositories/apify//tags`. +REPOSITORY_PATH_INDEX = 4 +# Cookiecutter renders the template with the Jinja settings declared by the template itself. +COOKIECUTTER_CONFIG = json.loads( + (update_playwright_version.DOCKERFILE.parent.parent / 'cookiecutter.json').read_text(encoding='utf-8') +) + + +class FakeResponse: + """Minimal stand-in for the context manager returned by `urllib.request.urlopen`.""" + + def __init__(self, payload: Mapping[str, Any]) -> None: + self._payload = payload + + def __enter__(self) -> FakeResponse: + return self + + def __exit__(self, *_: object) -> None: + return None + + def read(self) -> bytes: + return json.dumps(self._payload).encode() + + +def make_urlopen(tags_by_repository: Mapping[str, list[str]]) -> Callable[..., FakeResponse]: + """Build a `urlopen` replacement serving the given tags for each Apify image repository.""" + + def urlopen(url: str, **_: object) -> FakeResponse: + repository = urlparse(url).path.split('/')[REPOSITORY_PATH_INDEX] + tags = tags_by_repository[repository] + return FakeResponse({'results': [{'name': tag} for tag in tags], 'next': None}) + + return urlopen + + +@pytest.fixture +def template_copy(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Copy the template Dockerfile to a temporary path and point the script at the copy.""" + dockerfile = tmp_path / 'Dockerfile' + dockerfile.write_text(update_playwright_version.DOCKERFILE.read_text(encoding='utf-8'), encoding='utf-8') + monkeypatch.setattr(update_playwright_version, 'DOCKERFILE', dockerfile) + return dockerfile + + +def render_dockerfile(content: str, crawler_type: str) -> str: + """Render the template Dockerfile the way Cookiecutter does for the given crawler type.""" + template = Environment(**COOKIECUTTER_CONFIG['_jinja2_env_vars']).from_string(content) # noqa: S701 + return template.render( + cookiecutter={ + '__package_name': 'demo_project', + 'crawler_type': crawler_type, + 'package_manager': 'pip', + } + ) + + +def test_pins_track_the_image_repositories_they_select(template_copy: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The shared pin advances to the newest version common to every image it selects, while Camoufox keeps its own, + separately pinned version.""" + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(TAGS_BY_REPOSITORY)) + + update_playwright_version.main() + + content = template_copy.read_text(encoding='utf-8') + assert f"# % set {SHARED_VERSION_VARIABLE} = '{EXPECTED_SHARED_VERSION}'" in content + assert f"# % set {CAMOUFOX_VERSION_VARIABLE} = '{EXPECTED_CAMOUFOX_VERSION}'" in content + + +def test_reports_when_the_shared_pin_is_current( + template_copy: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Prints an up-to-date message, instead of bumping, when the shared pin matches the latest shared version.""" + content = template_copy.read_text(encoding='utf-8') + shared_line = re.compile(rf"(# % set {SHARED_VERSION_VARIABLE} = ')[^']+(')") + template_copy.write_text(shared_line.sub(rf'\g<1>{EXPECTED_SHARED_VERSION}\g<2>', content), encoding='utf-8') + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(TAGS_BY_REPOSITORY)) + + update_playwright_version.main() + + assert f'{SHARED_VERSION_VARIABLE} is already up to date ({EXPECTED_SHARED_VERSION}).' in capsys.readouterr().out + + +def test_generated_dockerfile_pins_camoufox_to_its_own_version( + template_copy: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The rendered Dockerfile pins Camoufox to its own version and every other crawler type to the shared version.""" + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(TAGS_BY_REPOSITORY)) + update_playwright_version.main() + content = template_copy.read_text(encoding='utf-8') + + camoufox = render_dockerfile(content, 'playwright-camoufox') + assert f'FROM apify/actor-python-playwright-camoufox:3.13-{EXPECTED_CAMOUFOX_VERSION}' in camoufox + assert f'playwright=={EXPECTED_CAMOUFOX_VERSION}' in camoufox + + chrome = render_dockerfile(content, 'playwright-chrome') + assert f'FROM apify/actor-python-playwright-chrome:3.13-{EXPECTED_SHARED_VERSION}' in chrome + assert f'playwright=={EXPECTED_SHARED_VERSION}' in chrome + + +def test_fails_when_the_shared_pinned_version_line_is_missing( + template_copy: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Exits when the Dockerfile has no pinned shared version line to read or update.""" + content = template_copy.read_text(encoding='utf-8') + shared_line = re.compile(rf"# % set {SHARED_VERSION_VARIABLE} = '[^']+'\n") + template_copy.write_text(shared_line.sub('', content), encoding='utf-8') + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(TAGS_BY_REPOSITORY)) + + with pytest.raises(SystemExit, match=SHARED_VERSION_VARIABLE): + update_playwright_version.main() + + +def test_fails_when_the_python_base_image_prefix_is_missing( + template_copy: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Exits when the Dockerfile has no Python base image prefix to resolve tags for.""" + content = template_copy.read_text(encoding='utf-8') + template_copy.write_text(content.replace('python-playwright', 'python-standard'), encoding='utf-8') + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(TAGS_BY_REPOSITORY)) + + with pytest.raises(SystemExit, match='Python base image prefix'): + update_playwright_version.main() + + +def test_fails_when_the_camoufox_pinned_version_line_is_missing( + template_copy: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Exits when the Dockerfile has no pinned Camoufox version line to validate.""" + content = template_copy.read_text(encoding='utf-8') + camoufox_line = re.compile(rf"# % set {CAMOUFOX_VERSION_VARIABLE} = '[^']+'\n") + template_copy.write_text(camoufox_line.sub('', content), encoding='utf-8') + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(TAGS_BY_REPOSITORY)) + + with pytest.raises(SystemExit, match=CAMOUFOX_VERSION_VARIABLE): + update_playwright_version.main() + + +def test_fails_when_the_camoufox_pinned_tag_is_not_published( + template_copy: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Exits when Camoufox's pinned tag is not published, even if the shared pin would bump.""" + original = template_copy.read_text(encoding='utf-8') + tags = {**TAGS_BY_REPOSITORY, CAMOUFOX_REPOSITORY: ['3.13-1.61.0']} + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(tags)) + + with pytest.raises(SystemExit, match=f'{CAMOUFOX_REPOSITORY}.*3.13-{EXPECTED_CAMOUFOX_VERSION}'): + update_playwright_version.main() + + # The shared pin's bump must not be persisted, nor reported, when the later Camoufox validation fails. + assert template_copy.read_text(encoding='utf-8') == original + assert capsys.readouterr().out == '' + + +@pytest.mark.usefixtures('template_copy') +def test_fails_when_an_image_has_no_stable_tag(monkeypatch: pytest.MonkeyPatch) -> None: + """Exits identifying the image that publishes no stable tag for the Python line.""" + # Chrome publishes no stable tag at all, so the shared pin cannot be resolved. + tags = {**TAGS_BY_REPOSITORY, CHROME_REPOSITORY: UNSTABLE_TAGS} + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(tags)) + + with pytest.raises(SystemExit, match=CHROME_REPOSITORY): + update_playwright_version.main() + + +@pytest.mark.usefixtures('template_copy') +def test_fails_when_no_version_is_shared_by_every_image(monkeypatch: pytest.MonkeyPatch) -> None: + """Exits when every image publishes a stable tag but no version is common to all of them.""" + # Chrome only publishes a version none of the other images have, so no version overlaps. + tags = {**TAGS_BY_REPOSITORY, CHROME_REPOSITORY: ['3.13-1.63.0']} + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(tags)) + + with pytest.raises(SystemExit, match=f'No 3.13- tag is shared by all of.*{CHROME_REPOSITORY}'): + update_playwright_version.main() + + +def test_fails_when_the_shared_pin_is_ahead_of_every_published_image( + template_copy: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Exits, without persisting anything, when the shared pin is newer than any published tag.""" + content = template_copy.read_text(encoding='utf-8') + shared_line = re.compile(rf"(# % set {SHARED_VERSION_VARIABLE} = ')[^']+(')") + template_copy.write_text(shared_line.sub(rf'\g<1>{UNPUBLISHED_VERSION}\g<2>', content), encoding='utf-8') + monkeypatch.setattr(update_playwright_version.urllib.request, 'urlopen', make_urlopen(TAGS_BY_REPOSITORY)) + + with pytest.raises(SystemExit, match=f'{UNPUBLISHED_VERSION}.*{CHROME_REPOSITORY}'): + update_playwright_version.main() + + assert f"# % set {SHARED_VERSION_VARIABLE} = '{UNPUBLISHED_VERSION}'" in template_copy.read_text(encoding='utf-8')