From d4faa54fddb76d7012d44b85b9b24c544e714ea9 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 22 Jan 2026 20:06:59 +0100 Subject: [PATCH 1/3] chore: Update to improved shared Python workflows Workflow improvements: - Update shared workflow input names to snake_case - Add tests_concurrency input for controlling parallel test execution - Use @improve-shared-python-workflows branch for testing Pytest configuration: - Standardize addopts to "-r a --verbose" - Remove --verbose from poe task commands (now in addopts) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/_check_code.yaml | 8 +++--- .github/workflows/_tests.yaml | 15 +++++----- .github/workflows/on_schedule_tests.yaml | 1 + pyproject.toml | 35 +++++++++++++++++++----- 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/.github/workflows/_check_code.yaml b/.github/workflows/_check_code.yaml index bb6b24e508..68914d485b 100644 --- a/.github/workflows/_check_code.yaml +++ b/.github/workflows/_check_code.yaml @@ -28,12 +28,12 @@ jobs: lint_check: name: Lint check - uses: apify/workflows/.github/workflows/python_lint_check.yaml@main + uses: apify/workflows/.github/workflows/python_lint_check.yaml@improve-shared-python-workflows with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' type_check: name: Type check - uses: apify/workflows/.github/workflows/python_type_check.yaml@main + uses: apify/workflows/.github/workflows/python_type_check.yaml@improve-shared-python-workflows with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml index 3165a30c0d..db0c8f518a 100644 --- a/.github/workflows/_tests.yaml +++ b/.github/workflows/_tests.yaml @@ -10,12 +10,11 @@ on: jobs: unit_tests: name: Unit tests - uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main - secrets: - httpbin_url: ${{ secrets.APIFY_HTTPBIN_TOKEN && format('https://httpbin.apify.actor?token={0}', secrets.APIFY_HTTPBIN_TOKEN) || 'https://httpbin.org' }} - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@improve-shared-python-workflows + secrets: inherit with: - python-versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' - operating-systems: '["ubuntu-latest", "windows-latest", "macos-latest"]' - python-version-for-codecov: "3.14" - operating-system-for-codecov: ubuntu-latest + python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' + operating_systems: '["ubuntu-latest", "windows-latest", "macos-latest"]' + python_version_for_codecov: "3.14" + operating_system_for_codecov: ubuntu-latest + tests_concurrency: "16" diff --git a/.github/workflows/on_schedule_tests.yaml b/.github/workflows/on_schedule_tests.yaml index fa50bdc468..de9fcdc733 100644 --- a/.github/workflows/on_schedule_tests.yaml +++ b/.github/workflows/on_schedule_tests.yaml @@ -15,6 +15,7 @@ concurrency: env: NODE_VERSION: 22 PYTHON_VERSION: 3.14 + TESTS_CONCURRENCY: 1 jobs: end_to_end_tests: diff --git a/pyproject.toml b/pyproject.toml index f228294f24..e3c2322781 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -217,10 +217,10 @@ builtins-ignorelist = ["id"] known-first-party = ["crawlee"] [tool.pytest.ini_options] -addopts = "-ra" +addopts = "-r a --verbose" asyncio_default_fixture_loop_scope = "function" asyncio_mode = "auto" -timeout = 300 +timeout = 1800 markers = [ "run_alone: marks tests that must run in isolation", ] @@ -277,18 +277,39 @@ shell = "uv run ruff check --fix && uv run ruff format" [tool.poe.tasks.unit-tests] shell = """ -uv run pytest --numprocesses=1 --verbose -m "run_alone" tests/unit && \ -uv run pytest --numprocesses=auto --verbose -m "not run_alone" tests/unit +uv run pytest \ + --numprocesses=1 \ + -m "run_alone" \ + tests/unit && \ +uv run pytest \ + --numprocesses=${TESTS_CONCURRENCY:-auto} \ + -m "not run_alone" \ + tests/unit """ [tool.poe.tasks.unit-tests-cov] shell = """ -uv run pytest --numprocesses=1 --verbose -m "run_alone" --cov=src/crawlee --cov-report=xml:coverage-unit.xml tests/unit && \ -uv run pytest --numprocesses=auto --verbose -m "not run_alone" --cov=src/crawlee --cov-report=xml:coverage-unit.xml --cov-append tests/unit +uv run pytest \ + --numprocesses=1 \ + -m "run_alone" \ + --cov=src/crawlee \ + --cov-report=xml:coverage-unit.xml \ + tests/unit && \ +uv run pytest \ + --numprocesses=${TESTS_CONCURRENCY:-auto} \ + -m "not run_alone" \ + --cov=src/crawlee \ + --cov-report=xml:coverage-unit.xml \ + --cov-append \ + tests/unit """ [tool.poe.tasks.e2e-templates-tests] -cmd = "uv run pytest --numprocesses=${E2E_TESTS_CONCURRENCY:-1} --verbose tests/e2e/project_template --timeout=600" +cmd = """ +uv run pytest \ + --numprocesses=${TESTS_CONCURRENCY:-auto} \ + tests/e2e/project_template +""" [tool.poe.tasks.build-docs] shell = "./build_api_reference.sh && corepack enable && yarn && yarn build" From 3ec78c282f11a5eedabee5059d0e2522d54aa669 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 23 Jan 2026 09:54:23 +0100 Subject: [PATCH 2/3] main branch --- .github/workflows/_check_code.yaml | 4 ++-- .github/workflows/_tests.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/_check_code.yaml b/.github/workflows/_check_code.yaml index 68914d485b..be9db96778 100644 --- a/.github/workflows/_check_code.yaml +++ b/.github/workflows/_check_code.yaml @@ -28,12 +28,12 @@ jobs: lint_check: name: Lint check - uses: apify/workflows/.github/workflows/python_lint_check.yaml@improve-shared-python-workflows + uses: apify/workflows/.github/workflows/python_lint_check.yaml@main with: python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' type_check: name: Type check - uses: apify/workflows/.github/workflows/python_type_check.yaml@improve-shared-python-workflows + uses: apify/workflows/.github/workflows/python_type_check.yaml@main with: python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml index db0c8f518a..792d8d885c 100644 --- a/.github/workflows/_tests.yaml +++ b/.github/workflows/_tests.yaml @@ -10,7 +10,7 @@ on: jobs: unit_tests: name: Unit tests - uses: apify/workflows/.github/workflows/python_unit_tests.yaml@improve-shared-python-workflows + uses: apify/workflows/.github/workflows/python_unit_tests.yaml@main secrets: inherit with: python_versions: '["3.10", "3.11", "3.12", "3.13", "3.14"]' From e47458823224b3a4175125b975af454a9504104e Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 23 Jan 2026 10:27:39 +0100 Subject: [PATCH 3/3] only 2 concurrency --- .github/workflows/_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_tests.yaml b/.github/workflows/_tests.yaml index 792d8d885c..2582500647 100644 --- a/.github/workflows/_tests.yaml +++ b/.github/workflows/_tests.yaml @@ -17,4 +17,4 @@ jobs: operating_systems: '["ubuntu-latest", "windows-latest", "macos-latest"]' python_version_for_codecov: "3.14" operating_system_for_codecov: ubuntu-latest - tests_concurrency: "16" + tests_concurrency: "2"