diff --git a/.github/workflows/_release_docs.yaml b/.github/workflows/_release_docs.yaml index af809838..afd9abea 100644 --- a/.github/workflows/_release_docs.yaml +++ b/.github/workflows/_release_docs.yaml @@ -68,13 +68,10 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install Python dependencies - run: make install-dev + run: uv run poe install-dev - - name: Build generated API reference - run: make build-api-reference - - - name: Build Docusaurus docs - run: make build-docs + - name: Build docs + run: uv run poe build-docs env: APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 47a52fc0..c823138b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,12 +3,12 @@ repos: hooks: - id: lint-check name: Lint check - entry: make lint + entry: uv run poe lint language: system pass_filenames: false - id: type-check name: Type check - entry: make type-check + entry: uv run poe type-check language: system pass_filenames: false diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4fd98ddf..86b6cd9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,12 +8,33 @@ For local development, it is required to have Python 3.10 (or a later version) i We use [uv](https://docs.astral.sh/uv/) for project management. Install it and set up your IDE accordingly. +We use [Poe the Poet](https://poethepoet.natn.io/) as a task runner, similar to npm scripts in `package.json`. +All tasks are defined in `pyproject.toml` under `[tool.poe.tasks]` and can be run with `uv run poe `. + +### Available tasks + +| Task | Description | +| ---- | ----------- | +| `install-dev` | Install development dependencies | +| `check-code` | Run lint, type-check, and unit-tests | +| `lint` | Run linter | +| `format` | Fix lint issues and format code | +| `type-check` | Run type checker | +| `unit-tests` | Run unit tests | +| `unit-tests-cov` | Run unit tests with coverage | +| `integration-tests` | Run integration tests | +| `integration-tests-cov` | Run integration tests with coverage | +| `build-docs` | Build documentation website | +| `run-docs` | Run documentation website locally | +| `build` | Build package | +| `clean` | Remove build artifacts and clean caches | + ## Dependencies To install this package and its development dependencies, run: ```sh -make install-dev +uv run poe install-dev ``` ## Code checking @@ -21,7 +42,7 @@ make install-dev To execute all code checking tools together, run: ```sh -make check-code +uv run poe check-code ``` ### Linting @@ -31,7 +52,7 @@ We utilize [ruff](https://docs.astral.sh/ruff/) for linting, which analyzes code To run linting: ```sh -make lint +uv run poe lint ``` ### Formatting @@ -41,7 +62,7 @@ Our automated code formatting also leverages [ruff](https://docs.astral.sh/ruff/ To run formatting: ```sh -make format +uv run poe format ``` ### Type checking @@ -51,78 +72,50 @@ Type checking is handled by [ty](https://docs.astral.sh/ty/), verifying code aga To run type checking: ```sh -make type-check +uv run poe type-check ``` ### Unit tests -We employ pytest as our testing framework, equipped with various plugins. Check pyproject.toml for configuration details and installed plugins. - We use [pytest](https://docs.pytest.org/) as a testing framework with many plugins. Check `pyproject.toml` for configuration details and installed plugins. To run unit tests: ```sh -make unit-tests +uv run poe unit-tests ``` -To run unit tests with HTML coverage report: +To run unit tests with XML coverage report: ```sh -make unit-tests-cov +uv run poe unit-tests-cov ``` ## Integration tests -We have integration tests which build and run Actors using the Python SDK on the Apify Platform. To run these tests, -you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to -use for the tests, and then start them with `make integration-tests`. - -If you want to run the integration tests on a different environment than the main Apify Platform, you need to set -the `APIFY_INTEGRATION_TESTS_API_URL` environment variable to the right URL to the Apify API you want to use. - -## Documentation - -We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). - -Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages. - -To run the documentation locally, you need to have Node.js version 20 or higher installed. Once you have the correct version of Node.js, follow these steps: - -Navigate to the `website/` directory: - -```sh -cd website/ -``` +We have integration tests which build and run Actors using the Python SDK on the Apify platform. -Enable Corepack, which installs Yarn automatically: +Prerequisites: -```sh -corepack enable -``` +- Set `APIFY_TEST_USER_API_TOKEN` to your Apify API token +- Optionally set `APIFY_INTEGRATION_TESTS_API_URL` to use a different Apify API environment -Build the API reference: +To run integration tests: ```sh -./build_api_reference.sh +uv run poe integration-tests ``` -Install the necessary dependencies: - -```sh -yarn -``` +## Documentation -Start the project in development mode with Hot Module Replacement (HMR): +We adhere to the [Google docstring format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) for documenting our codebase. Every user-facing class or method is documented. Documentation standards are enforced using [Ruff](https://docs.astral.sh/ruff/). -```sh -yarn start -``` +Our API documentation is generated from these docstrings using [pydoc-markdown](https://pypi.org/project/pydoc-markdown/) with additional post-processing. Markdown files in the `docs/` folder complement the autogenerated content. Final documentation is rendered using [Docusaurus](https://docusaurus.io/) and published to GitHub Pages. -Or using `make`: +To run the documentation locally (requires Node.js): ```sh -make run-doc +uv run poe run-docs ``` ## Release process @@ -147,14 +140,14 @@ name = "apify" version = "x.z.y" ``` -4. Generate the distribution archives for the package: +4. Build the package: -```shell -uv build +```sh +uv run poe build ``` -5. Set up the PyPI API token for authentication and upload the package to PyPI: +5. Upload to PyPI: -```shell +```sh uv publish --token YOUR_API_TOKEN ``` diff --git a/Makefile b/Makefile deleted file mode 100644 index ed52ef79..00000000 --- a/Makefile +++ /dev/null @@ -1,71 +0,0 @@ -.PHONY: clean install-dev build publish-to-pypi lint type-check unit-tests unit-tests-cov integration-tests \ - integration-tests-cov format check-code build-api-reference build-docs run-docs - -# This is default for local testing, but GitHub workflows override it to a higher value in CI -INTEGRATION_TESTS_CONCURRENCY = 1 - -clean: - rm -rf .ty_cache .pytest_cache .ruff_cache build dist htmlcov .coverage - -install-dev: - uv sync --all-extras - uv run pre-commit install - -build: - uv build --verbose - -# APIFY_PYPI_TOKEN_CRAWLEE is expected to be set in the environment -publish-to-pypi: - uv publish --verbose --token "${APIFY_PYPI_TOKEN_CRAWLEE}" - -lint: - uv run ruff format --check - uv run ruff check - -type-check: - uv run ty check - -unit-tests: - uv run pytest \ - --numprocesses=auto \ - --verbose \ - tests/unit - -unit-tests-cov: - uv run pytest \ - --numprocesses=auto \ - --verbose \ - --cov=src/apify \ - --cov-report=xml:coverage-unit.xml \ - tests/unit - -integration-tests: - uv run pytest \ - --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) \ - --verbose \ - tests/integration - -integration-tests-cov: - uv run pytest \ - --numprocesses=$(INTEGRATION_TESTS_CONCURRENCY) \ - --verbose \ - --cov=src/apify \ - --cov-report=xml:coverage-integration.xml \ - tests/integration - -format: - uv run ruff check --fix - uv run ruff format - -# The check-code target runs a series of checks equivalent to those performed by pre-commit hooks -# and the run_checks.yaml GitHub Actions workflow. -check-code: lint type-check unit-tests - -build-api-reference: - cd website && uv run ./build_api_reference.sh - -build-docs: - cd website && uv run npm clean-install && uv run npm run build - -run-docs: build-api-reference - cd website && uv run npm clean-install && uv run npm run start diff --git a/docs/02_concepts/12_pay_per_event.mdx b/docs/02_concepts/12_pay_per_event.mdx index 503d3158..1c1653b1 100644 --- a/docs/02_concepts/12_pay_per_event.mdx +++ b/docs/02_concepts/12_pay_per_event.mdx @@ -43,7 +43,7 @@ When you plan to start using the pay-per-event pricing model for an Actor that i It is encouraged to test your monetization code on your machine before releasing it to the public. To tell your Actor that it should work in pay-per-event mode, pass it the `ACTOR_TEST_PAY_PER_EVENT` environment variable: -```shell +```sh ACTOR_TEST_PAY_PER_EVENT=true python -m youractor ``` diff --git a/pyproject.toml b/pyproject.toml index 44855d2e..b524d7c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,7 @@ dev = [ "crawlee[parsel]", "dycw-pytest-only<3.0.0", "griffe", + "poethepoet<1.0.0", "pre-commit<5.0.0", "pydoc-markdown<5.0.0", "pytest-asyncio<2.0.0", @@ -216,3 +217,33 @@ exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"] [tool.ipdb] context = 7 + +# Run tasks with: uv run poe +[tool.poe.tasks] +clean = "rm -rf .coverage .pytest_cache .ruff_cache .ty_cache build dist htmlcov" +install-sync = "uv sync --all-extras" +build = "uv build --verbose" +publish-to-pypi = "uv publish --verbose --token ${APIFY_PYPI_TOKEN_CRAWLEE}" +type-check = "uv run ty check" +unit-tests = "uv run pytest --numprocesses=auto --verbose tests/unit" +unit-tests-cov = "uv run pytest --numprocesses=auto --verbose --cov=src/apify --cov-report=xml:coverage-unit.xml tests/unit" +integration-tests = "uv run pytest --numprocesses=${INTEGRATION_TESTS_CONCURRENCY:-1} --verbose tests/integration" +integration-tests-cov = "uv run pytest --numprocesses=${INTEGRATION_TESTS_CONCURRENCY:-1} --verbose --cov=src/apify --cov-report=xml:coverage-integration.xml tests/integration" +check-code = ["lint", "type-check", "unit-tests"] + +[tool.poe.tasks.install-dev] +shell = "uv sync --all-extras && uv run pre-commit install" + +[tool.poe.tasks.lint] +shell = "uv run ruff format --check && uv run ruff check" + +[tool.poe.tasks.format] +shell = "uv run ruff check --fix && uv run ruff format" + +[tool.poe.tasks.build-docs] +shell = "./build_api_reference.sh && npm ci && npm run build" +cwd = "website" + +[tool.poe.tasks.run-docs] +shell = "./build_api_reference.sh && npm ci && npm run start" +cwd = "website" diff --git a/tests/integration/README.md b/tests/integration/README.md index 4e07ec51..46e3b433 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -4,7 +4,7 @@ There are two different groups of integration tests in this repository: - Apify API integration tests. These test that the Apify SDK is correctly communicating with Apify API through Apify client. - Actor integration tests. These test that the Apify SDK can be used in Actors deployed to Apify platform. These are very high level tests, and they test communication with the API and correct interaction with the Apify platform. -To run these tests, you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to use for the tests, and then start them with `make integration-tests`. +To run these tests, you need to set the `APIFY_TEST_USER_API_TOKEN` environment variable to the API token of the Apify user you want to use for the tests, and then start them with `uv run poe integration-tests`. ## Apify API integration tests The tests are making real requests to the Apify API as opposed to the unit tests that are mocking such API calls. On the other hand they are faster than `Actor integration tests` as they do not require building and deploying the Actor. These test can be also fully debugged locally. Preferably try to write integration tests on this level if possible. diff --git a/uv.lock b/uv.lock index c9cde3d5..3785550f 100644 --- a/uv.lock +++ b/uv.lock @@ -55,6 +55,7 @@ dev = [ { name = "crawlee", extra = ["parsel"] }, { name = "dycw-pytest-only" }, { name = "griffe" }, + { name = "poethepoet" }, { name = "pre-commit" }, { name = "pydoc-markdown" }, { name = "pytest" }, @@ -95,6 +96,7 @@ dev = [ { name = "crawlee", extras = ["parsel"] }, { name = "dycw-pytest-only", specifier = "<3.0.0" }, { name = "griffe" }, + { name = "poethepoet", specifier = "<1.0.0" }, { name = "pre-commit", specifier = "<5.0.0" }, { name = "pydoc-markdown", specifier = "<5.0.0" }, { name = "pytest", specifier = "<9.0.0" }, @@ -1500,6 +1502,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/18/35d1d947553d24909dca37e2ff11720eecb601360d1bac8d7a9a1bc7eb08/parsel-1.10.0-py2.py3-none-any.whl", hash = "sha256:6a0c28bd81f9df34ba665884c88efa0b18b8d2c44c81f64e27f2f0cb37d46169", size = 17266, upload-time = "2025-01-17T15:38:27.83Z" }, ] +[[package]] +name = "pastel" +version = "0.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/f1/4594f5e0fcddb6953e5b8fe00da8c317b8b41b547e2b3ae2da7512943c62/pastel-0.2.1.tar.gz", hash = "sha256:e6581ac04e973cac858828c6202c1e1e81fee1dc7de7683f3e1ffe0bfd8a573d", size = 7555, upload-time = "2020-09-16T19:21:12.43Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/18/a8444036c6dd65ba3624c63b734d3ba95ba63ace513078e1580590075d21/pastel-0.2.1-py2.py3-none-any.whl", hash = "sha256:4349225fcdf6c2bb34d483e523475de5bb04a5c10ef711263452cb37d7dd4364", size = 5955, upload-time = "2020-09-16T19:21:11.409Z" }, +] + [[package]] name = "pathspec" version = "1.0.3" @@ -1527,6 +1538,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "poethepoet" +version = "0.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pastel" }, + { name = "pyyaml" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/9d/054c8435b03324ed9abd5d5ab8c45065b1f42c23952cd23f13a5921d8465/poethepoet-0.40.0.tar.gz", hash = "sha256:91835f00d03d6c4f0e146f80fa510e298ad865e7edd27fe4cb9c94fdc090791b", size = 81114, upload-time = "2026-01-05T19:09:13.116Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/bc/73327d12b176abea7a3c6c7d760e1a953992f7b59d72c0354e39d7a353b5/poethepoet-0.40.0-py3-none-any.whl", hash = "sha256:afd276ae31d5c53573c0c14898118d4848ccee3709b6b0be6a1c6cbe522bbc8a", size = 106672, upload-time = "2026-01-05T19:09:11.536Z" }, +] + [[package]] name = "pre-commit" version = "4.5.1"