diff --git a/docs/by-example/client.rst b/docs/by-example/client.rst index a06e1036..995ee745 100644 --- a/docs/by-example/client.rst +++ b/docs/by-example/client.rst @@ -25,7 +25,7 @@ replication. In order for clients to make use of this property it is recommended to specify all hosts of the cluster. This way if a server does not respond, the request is automatically routed to the next server: - >>> invalid_host = 'http://not_responding_host:4200' + >>> invalid_host = 'http://127.0.0.1:4201' >>> connection = client.connect([invalid_host, crate_host]) >>> connection.close() @@ -49,7 +49,7 @@ It's possible to define a default timeout value in seconds for all servers using the optional parameter ``timeout``. In this case, it will serve as a total timeout (connect and read): - >>> connection = client.connect([crate_host, invalid_host], timeout=5) + >>> connection = client.connect([crate_host, invalid_host], timeout=1) >>> connection.close() If you want to adjust the connect- vs. read-timeout values individually, @@ -58,7 +58,7 @@ please use the ``urllib3.Timeout`` object like: >>> import urllib3 >>> connection = client.connect( ... [crate_host, invalid_host], - ... timeout=urllib3.Timeout(connect=5, read=None)) + ... timeout=urllib3.Timeout(connect=1, read=None)) >>> connection.close() Authentication diff --git a/docs/by-example/http.rst b/docs/by-example/http.rst index 75a9e9ef..6a067eaa 100644 --- a/docs/by-example/http.rst +++ b/docs/by-example/http.rst @@ -34,23 +34,23 @@ If no ``server`` argument (or no argument at all) is passed, the default one When using a list of servers, the servers are selected by round-robin: - >>> invalid_host = "invalid_host:9999" - >>> even_more_invalid_host = "even_more_invalid_host:9999" - >>> http_client = HttpClient([crate_host, invalid_host, even_more_invalid_host], timeout=0.3) + >>> invalid_host1 = "192.0.2.1:9999" + >>> invalid_host2 = "192.0.2.2:9999" + >>> http_client = HttpClient([crate_host, invalid_host1, invalid_host2], timeout=0.3) >>> http_client._get_server() 'http://127.0.0.1:44209' >>> http_client._get_server() - 'http://invalid_host:9999' + 'http://192.0.2.1:9999' >>> http_client._get_server() - 'http://even_more_invalid_host:9999' + 'http://192.0.2.2:9999' >>> http_client.close() Servers with connection errors will be removed from the active server list: - >>> http_client = HttpClient([invalid_host, even_more_invalid_host, crate_host], timeout=0.3) + >>> http_client = HttpClient([invalid_host1, invalid_host2, crate_host], timeout=0.3) >>> result = http_client.sql('select name from locations') >>> http_client._active_servers ['http://127.0.0.1:44209'] @@ -64,19 +64,17 @@ sleep after the first request:: >>> import time; time.sleep(1) >>> server = http_client._get_server() >>> http_client._active_servers - ['http://invalid_host:9999', - 'http://even_more_invalid_host:9999', - 'http://127.0.0.1:44209'] + ['http://127.0.0.1:44209', 'http://192.0.2.2:9999', 'http://192.0.2.1:9999'] >>> http_client.close() If no active servers are available and the retry interval is not reached, just use the oldest inactive one: - >>> http_client = HttpClient([invalid_host, even_more_invalid_host, crate_host], timeout=0.3) + >>> http_client = HttpClient([invalid_host1, invalid_host2, crate_host], timeout=0.3) >>> result = http_client.sql('select name from locations') >>> http_client._active_servers = [] >>> http_client._get_server() - 'http://invalid_host:9999' + 'http://192.0.2.1:9999' >>> http_client.close() SQL Statements diff --git a/docs/by-example/https.rst b/docs/by-example/https.rst index b82db341..116dabb8 100644 --- a/docs/by-example/https.rst +++ b/docs/by-example/https.rst @@ -36,22 +36,22 @@ With certificate verification When using a valid CA certificate, the connection will be successful: - >>> client = HttpClient([crate_host], ca_cert=cacert_valid) + >>> client = HttpClient([https_host], ca_cert=cacert_valid) >>> client.server_infos(client._get_server()) ('https://localhost:65534', 'test', '0.0.0') When not providing a ``ca_cert`` file, the connection will fail: - >>> client = HttpClient([crate_host]) - >>> client.server_infos(crate_host) + >>> client = HttpClient([https_host]) + >>> client.server_infos(https_host) Traceback (most recent call last): ... crate.client.exceptions.ConnectionError: Server not available, ...certificate verify failed... Also, when providing an invalid ``ca_cert``, an error is raised: - >>> client = HttpClient([crate_host], ca_cert=cacert_invalid) - >>> client.server_infos(crate_host) + >>> client = HttpClient([https_host], ca_cert=cacert_invalid) + >>> client.server_infos(https_host) Traceback (most recent call last): ... crate.client.exceptions.ConnectionError: Server not available, ...certificate verify failed... @@ -63,15 +63,15 @@ Without certificate verification When turning off certificate verification, calling the server will succeed, even when not providing a valid CA certificate: - >>> client = HttpClient([crate_host], verify_ssl_cert=False) - >>> client.server_infos(crate_host) + >>> client = HttpClient([https_host], verify_ssl_cert=False) + >>> client.server_infos(https_host) ('https://localhost:65534', 'test', '0.0.0') Without verification, calling the server will even work when using an invalid ``ca_cert``: - >>> client = HttpClient([crate_host], verify_ssl_cert=False, ca_cert=cacert_invalid) - >>> client.server_infos(crate_host) + >>> client = HttpClient([https_host], verify_ssl_cert=False, ca_cert=cacert_invalid) + >>> client.server_infos(https_host) ('https://localhost:65534', 'test', '0.0.0') @@ -85,22 +85,22 @@ The ``HttpClient`` constructor takes two keyword arguments: ``cert_file`` and ``key_file``. Both should be strings pointing to the path of the client certificate and key file: - >>> client = HttpClient([crate_host], ca_cert=cacert_valid, cert_file=clientcert_valid, key_file=clientcert_valid) - >>> client.server_infos(crate_host) + >>> client = HttpClient([https_host], ca_cert=cacert_valid, cert_file=clientcert_valid, key_file=clientcert_valid) + >>> client.server_infos(https_host) ('https://localhost:65534', 'test', '0.0.0') When using an invalid client certificate, the connection will fail: - >>> client = HttpClient([crate_host], ca_cert=cacert_valid, cert_file=clientcert_invalid, key_file=clientcert_invalid) - >>> client.server_infos(crate_host) + >>> client = HttpClient([https_host], ca_cert=cacert_valid, cert_file=clientcert_invalid, key_file=clientcert_invalid) + >>> client.server_infos(https_host) Traceback (most recent call last): ... crate.client.exceptions.ConnectionError: Server not available, exception: HTTPSConnectionPool... The connection will also fail when providing an invalid CA certificate: - >>> client = HttpClient([crate_host], ca_cert=cacert_invalid, cert_file=clientcert_valid, key_file=clientcert_valid) - >>> client.server_infos(crate_host) + >>> client = HttpClient([https_host], ca_cert=cacert_invalid, cert_file=clientcert_valid, key_file=clientcert_valid) + >>> client.server_infos(https_host) Traceback (most recent call last): ... crate.client.exceptions.ConnectionError: Server not available, exception: HTTPSConnectionPool... @@ -113,8 +113,8 @@ urrlib3 v2 dropped support for TLS 1.0 and TLS 1.1 by default, see `Modern secur HTTPS requires TLS 1.2+`_. If you need to re-enable it, use the ``ssl_relax_minimum_version`` flag, which will configure ``kwargs["ssl_minimum_version"] = ssl.TLSVersion.MINIMUM_SUPPORTED``. - >>> client = HttpClient([crate_host], ssl_relax_minimum_version=True, verify_ssl_cert=False) - >>> client.server_infos(crate_host) + >>> client = HttpClient([https_host], ssl_relax_minimum_version=True, verify_ssl_cert=False) + >>> client.server_infos(https_host) ('https://localhost:65534', 'test', '0.0.0') diff --git a/pyproject.toml b/pyproject.toml index 5cd5c1af..7ffcd94e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,6 @@ dev = [ "pytest<10", "pytz", "ruff<0.15", - "stopit<1.2", ] docs = [ "sphinx", @@ -107,7 +106,8 @@ non_interactive = true [tool.pytest.ini_options] -addopts = "-rA --verbosity=3" +addopts = "-rA --verbosity=3 --doctest-modules" +doctest_optionflags = "NORMALIZE_WHITESPACE ELLIPSIS" minversion = "2.0" testpaths = [ "tests", diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index 6ea9464d..960a4889 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -34,6 +34,7 @@ import tempfile import threading import time +from typing import Optional import urllib3 @@ -242,7 +243,7 @@ def __init__( else: self.http_url = http_url_from_host_port(host, port) - self.process = None + self.process: Optional[subprocess.Popen] = None self.verbose = verbose self.env = env or {} self.env.setdefault("CRATE_USE_IPV4", "true") @@ -364,7 +365,9 @@ def stop(self): if self.process: self.process.terminate() self.process.communicate(timeout=10) - self.process.stdout.close() + stdout = self.process.stdout + if stdout: + stdout.close() self.process = None self.monitor.stop() self._clean() diff --git a/tests/client/layer.py b/tests/client/layer.py deleted file mode 100644 index c381299d..00000000 --- a/tests/client/layer.py +++ /dev/null @@ -1,278 +0,0 @@ -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - -from __future__ import absolute_import - -import json -import logging -import socket -import ssl -import threading -import time -import unittest -from http.server import BaseHTTPRequestHandler, HTTPServer -from pprint import pprint - -import stopit - -from crate.client import connect -from crate.testing.layer import CrateLayer - -from .settings import ( - assets_path, - crate_host, - crate_path, - crate_port, - crate_transport_port, - localhost, -) - -makeSuite = unittest.TestLoader().loadTestsFromTestCase - -log = logging.getLogger("crate.testing.layer") -ch = logging.StreamHandler() -ch.setLevel(logging.ERROR) -log.addHandler(ch) - - -def cprint(s): - if isinstance(s, bytes): - s = s.decode("utf-8") - print(s) # noqa: T201 - - -settings = { - "udc.enabled": "false", - "lang.js.enabled": "true", - "auth.host_based.enabled": "true", - "auth.host_based.config.0.user": "crate", - "auth.host_based.config.0.method": "trust", - "auth.host_based.config.98.user": "trusted_me", - "auth.host_based.config.98.method": "trust", - "auth.host_based.config.99.user": "me", - "auth.host_based.config.99.method": "password", -} -crate_layer = None - - -def ensure_cratedb_layer(): - """ - In order to skip individual tests by manually disabling them within - `def test_suite()`, it is crucial make the test layer not run on each - and every occasion. So, things like this will be possible:: - - ./bin/test -vvvv --ignore_dir=testing - - TODO: Through a subsequent patch, the possibility to individually - unselect specific tests might be added to `def test_suite()` - on behalf of environment variables. - A blueprint for this kind of logic can be found at - https://github.com/crate/crate/commit/414cd833. - """ - global crate_layer - - if crate_layer is None: - crate_layer = CrateLayer( - "crate", - crate_home=crate_path(), - port=crate_port, - host=localhost, - transport_port=crate_transport_port, - settings=settings, - ) - return crate_layer - - -def setUpCrateLayerBaseline(test): - if hasattr(test, "globs"): - test.globs["crate_host"] = crate_host - test.globs["pprint"] = pprint - test.globs["print"] = cprint - - with connect(crate_host) as conn: - cursor = conn.cursor() - - with open(assets_path("mappings/locations.sql")) as s: - stmt = s.read() - cursor.execute(stmt) - stmt = ( - "select count(*) from information_schema.tables " - "where table_name = 'locations'" - ) - cursor.execute(stmt) - assert cursor.fetchall()[0][0] == 1 # noqa: S101 - - data_path = assets_path("import/test_a.json") - # load testing data into crate - cursor.execute("copy locations from ?", (data_path,)) - # refresh location table so imported data is visible immediately - cursor.execute("refresh table locations") - # create blob table - cursor.execute( - "create blob table myfiles clustered into 1 shards " - + "with (number_of_replicas=0)" - ) - - # create users - cursor.execute("CREATE USER me WITH (password = 'my_secret_pw')") - cursor.execute("CREATE USER trusted_me") - - cursor.close() - - -def tearDownDropEntitiesBaseline(test): - """ - Drop all tables, views, and users created by `setUpWithCrateLayer*`. - """ - ddl_statements = [ - "DROP TABLE foobar", - "DROP TABLE locations", - "DROP BLOB TABLE myfiles", - "DROP USER me", - "DROP USER trusted_me", - ] - _execute_statements(ddl_statements) - - -class HttpsTestServerLayer: - PORT = 65534 - HOST = "localhost" - CERT_FILE = assets_path("pki/server_valid.pem") - CACERT_FILE = assets_path("pki/cacert_valid.pem") - - __name__ = "httpsserver" - __bases__ = () - - class HttpsServer(HTTPServer): - def get_request(self): - # Prepare SSL context. - context = ssl._create_unverified_context( # noqa: S323 - protocol=ssl.PROTOCOL_TLS_SERVER, - cert_reqs=ssl.CERT_OPTIONAL, - check_hostname=False, - purpose=ssl.Purpose.CLIENT_AUTH, - certfile=HttpsTestServerLayer.CERT_FILE, - keyfile=HttpsTestServerLayer.CERT_FILE, - cafile=HttpsTestServerLayer.CACERT_FILE, - ) # noqa: S323 - - # Set minimum protocol version, TLSv1 and TLSv1.1 are unsafe. - context.minimum_version = ssl.TLSVersion.TLSv1_2 - - # Wrap TLS encryption around socket. - socket, client_address = HTTPServer.get_request(self) - socket = context.wrap_socket(socket, server_side=True) - - return socket, client_address - - class HttpsHandler(BaseHTTPRequestHandler): - payload = json.dumps( - { - "name": "test", - "status": 200, - } - ) - - def do_GET(self): - self.send_response(200) - payload = self.payload.encode("UTF-8") - self.send_header("Content-Length", len(payload)) - self.send_header("Content-Type", "application/json; charset=UTF-8") - self.end_headers() - self.wfile.write(payload) - - def setUp(self): - self.server = self.HttpsServer( - (self.HOST, self.PORT), self.HttpsHandler - ) - thread = threading.Thread(target=self.serve_forever) - thread.daemon = True # quit interpreter when only thread exists - thread.start() - self.waitForServer() - - def serve_forever(self): - log.info("listening on", self.HOST, self.PORT) - self.server.serve_forever() - log.info("server stopped.") - - def tearDown(self): - self.server.shutdown() - self.server.server_close() - - def isUp(self): - """ - Test if a host is up. - """ - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ex = s.connect_ex((self.HOST, self.PORT)) - s.close() - return ex == 0 - - def waitForServer(self, timeout=5): - """ - Wait for the host to be available. - """ - with stopit.ThreadingTimeout(timeout) as to_ctx_mgr: - while True: - if self.isUp(): - break - time.sleep(0.001) - - if not to_ctx_mgr: - raise TimeoutError( - "Could not properly start embedded webserver " - "within {} seconds".format(timeout) - ) - - -def setUpWithHttps(test): - test.globs["crate_host"] = "https://{0}:{1}".format( - HttpsTestServerLayer.HOST, HttpsTestServerLayer.PORT - ) - test.globs["pprint"] = pprint - test.globs["print"] = cprint - - test.globs["cacert_valid"] = assets_path("pki/cacert_valid.pem") - test.globs["cacert_invalid"] = assets_path("pki/cacert_invalid.pem") - test.globs["clientcert_valid"] = assets_path("pki/client_valid.pem") - test.globs["clientcert_invalid"] = assets_path("pki/client_invalid.pem") - - -def _execute_statements(statements, on_error="ignore"): - with connect(crate_host) as conn: - cursor = conn.cursor() - for stmt in statements: - _execute_statement(cursor, stmt, on_error=on_error) - cursor.close() - - -def _execute_statement(cursor, stmt, on_error="ignore"): - try: - cursor.execute(stmt) - except Exception: # pragma: no cover - # FIXME: Why does this trip on statements like `DROP TABLE cities`? - # Note: When needing to debug the test environment, you may want to - # enable this logger statement. - # log.exception("Executing SQL statement failed") # noqa: ERA001 - if on_error == "ignore": - pass - elif on_error == "raise": - raise diff --git a/tests/client/tests.py b/tests/client/tests.py deleted file mode 100644 index ae6a479f..00000000 --- a/tests/client/tests.py +++ /dev/null @@ -1,56 +0,0 @@ -import doctest -import unittest - -from .layer import ( - HttpsTestServerLayer, - ensure_cratedb_layer, - setUpCrateLayerBaseline, - setUpWithHttps, - tearDownDropEntitiesBaseline, -) - - -def test_suite(): - suite = unittest.TestSuite() - flags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS - - # Unit tests. - suite.addTest(doctest.DocTestSuite("crate.client.connection")) - suite.addTest(doctest.DocTestSuite("crate.client.http")) - - s = doctest.DocFileSuite( - "docs/by-example/connection.rst", - "docs/by-example/cursor.rst", - module_relative=False, - optionflags=flags, - encoding="utf-8", - ) - suite.addTest(s) - - s = doctest.DocFileSuite( - "docs/by-example/https.rst", - module_relative=False, - setUp=setUpWithHttps, - optionflags=flags, - encoding="utf-8", - ) - s.layer = HttpsTestServerLayer() - suite.addTest(s) - - # Integration tests. - layer = ensure_cratedb_layer() - - s = doctest.DocFileSuite( - "docs/by-example/http.rst", - "docs/by-example/client.rst", - "docs/by-example/blob.rst", - module_relative=False, - setUp=setUpCrateLayerBaseline, - tearDown=tearDownDropEntitiesBaseline, - optionflags=flags, - encoding="utf-8", - ) - s.layer = layer - suite.addTest(s) - - return suite diff --git a/tests/conftest.py b/tests/conftest.py index 66626354..a64b5762 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,16 +1,43 @@ +import json +import logging import multiprocessing +import os +import platform import socket +import ssl +import sys +import tarfile import threading +import time +import zipfile from contextlib import contextmanager from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path from unittest.mock import MagicMock +from urllib.request import urlretrieve import pytest import urllib3 import crate +from crate.client import connect +from crate.testing.layer import CrateLayer +from tests.client.settings import assets_path + +log = logging.getLogger("tests.conftest") + REQUEST_PATH = "crate.client.http.Server.request" +URL_TMPL = "https://cdn.crate.io/downloads/releases/cratedb/{arch}_{os}/crate-6.1.2.{ext}" + +project_root = Path(__file__).parent.parent +cratedb_path = project_root / "parts/crate" + + +crate_port = 44209 +crate_transport_port = 44309 +localhost = "127.0.0.1" +crate_host = f"http://{localhost}:{crate_port}" def fake_response( @@ -90,3 +117,193 @@ def _serve(handler_cls=BaseHTTPRequestHandler): thread.join() return _serve + + +def get_crate_url() -> str: + extension = "tar.gz" + + machine = platform.machine() + if machine.startswith("arm") or machine == "aarch64": + arch = "aarch64" + else: + arch = "x64" + + if sys.platform.startswith("linux"): + os = "linux" + elif sys.platform.startswith("win32"): + os = "windows" + extension = "zip" + elif sys.platform.startswith("darwin"): + os = "mac" + + # there are no aarch64/arm64 distributions available + # x64 should work via emulation layer + arch = "x64" + else: + raise ValueError(f"Unsupported platform: {sys.platform}") + + return URL_TMPL.format(arch=arch, os=os, ext=extension) + + +def download_cratedb(path: Path): + url = get_crate_url() + if path.exists(): + return + if not url.startswith("https:"): + raise ValueError("Invalid url") + filename, _msg = urlretrieve(url) + if sys.platform.startswith("win32"): + with zipfile.ZipFile(filename) as z: + first_file = z.namelist()[0] + folder_name = os.path.dirname(first_file) + z.extractall(path.parent) + (path.parent / folder_name).rename(path) + else: + with tarfile.open(filename) as t: + first_file = t.getnames()[0] + folder_name = os.path.dirname(first_file) + t.extractall(path.parent, filter="data") + (path.parent / folder_name).rename(path) + + +def create_test_data(cursor): + with open(project_root / "tests/assets/mappings/locations.sql") as s: + stmt = s.read() + cursor.execute(stmt) + stmt = ( + "select count(*) from information_schema.tables " + "where table_name = 'locations'" + ) + cursor.execute(stmt) + assert cursor.fetchall()[0][0] == 1 # noqa: S101 + + data_path = str(project_root / "tests/assets/import/test_a.json") + # load testing data into crate + cursor.execute("copy locations from ?", (data_path,)) + # refresh location table so imported data is visible immediately + cursor.execute("refresh table locations") + # create blob table + cursor.execute( + "create blob table myfiles clustered into 1 shards " + + "with (number_of_replicas=0)" + ) + + # create users + cursor.execute("CREATE USER me WITH (password = 'my_secret_pw')") + cursor.execute("CREATE USER trusted_me") + + +@pytest.fixture() +def doctest_node(): + download_cratedb(cratedb_path) + settings = { + "udc.enabled": "false", + "lang.js.enabled": "true", + "auth.host_based.enabled": "true", + "auth.host_based.config.0.user": "crate", + "auth.host_based.config.0.method": "trust", + "auth.host_based.config.98.user": "trusted_me", + "auth.host_based.config.98.method": "trust", + "auth.host_based.config.99.user": "me", + "auth.host_based.config.99.method": "password", + "discovery.type": "single-node", + } + crate_layer = CrateLayer( + "crate", + crate_home=cratedb_path, + port=crate_port, + host=localhost, + transport_port=crate_transport_port, + settings=settings, + ) + crate_layer.start() + with connect(crate_host) as conn: + cursor = conn.cursor() + create_test_data(cursor) + cursor.close() + + yield crate_layer + crate_layer.stop() + + +class HttpsServer(HTTPServer): + + PORT = 65534 + HOST = "localhost" + CERT_FILE = assets_path("pki/server_valid.pem") + CACERT_FILE = assets_path("pki/cacert_valid.pem") + + def get_request(self): + # Prepare SSL context. + context = ssl._create_unverified_context( # noqa: S323 + protocol=ssl.PROTOCOL_TLS_SERVER, + cert_reqs=ssl.CERT_OPTIONAL, + check_hostname=False, + purpose=ssl.Purpose.CLIENT_AUTH, + certfile=HttpsServer.CERT_FILE, + keyfile=HttpsServer.CERT_FILE, + cafile=HttpsServer.CACERT_FILE, + ) # noqa: S323 + + # Set minimum protocol version, TLSv1 and TLSv1.1 are unsafe. + context.minimum_version = ssl.TLSVersion.TLSv1_2 + + # Wrap TLS encryption around socket. + socket, client_address = HTTPServer.get_request(self) + socket = context.wrap_socket(socket, server_side=True) + + return socket, client_address + + +class HttpsHandler(BaseHTTPRequestHandler): + payload = json.dumps( + { + "name": "test", + "status": 200, + } + ) + + def do_GET(self): + self.send_response(200) + payload = self.payload.encode("UTF-8") + self.send_header("Content-Length", f"{len(payload)}") + self.send_header("Content-Type", "application/json; charset=UTF-8") + self.end_headers() + self.wfile.write(payload) + + +def is_up(host: str, port: int) -> bool: + """ + Test if a host is up. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ex = s.connect_ex((host, port)) + s.close() + return ex == 0 + + +@pytest.fixture +def https_server(): + port = HttpsServer.PORT + host = HttpsServer.HOST + server_address = (host, port) + server = HttpsServer(server_address, HttpsHandler) + thread = threading.Thread(target=server.serve_forever) + thread.daemon = True + thread.start() + + start = time.monotonic() + timeout = 5 + while True: + if is_up(host, port): + break + now = time.monotonic() + if now - start > timeout: + raise TimeoutError( + "Could not properly start embedded webserver " + "within {} seconds".format(timeout) + ) + + yield server + server.shutdown() + server.server_close() diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 00000000..5fe52d80 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,37 @@ +import doctest +from pprint import pprint + +from tests.client.settings import assets_path + +from .conftest import HttpsServer, crate_host + + +def cprint(s): + if isinstance(s, bytes): + s = s.decode("utf-8") + print(s) # noqa: T201 + + +def test_docs(doctest_node, https_server): + flags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS + globs = { + "pprint": pprint, + "print": cprint, + "crate_host": crate_host, + "https_host": f"https://{HttpsServer.HOST}:{HttpsServer.PORT}", + "cacert_valid": assets_path("pki/cacert_valid.pem"), + "cacert_invalid": assets_path("pki/cacert_invalid.pem"), + "clientcert_valid": assets_path("pki/client_valid.pem"), + "clientcert_invalid": assets_path("pki/client_invalid.pem"), + } + + def test(path): + failures, tests = doctest.testfile(path, optionflags=flags, globs=globs) + assert not failures + + test("../docs/by-example/connection.rst") + test("../docs/by-example/cursor.rst") + test("../docs/by-example/http.rst") + test("../docs/by-example/client.rst") + test("../docs/by-example/blob.rst") + test("../docs/by-example/https.rst") diff --git a/tests/testing/test_layer.py b/tests/testing/test_layer.py index ae799354..12b08a77 100644 --- a/tests/testing/test_layer.py +++ b/tests/testing/test_layer.py @@ -21,12 +21,8 @@ import json import os -import platform -import sys -import tarfile import tempfile import urllib -import zipfile from io import BytesIO from pathlib import Path from unittest import TestCase, mock @@ -41,37 +37,10 @@ prepend_http, wait_for_http_url, ) +from tests.conftest import download_cratedb from .settings import crate_path -URL_TMPL = "https://cdn.crate.io/downloads/releases/cratedb/{arch}_{os}/crate-6.1.2.{ext}" - - -def get_crate_url() -> str: - extension = "tar.gz" - - machine = platform.machine() - if machine.startswith("arm") or machine == "aarch64": - arch = "aarch64" - else: - arch = "x64" - - if sys.platform.startswith("linux"): - os = "linux" - elif sys.platform.startswith("win32"): - os = "windows" - extension = "zip" - elif sys.platform.startswith("darwin"): - os = "mac" - - # there are no aarch64/arm64 distributions available - # x64 should work via emulation layer - arch = "x64" - else: - raise ValueError(f"Unsupported platform: {sys.platform}") - - return URL_TMPL.format(arch=arch, os=os, ext=extension) - class LayerUtilsTest(TestCase): def test_prepend_http(self): @@ -163,25 +132,7 @@ def test_java_home_env_override(self): class LayerTest(TestCase): @classmethod def setup_class(cls): - url = get_crate_url() - target_path = Path(crate_path()) - if target_path.exists(): - return - if not url.startswith("https:"): - raise ValueError("Invalid url") - filename, _msg = urllib.request.urlretrieve(url) - if sys.platform.startswith("win32"): - with zipfile.ZipFile(filename) as z: - first_file = z.namelist()[0] - folder_name = os.path.dirname(first_file) - z.extractall(target_path.parent) - (target_path.parent / folder_name).rename(target_path) - else: - with tarfile.open(filename) as t: - first_file = t.getnames()[0] - folder_name = os.path.dirname(first_file) - t.extractall(target_path.parent, filter="data") - (target_path.parent / folder_name).rename(target_path) + download_cratedb(Path(crate_path())) def test_basic(self): """ diff --git a/tests/testing/tests.py b/tests/testing/tests.py deleted file mode 100644 index 4ba58d91..00000000 --- a/tests/testing/tests.py +++ /dev/null @@ -1,34 +0,0 @@ -# vi: set encoding=utf-8 -# -*- coding: utf-8; -*- -# -# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor -# license agreements. See the NOTICE file distributed with this work for -# additional information regarding copyright ownership. Crate licenses -# this file to you under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. You may -# obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# However, if you have executed another commercial license agreement -# with Crate these terms will supersede the license and you may use the -# software solely pursuant to the terms of the relevant commercial agreement. - -import unittest - -from .test_layer import LayerTest, LayerUtilsTest - -makeSuite = unittest.TestLoader().loadTestsFromTestCase - - -def test_suite(): - suite = unittest.TestSuite() - suite.addTest(makeSuite(LayerUtilsTest)) - suite.addTest(makeSuite(LayerTest)) - return suite