diff --git a/.travis.yml b/.travis.yml index 55b7afa9..03c1d9db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,11 +19,14 @@ notifications: on_failure: always env: + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.7 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8.0 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.8 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.9 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.10 PG_VERSION=17 - TEST_PLATFORM=std2-all PYTHON_VERSION=3.11 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.12 PG_VERSION=17 + - TEST_PLATFORM=std2-all PYTHON_VERSION=3.13 PG_VERSION=17 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=16 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=15 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=14 @@ -32,6 +35,7 @@ env: - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=11 - TEST_PLATFORM=std PYTHON_VERSION=3 PG_VERSION=10 - TEST_PLATFORM=std-all PYTHON_VERSION=3 PG_VERSION=17 + - TEST_PLATFORM=std-all PYTHON_VERSION=3 PG_VERSION=18 - TEST_PLATFORM=ubuntu_24_04 PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=altlinux_10 PYTHON_VERSION=3 PG_VERSION=17 - TEST_PLATFORM=altlinux_11 PYTHON_VERSION=3 PG_VERSION=17 diff --git a/Dockerfile--std-all.tmpl b/Dockerfile--std-all.tmpl index c41c5a06..d19f52a6 100644 --- a/Dockerfile--std-all.tmpl +++ b/Dockerfile--std-all.tmpl @@ -4,11 +4,6 @@ ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 -# --------------------------------------------- base2_with_python-2 -FROM base1 as base2_with_python-2 -RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip -ENV PYTHON_VERSION=2 - # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv diff --git a/Dockerfile--std.tmpl b/Dockerfile--std.tmpl index 91886ede..67aa30b4 100644 --- a/Dockerfile--std.tmpl +++ b/Dockerfile--std.tmpl @@ -4,11 +4,6 @@ ARG PYTHON_VERSION # --------------------------------------------- base1 FROM postgres:${PG_VERSION}-alpine as base1 -# --------------------------------------------- base2_with_python-2 -FROM base1 as base2_with_python-2 -RUN apk add --no-cache curl python2 python2-dev build-base musl-dev linux-headers py-virtualenv py-pip -ENV PYTHON_VERSION=2 - # --------------------------------------------- base2_with_python-3 FROM base1 as base2_with_python-3 RUN apk add --no-cache curl python3 python3-dev build-base musl-dev linux-headers py-virtualenv diff --git a/Dockerfile--std2-all.tmpl b/Dockerfile--std2-all.tmpl index 10d8280c..23b95507 100644 --- a/Dockerfile--std2-all.tmpl +++ b/Dockerfile--std2-all.tmpl @@ -20,6 +20,10 @@ RUN apk add openssl openssl-dev RUN apk add sqlite-dev RUN apk add bzip2-dev +# --------------------------------------------- base3_with_python-3.7 +FROM base2_with_python-3 as base3_with_python-3.7 +ENV PYTHON_VERSION=3.7 + # --------------------------------------------- base3_with_python-3.8.0 FROM base2_with_python-3 as base3_with_python-3.8.0 ENV PYTHON_VERSION=3.8.0 @@ -40,6 +44,14 @@ ENV PYTHON_VERSION=3.10 FROM base2_with_python-3 as base3_with_python-3.11 ENV PYTHON_VERSION=3.11 +# --------------------------------------------- base3_with_python-3.12 +FROM base2_with_python-3 as base3_with_python-3.12 +ENV PYTHON_VERSION=3.12 + +# --------------------------------------------- base3_with_python-3.13 +FROM base2_with_python-3 as base3_with_python-3.13 +ENV PYTHON_VERSION=3.13 + # --------------------------------------------- final FROM base3_with_python-${PYTHON_VERSION} as final diff --git a/README.md b/README.md index a3b854f8..bfe07eb8 100644 --- a/README.md +++ b/README.md @@ -6,69 +6,69 @@ # testgres -PostgreSQL testing utility. Both Python 2.7 and 3.3+ are supported. - +Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.17 and newer. ## Installation -To install `testgres`, run: +Install `testgres` from PyPI: -``` +```sh pip install testgres ``` -We encourage you to use `virtualenv` for your testing environment. - +Use a dedicated virtual environment for isolated test dependencies. ## Usage ### Environment -> Note: by default testgres runs `initdb`, `pg_ctl`, `psql` provided by `PATH`. +> Note: by default `testgres` invokes `initdb`, `pg_ctl`, and `psql` binaries found in `PATH`. -There are several ways to specify a custom postgres installation: +Specify a custom PostgreSQL installation in one of the following ways: -* export `PG_CONFIG` environment variable pointing to the `pg_config` executable; -* export `PG_BIN` environment variable pointing to the directory with executable files. +- Set the `PG_CONFIG` environment variable to point to the `pg_config` executable. +- Set the `PG_BIN` environment variable to point to the directory with PostgreSQL binaries. Example: -```bash -export PG_BIN=$HOME/pg_10/bin +```sh +export PG_BIN=$HOME/pg_16/bin python my_tests.py ``` - ### Examples -Here is an example of what you can do with `testgres`: +Create a temporary node, run queries, and let `testgres` clean up automatically: ```python -# create a node with random name, port, etc +# create a node with a random name, port, and data directory with testgres.get_new_node() as node: - # run inidb + # run initdb node.init() # start PostgreSQL node.start() - # execute a query in a default DB + # execute a query in the default database print(node.execute('select 1')) -# ... node stops and its files are about to be removed +# the node is stopped and its files are removed automatically ``` -There are four API methods for running queries: +### Query helpers + +`testgres` provides four helpers for executing queries against the node: | Command | Description | -|----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------| -| `node.psql(query, ...)` | Runs query via `psql` command and returns tuple `(error code, stdout, stderr)`. | -| `node.safe_psql(query, ...)` | Same as `psql()` except that it returns only `stdout`. If an error occurs during the execution, an exception will be thrown. | -| `node.execute(query, ...)` | Connects to PostgreSQL using `psycopg2` or `pg8000` (depends on which one is installed in your system) and returns two-dimensional array with data. | -| `node.connect(dbname, ...)` | Returns connection wrapper (`NodeConnection`) capable of running several queries within a single transaction. | +|---------|-------------| +| `node.psql(query, ...)` | Runs the query via `psql` and returns a tuple `(returncode, stdout, stderr)`. | +| `node.safe_psql(query, ...)` | Same as `psql()` but returns only `stdout` and raises if the command fails. | +| `node.execute(query, ...)` | Connects via `psycopg2` or `pg8000` (whichever is available) and returns a list of tuples. | +| `node.connect(dbname, ...)` | Returns a `NodeConnection` wrapper for executing multiple statements within a transaction. | + +Example of transactional usage: -The last one is the most powerful: you can use `begin(isolation_level)`, `commit()` and `rollback()`: ```python with node.connect() as con: con.begin('serializable') @@ -76,16 +76,13 @@ with node.connect() as con: con.rollback() ``` - ### Logging -By default, `cleanup()` removes all temporary files (DB files, logs etc) that were created by testgres' API methods. -If you'd like to keep logs, execute `configure_testgres(node_cleanup_full=False)` before running any tests. +By default `cleanup()` removes all temporary files (data directories, logs, and so on) created by the API. Call `configure_testgres(node_cleanup_full=False)` before starting nodes if you want to keep logs for inspection. -> Note: context managers (aka `with`) call `stop()` and `cleanup()` automatically. +> Note: context managers (the `with` statement) call `stop()` and `cleanup()` automatically. -`testgres` supports [python logging](https://docs.python.org/3.6/library/logging.html), -which means that you can aggregate logs from several nodes into one file: +`testgres` integrates with the standard [Python logging](https://docs.python.org/3/library/logging.html) module, so you can aggregate logs from multiple nodes: ```python import logging @@ -93,12 +90,11 @@ import logging # write everything to /tmp/testgres.log logging.basicConfig(filename='/tmp/testgres.log') -# enable logging, and create two different nodes +# enable logging and create two nodes testgres.configure_testgres(use_python_logging=True) node1 = testgres.get_new_node().init().start() node2 = testgres.get_new_node().init().start() -# execute a few queries node1.execute('select 1') node2.execute('select 2') @@ -106,104 +102,103 @@ node2.execute('select 2') testgres.configure_testgres(use_python_logging=False) ``` -Look at `tests/test_simple.py` file for a complete example of the logging -configuration. - +See `tests/test_simple.py` for a complete logging example. -### Backup & replication +### Backup and replication -It's quite easy to create a backup and start a new replica: +Creating backups and spawning replicas is straightforward: ```python with testgres.get_new_node('master') as master: master.init().start() - # create a backup with master.backup() as backup: - - # create and start a new replica replica = backup.spawn_replica('replica').start() - - # catch up with master node replica.catchup() - # execute a dummy query print(replica.execute('postgres', 'select 1')) ``` ### Benchmarks -`testgres` is also capable of running benchmarks using `pgbench`: +Use `pgbench` through `testgres` to run quick benchmarks: ```python with testgres.get_new_node('master') as master: - # start a new node master.init().start() - # initialize default DB and run bench for 10 seconds - res = master.pgbench_init(scale=2).pgbench_run(time=10) - print(res) + result = master.pgbench_init(scale=2).pgbench_run(time=10) + print(result) ``` - ### Custom configuration -It's often useful to extend default configuration provided by `testgres`. - -`testgres` has `default_conf()` function that helps control some basic -options. The `append_conf()` function can be used to add custom -lines to configuration lines: +`testgres` ships with sensible defaults. Adjust them as needed with `default_conf()` and `append_conf()`: ```python -ext_conf = "shared_preload_libraries = 'postgres_fdw'" +extra_conf = "shared_preload_libraries = 'postgres_fdw'" -# initialize a new node with testgres.get_new_node().init() as master: - - # ... do something ... - - # reset main config file - master.default_conf(fsync=True, - allow_streaming=True) - - # add a new config line - master.append_conf('postgresql.conf', ext_conf) + master.default_conf(fsync=True, allow_streaming=True) + master.append_conf('postgresql.conf', extra_conf) ``` -Note that `default_conf()` is called by `init()` function; both of them overwrite -the configuration file, which means that they should be called before `append_conf()`. +`default_conf()` is called by `init()` and rewrites the configuration file. Apply `append_conf()` afterwards to keep custom lines. ### Remote mode -Testgres supports the creation of PostgreSQL nodes on a remote host. This is useful when you want to run distributed tests involving multiple nodes spread across different machines. -To use this feature, you need to use the RemoteOperations class. This feature is only supported with Linux. -Here is an example of how you might set this up: +You can provision nodes on a remote host (Linux only) by wiring `RemoteOperations` into the configuration: ```python from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node -# Set up connection params conn_params = ConnectionParams( - host='your_host', # replace with your host - username='user_name', # replace with your username - ssh_key='path_to_ssh_key' # replace with your SSH key path + host='example.com', + username='postgres', + ssh_key='/path/to/ssh/key' ) os_ops = RemoteOperations(conn_params) -# Add remote testgres config before test TestgresConfig.set_os_ops(os_ops=os_ops) -# Proceed with your test -def test_basic_query(self): +def test_basic_query(): with get_remote_node(conn_params=conn_params) as node: node.init().start() - res = node.execute('SELECT 1') - self.assertEqual(res, [(1,)]) + assert node.execute('SELECT 1') == [(1,)] ``` +### Pytest integration + +Use fixtures to create and clean up nodes automatically when testing with `pytest`: + +```python +import pytest +import testgres + +@pytest.fixture +def pg_node(): + node = testgres.get_new_node().init().start() + try: + yield node + finally: + node.stop() + node.cleanup() + +def test_simple(pg_node): + assert pg_node.execute('select 1')[0][0] == 1 +``` + +This pattern keeps tests concise and ensures that every node is stopped and removed even if the test fails. + +### Scaling tips + +- Run tests in parallel with `pytest -n auto` (requires `pytest-xdist`). Ensure each node uses a distinct port by setting `PGPORT` in the fixture or by passing the `port` argument to `get_new_node()`. +- Always call `node.cleanup()` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories. +- Prefer `node.safe_psql()` for lightweight assertions that should fail fast; use `node.execute()` when you need structured Python results. + ## Authors [Ildar Musin](https://github.com/zilder) [Dmitry Ivanov](https://github.com/funbringer) [Ildus Kurbangaliev](https://github.com/ildus) -[Yury Zhuravlev](https://github.com/stalkerg) +[Yury Zhuravlev](https://github.com/stalkerg) diff --git a/__init__.py b/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/Makefile b/docs/Makefile index f33f6be0..d6818981 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,4 +17,5 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile + @pip install --force-reinstall .. @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 688a850f..e931931e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,18 +14,26 @@ # import os import sys -sys.path.insert(0, os.path.abspath('../..')) +import testgres + +assert testgres.__path__ is not None +assert len(testgres.__path__) == 1 +assert type(testgres.__path__[0] == str) # noqa: E721 +p = os.path.dirname(testgres.__path__[0]) +assert type(p) == str # noqa: E721 +sys.path.insert(0, os.path.abspath(p)) # -- Project information ----------------------------------------------------- project = u'testgres' -copyright = u'2016-2023, Postgres Professional' +package_name = u'testgres' +copyright = u'2016-2025, Postgres Professional' author = u'Postgres Professional' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'1.5' +release = u'1.12.0' # -- General configuration --------------------------------------------------- @@ -55,7 +63,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. diff --git a/docs/source/index.rst b/docs/source/index.rst index 566d9a50..c2104a65 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -2,7 +2,7 @@ Testgres documentation ====================== -Testgres is a PostgreSQL testing framework. +Utility for orchestrating temporary PostgreSQL clusters in Python tests. Supports Python 3.7.17 and newer. Installation ============ @@ -21,9 +21,9 @@ Usage Environment ----------- -Note: by default testgres runs ``initdb``, ``pg_ctl``, ``psql`` provided by ``PATH``. +Note: by default ``testgres`` runs ``initdb``, ``pg_ctl``, and ``psql`` found in ``PATH``. -There are several ways to specify a custom postgres installation: +There are several ways to specify a custom PostgreSQL installation: - export ``PG_CONFIG`` environment variable pointing to the ``pg_config`` executable; - export ``PG_BIN`` environment variable pointing to the directory with executable files. @@ -32,29 +32,76 @@ Example: .. code-block:: bash - export PG_BIN=$HOME/pg_10/bin + export PG_BIN=$HOME/pg_16/bin python my_tests.py Examples -------- -Here is an example of what you can do with ``testgres``: +Create a temporary node, run queries, and let ``testgres`` clean up automatically: .. code-block:: python - # create a node with random name, port, etc + # create a node with a random name, port, and data directory with testgres.get_new_node() as node: - # run inidb + # run initdb node.init() # start PostgreSQL node.start() - # execute a query in a default DB + # execute a query in the default database print(node.execute('select 1')) - # ... node stops and its files are about to be removed + # the node is stopped and its files are removed automatically + +Query helpers +------------- + +``testgres`` provides four helpers for executing queries against the node: + +========================== ======================================================= +Command Description +========================== ======================================================= +``node.psql(query, ...)`` Runs the query via ``psql`` and returns ``(code, out, err)``. +``node.safe_psql(...)`` Returns only ``stdout`` and raises if the command fails. +``node.execute(...)`` Uses ``psycopg2``/``pg8000`` and returns a list of tuples. +``node.connect(...)`` Returns a ``NodeConnection`` for transactional usage. +========================== ======================================================= + +Example: + +.. code-block:: python + + with node.connect() as con: + con.begin('serializable') + print(con.execute('select %s', 1)) + con.rollback() + +Logging +------- + +By default ``cleanup()`` removes all temporary files (data directories, logs, and so on) created by the API. Call ``configure_testgres(node_cleanup_full=False)`` before starting nodes if you want to keep logs for inspection. + +Note: context managers (the ``with`` statement) call ``stop()`` and ``cleanup()`` automatically. + +``testgres`` integrates with the standard `Python logging `_ module, so you can aggregate logs from multiple nodes: + +.. code-block:: python + + import logging + + logging.basicConfig(filename='/tmp/testgres.log') + + testgres.configure_testgres(use_python_logging=True) + node1 = testgres.get_new_node().init().start() + node2 = testgres.get_new_node().init().start() + + node1.execute('select 1') + node2.execute('select 2') + + testgres.configure_testgres(use_python_logging=False) Backup & replication -------------------- @@ -72,12 +119,90 @@ It's quite easy to create a backup and start a new replica: # create and start a new replica replica = backup.spawn_replica('replica').start() - # catch up with master node replica.catchup() - # execute a dummy query print(replica.execute('postgres', 'select 1')) +Benchmarks +---------- + +Use ``pgbench`` through ``testgres`` to run quick benchmarks: + +.. code-block:: python + + with testgres.get_new_node('master') as master: + master.init().start() + + result = master.pgbench_init(scale=2).pgbench_run(time=10) + print(result) + +Custom configuration +-------------------- + +``testgres`` ships with sensible defaults. Adjust them as needed with ``default_conf()`` and ``append_conf()``: + +.. code-block:: python + + extra_conf = "shared_preload_libraries = 'postgres_fdw'" + + with testgres.get_new_node().init() as master: + master.default_conf(fsync=True, allow_streaming=True) + master.append_conf('postgresql.conf', extra_conf) + +``default_conf()`` is called by ``init()`` and rewrites the configuration file. Apply ``append_conf()`` afterwards to keep custom lines. + +Remote mode +----------- + +Provision nodes on a remote host (Linux only) by wiring ``RemoteOperations`` into the configuration: + +.. code-block:: python + + from testgres import ConnectionParams, RemoteOperations, TestgresConfig, get_remote_node + + conn_params = ConnectionParams( + host='example.com', + username='postgres', + ssh_key='/path/to/ssh/key' + ) + os_ops = RemoteOperations(conn_params) + + TestgresConfig.set_os_ops(os_ops=os_ops) + + def test_basic_query(): + with get_remote_node(conn_params=conn_params) as node: + node.init().start() + assert node.execute('SELECT 1') == [(1,)] + +Pytest integration +------------------ + +Use fixtures to create and clean up nodes automatically when testing with ``pytest``: + +.. code-block:: python + + import pytest + import testgres + + @pytest.fixture + def pg_node(): + node = testgres.get_new_node().init().start() + try: + yield node + finally: + node.stop() + node.cleanup() + + def test_simple(pg_node): + assert pg_node.execute('select 1')[0][0] == 1 + +Scaling tips +------------ + +* Run tests in parallel with ``pytest -n auto`` (requires ``pytest-xdist``). Set unique ports by passing ``port`` to ``get_new_node()`` or exporting ``PGPORT`` in the fixture. +* Always call ``node.cleanup()`` after each test, or rely on context managers/fixtures that do it for you, to avoid leftover data directories. +* Prefer ``safe_psql()`` for quick assertions, and ``execute()`` when you need Python data structures. + Modules ======= diff --git a/run_tests.sh b/run_tests.sh index 65c17dbf..6a733b66 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -18,7 +18,8 @@ rm -rf $VENV_PATH virtualenv --python="/usr/bin/python${PYTHON_VERSION}" "${VENV_PATH}" export VIRTUAL_ENV_DISABLE_PROMPT=1 source "${VENV_PATH}/bin/activate" -pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil +pip install -r tests/requirements.txt +pip install coverage flake8 Sphinx # install testgres' dependencies export PYTHONPATH=$(pwd) diff --git a/run_tests2.sh b/run_tests2.sh index 173b19dc..7e9e4cab 100755 --- a/run_tests2.sh +++ b/run_tests2.sh @@ -24,7 +24,8 @@ rm -rf $VENV_PATH python -m venv "${VENV_PATH}" export VIRTUAL_ENV_DISABLE_PROMPT=1 source "${VENV_PATH}/bin/activate" -pip install coverage flake8 psutil Sphinx pytest pytest-xdist psycopg2 six psutil +pip install -r tests/requirements.txt +pip install coverage flake8 Sphinx # install testgres' dependencies export PYTHONPATH=$(pwd) diff --git a/setup.py b/setup.py index 2c44b18f..f5f25e3a 100755 --- a/setup.py +++ b/setup.py @@ -11,7 +11,8 @@ "port-for>=0.4", "six>=1.9.0", "psutil", - "packaging" + "packaging", + "testgres.os_ops>=0.0.2,<1.0.0" ] # Add compatibility enum class @@ -27,9 +28,10 @@ readme = f.read() setup( - version='1.11.0', + version='1.12.0', name='testgres', - packages=['testgres', 'testgres.operations'], + packages=['testgres', 'testgres.impl'], + package_dir={"testgres": "src"}, description='Testing utility for PostgreSQL and its extensions', url='https://github.com/postgrespro/testgres', long_description=readme, diff --git a/testgres/__init__.py b/src/__init__.py similarity index 84% rename from testgres/__init__.py rename to src/__init__.py index 339ae62e..bebd6878 100644 --- a/testgres/__init__.py +++ b/src/__init__.py @@ -33,8 +33,9 @@ ProcessType, \ DumpFormat -from .node import PostgresNode, NodeApp +from .node import PostgresNode from .node import PortManager +from .node_app import NodeApp from .utils import \ reserve_port, \ @@ -50,9 +51,9 @@ from .config import testgres_config -from .operations.os_ops import OsOperations, ConnectionParams -from .operations.local_ops import LocalOperations -from .operations.remote_ops import RemoteOperations +from testgres.operations.os_ops import OsOperations, ConnectionParams +from testgres.operations.local_ops import LocalOperations +from testgres.operations.remote_ops import RemoteOperations __all__ = [ "get_new_node", @@ -62,8 +63,9 @@ "NodeConnection", "DatabaseError", "InternalError", "ProgrammingError", "OperationalError", "TestgresException", "ExecUtilException", "QueryException", "TimeoutException", "CatchUpException", "StartNodeException", "InitNodeException", "BackupException", "InvalidOperationException", "XLogMethod", "IsolationLevel", "NodeStatus", "ProcessType", "DumpFormat", - "PostgresNode", "NodeApp", - "PortManager", + NodeApp.__name__, + PostgresNode.__name__, + PortManager.__name__, "reserve_port", "release_port", "bound_ports", "get_bin_path", "get_pg_config", "get_pg_version", "First", "Any", "OsOperations", "LocalOperations", "RemoteOperations", "ConnectionParams" diff --git a/testgres/api.py b/src/api.py similarity index 100% rename from testgres/api.py rename to src/api.py diff --git a/testgres/backup.py b/src/backup.py similarity index 87% rename from testgres/backup.py rename to src/backup.py index 388697b7..06e6ef2d 100644 --- a/testgres/backup.py +++ b/src/backup.py @@ -1,7 +1,5 @@ # coding: utf-8 -import os - from six import raise_from from .enums import XLogMethod @@ -15,7 +13,7 @@ from .exceptions import BackupException -from .operations.os_ops import OsOperations +from testgres.operations.os_ops import OsOperations from .utils import \ get_bin_path2, \ @@ -29,7 +27,9 @@ class NodeBackup(object): """ @property def log_file(self): - return os.path.join(self.base_dir, BACKUP_LOG_FILE) + assert self.os_ops is not None + assert isinstance(self.os_ops, OsOperations) + return self.os_ops.build_path(self.base_dir, BACKUP_LOG_FILE) def __init__(self, node, @@ -75,7 +75,7 @@ def __init__(self, # private self._available = True - data_dir = os.path.join(self.base_dir, DATA_DIR) + data_dir = self.os_ops.build_path(self.base_dir, DATA_DIR) _params = [ get_bin_path2(self.os_ops, "pg_basebackup"), @@ -112,10 +112,13 @@ def _prepare_dir(self, destroy): available = not destroy if available: + assert self.os_ops is not None + assert isinstance(self.os_ops, OsOperations) + dest_base_dir = self.os_ops.mkdtemp(prefix=TMP_NODE) - data1 = os.path.join(self.base_dir, DATA_DIR) - data2 = os.path.join(dest_base_dir, DATA_DIR) + data1 = self.os_ops.build_path(self.base_dir, DATA_DIR) + data2 = self.os_ops.build_path(dest_base_dir, DATA_DIR) try: # Copy backup to new data dir @@ -160,10 +163,6 @@ def spawn_primary(self, name=None, destroy=True): assert type(node) == self.original_node.__class__ # noqa: E721 with clean_on_error(node) as node: - - # New nodes should always remove dir tree - node._should_rm_dirs = True - # Set a new port node.append_conf(filename=PG_CONF_FILE, line='\n') node.append_conf(filename=PG_CONF_FILE, port=node.port) @@ -184,14 +183,19 @@ def spawn_replica(self, name=None, destroy=True, slot=None): """ # Build a new PostgresNode - with clean_on_error(self.spawn_primary(name=name, - destroy=destroy)) as node: + node = self.spawn_primary(name=name, destroy=destroy) + assert node is not None + try: # Assign it a master and a recovery file (private magic) node._assign_master(self.original_node) node._create_recovery_conf(username=self.username, slot=slot) + except: # noqa: E722 + # TODO: Pass 'final=True' ? + node.cleanup(release_resources=True) + raise - return node + return node def cleanup(self): """ diff --git a/testgres/cache.py b/src/cache.py similarity index 85% rename from testgres/cache.py rename to src/cache.py index 3ac63326..95ae0a94 100644 --- a/testgres/cache.py +++ b/src/cache.py @@ -1,7 +1,5 @@ # coding: utf-8 -import os - from six import raise_from from .config import testgres_config @@ -18,16 +16,20 @@ get_bin_path2, \ execute_utility2 -from .operations.local_ops import LocalOperations -from .operations.os_ops import OsOperations +from testgres.operations.local_ops import LocalOperations +from testgres.operations.os_ops import OsOperations -def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = LocalOperations(), bin_path=None, cached=True): +def cached_initdb(data_dir, logfile=None, params=None, os_ops: OsOperations = None, bin_path=None, cached=True): """ Perform initdb or use cached node files. """ - assert os_ops is not None + assert os_ops is None or isinstance(os_ops, OsOperations) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + assert isinstance(os_ops, OsOperations) def make_utility_path(name): @@ -35,7 +37,7 @@ def make_utility_path(name): assert type(name) == str # noqa: E721 if bin_path: - return os.path.join(bin_path, name) + return os_ops.build_path(bin_path, name) return get_bin_path2(os_ops, name) @@ -68,7 +70,7 @@ def call_initdb(initdb_dir, log=logfile): # XXX: write new unique system id to control file # Some users might rely upon unique system ids, but # our initdb caching mechanism breaks this contract. - pg_control = os.path.join(data_dir, XLOG_CONTROL_FILE) + pg_control = os_ops.build_path(data_dir, XLOG_CONTROL_FILE) system_id = generate_system_id() cur_pg_control = os_ops.read(pg_control, binary=True) new_pg_control = system_id + cur_pg_control[len(system_id):] diff --git a/testgres/config.py b/src/config.py similarity index 97% rename from testgres/config.py rename to src/config.py index 67d467d3..1d09ccb8 100644 --- a/testgres/config.py +++ b/src/config.py @@ -9,8 +9,8 @@ from contextlib import contextmanager from .consts import TMP_CACHE -from .operations.os_ops import OsOperations -from .operations.local_ops import LocalOperations +from testgres.operations.os_ops import OsOperations +from testgres.operations.local_ops import LocalOperations log_level = os.getenv('LOGGING_LEVEL', 'WARNING').upper() log_format = os.getenv('LOGGING_FORMAT', '%(asctime)s - %(levelname)s - %(message)s') @@ -50,8 +50,9 @@ class GlobalConfig(object): _cached_initdb_dir = None """ underlying class attribute for cached_initdb_dir property """ - os_ops = LocalOperations() + os_ops = LocalOperations.get_single_instance() """ OsOperation object that allows work on remote host """ + @property def cached_initdb_dir(self): """ path to a temp directory for cached initdb. """ diff --git a/testgres/connection.py b/src/connection.py similarity index 100% rename from testgres/connection.py rename to src/connection.py diff --git a/testgres/consts.py b/src/consts.py similarity index 100% rename from testgres/consts.py rename to src/consts.py diff --git a/testgres/decorators.py b/src/decorators.py similarity index 100% rename from testgres/decorators.py rename to src/decorators.py diff --git a/testgres/defaults.py b/src/defaults.py similarity index 100% rename from testgres/defaults.py rename to src/defaults.py diff --git a/testgres/enums.py b/src/enums.py similarity index 100% rename from testgres/enums.py rename to src/enums.py diff --git a/src/exceptions.py b/src/exceptions.py new file mode 100644 index 00000000..b4aad645 --- /dev/null +++ b/src/exceptions.py @@ -0,0 +1,71 @@ +# coding: utf-8 + +import six + +from testgres.operations.exceptions import TestgresException +from testgres.operations.exceptions import ExecUtilException +from testgres.operations.exceptions import InvalidOperationException + + +class PortForException(TestgresException): + pass + + +@six.python_2_unicode_compatible +class QueryException(TestgresException): + def __init__(self, message=None, query=None): + super(QueryException, self).__init__(message) + + self.message = message + self.query = query + + def __str__(self): + msg = [] + + if self.message: + msg.append(self.message) + + if self.query: + msg.append(u'Query: {}'.format(self.query)) + + return six.text_type('\n').join(msg) + + +class TimeoutException(QueryException): + pass + + +class CatchUpException(QueryException): + pass + + +@six.python_2_unicode_compatible +class StartNodeException(TestgresException): + def __init__(self, message=None, files=None): + super(StartNodeException, self).__init__(message) + + self.message = message + self.files = files + + def __str__(self): + msg = [] + + if self.message: + msg.append(self.message) + + for f, lines in self.files or []: + msg.append(u'{}\n----\n{}\n'.format(f, lines)) + + return six.text_type('\n').join(msg) + + +class InitNodeException(TestgresException): + pass + + +class BackupException(TestgresException): + pass + + +assert ExecUtilException.__name__ == "ExecUtilException" +assert InvalidOperationException.__name__ == "InvalidOperationException" diff --git a/testgres/port_manager.py b/src/impl/port_manager__generic.py old mode 100644 new mode 100755 similarity index 57% rename from testgres/port_manager.py rename to src/impl/port_manager__generic.py index e2530470..6c156992 --- a/testgres/port_manager.py +++ b/src/impl/port_manager__generic.py @@ -1,53 +1,18 @@ -from .operations.os_ops import OsOperations +from testgres.operations.os_ops import OsOperations -from .exceptions import PortForException - -from . import utils +from ..port_manager import PortManager +from ..exceptions import PortForException import threading import random import typing - - -class PortManager: - def __init__(self): - super().__init__() - - def reserve_port(self) -> int: - raise NotImplementedError("PortManager::reserve_port is not implemented.") - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - raise NotImplementedError("PortManager::release_port is not implemented.") - - -class PortManager__ThisHost(PortManager): - sm_single_instance: PortManager = None - sm_single_instance_guard = threading.Lock() - - def __init__(self): - pass - - def __new__(cls) -> PortManager: - assert __class__ == PortManager__ThisHost - assert __class__.sm_single_instance_guard is not None - - if __class__.sm_single_instance is None: - with __class__.sm_single_instance_guard: - __class__.sm_single_instance = super().__new__(cls) - assert __class__.sm_single_instance - assert type(__class__.sm_single_instance) == __class__ # noqa: E721 - return __class__.sm_single_instance - - def reserve_port(self) -> int: - return utils.reserve_port() - - def release_port(self, number: int) -> None: - assert type(number) == int # noqa: E721 - return utils.release_port(number) +import logging class PortManager__Generic(PortManager): + _C_MIN_PORT_NUMBER = 1024 + _C_MAX_PORT_NUMBER = 65535 + _os_ops: OsOperations _guard: object # TODO: is there better to use bitmap fot _available_ports? @@ -55,12 +20,21 @@ class PortManager__Generic(PortManager): _reserved_ports: typing.Set[int] def __init__(self, os_ops: OsOperations): + assert __class__._C_MIN_PORT_NUMBER <= __class__._C_MAX_PORT_NUMBER + assert os_ops is not None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops self._guard = threading.Lock() - self._available_ports: typing.Set[int] = set(range(1024, 65535)) - self._reserved_ports: typing.Set[int] = set() + + self._available_ports = set( + range(__class__._C_MIN_PORT_NUMBER, __class__._C_MAX_PORT_NUMBER + 1) + ) + assert len(self._available_ports) == ( + (__class__._C_MAX_PORT_NUMBER - __class__._C_MIN_PORT_NUMBER) + 1 + ) + self._reserved_ports = set() + return def reserve_port(self) -> int: assert self._guard is not None @@ -74,9 +48,13 @@ def reserve_port(self) -> int: t = None for port in sampled_ports: + assert type(port) == int # noqa: E721 assert not (port in self._reserved_ports) assert port in self._available_ports + assert port >= __class__._C_MIN_PORT_NUMBER + assert port <= __class__._C_MAX_PORT_NUMBER + if not self._os_ops.is_port_free(port): continue @@ -84,12 +62,15 @@ def reserve_port(self) -> int: self._available_ports.discard(port) assert port in self._reserved_ports assert not (port in self._available_ports) + __class__.helper__send_debug_msg("Port {} is reserved.", port) return port raise PortForException("Can't select a port.") def release_port(self, number: int) -> None: assert type(number) == int # noqa: E721 + assert number >= __class__._C_MIN_PORT_NUMBER + assert number <= __class__._C_MAX_PORT_NUMBER assert self._guard is not None assert type(self._reserved_ports) == set # noqa: E721 @@ -101,3 +82,16 @@ def release_port(self, number: int) -> None: self._reserved_ports.discard(number) assert not (number in self._reserved_ports) assert number in self._available_ports + __class__.helper__send_debug_msg("Port {} is released.", number) + return + + @staticmethod + def helper__send_debug_msg(msg_template: str, *args) -> None: + assert msg_template is not None + assert args is not None + assert type(msg_template) == str # noqa: E721 + assert type(args) == tuple # noqa: E721 + assert msg_template != "" + s = "[port manager] " + s += msg_template.format(*args) + logging.debug(s) diff --git a/src/impl/port_manager__this_host.py b/src/impl/port_manager__this_host.py new file mode 100755 index 00000000..0d56f356 --- /dev/null +++ b/src/impl/port_manager__this_host.py @@ -0,0 +1,33 @@ +from ..port_manager import PortManager + +from .. import utils + +import threading + + +class PortManager__ThisHost(PortManager): + sm_single_instance: PortManager = None + sm_single_instance_guard = threading.Lock() + + @staticmethod + def get_single_instance() -> PortManager: + assert __class__ == PortManager__ThisHost + assert __class__.sm_single_instance_guard is not None + + if __class__.sm_single_instance is not None: + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + with __class__.sm_single_instance_guard: + if __class__.sm_single_instance is None: + __class__.sm_single_instance = __class__() + assert __class__.sm_single_instance is not None + assert type(__class__.sm_single_instance) == __class__ # noqa: E721 + return __class__.sm_single_instance + + def reserve_port(self) -> int: + return utils.reserve_port() + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + return utils.release_port(number) diff --git a/testgres/logger.py b/src/logger.py similarity index 100% rename from testgres/logger.py rename to src/logger.py diff --git a/testgres/node.py b/src/node.py similarity index 87% rename from testgres/node.py rename to src/node.py index 3a294044..be9408be 100644 --- a/testgres/node.py +++ b/src/node.py @@ -7,8 +7,6 @@ import signal import subprocess import threading -import tempfile -import platform from queue import Queue import time @@ -84,8 +82,8 @@ InvalidOperationException from .port_manager import PortManager -from .port_manager import PortManager__ThisHost -from .port_manager import PortManager__Generic +from .impl.port_manager__this_host import PortManager__ThisHost +from .impl.port_manager__generic import PortManager__Generic from .logger import TestgresLogger @@ -93,6 +91,8 @@ from .standby import First +from . import utils + from .utils import \ PgVer, \ eprint, \ @@ -104,10 +104,9 @@ from .backup import NodeBackup -from .operations.os_ops import ConnectionParams -from .operations.os_ops import OsOperations -from .operations.local_ops import LocalOperations -from .operations.remote_ops import RemoteOperations +from testgres.operations.os_ops import ConnectionParams +from testgres.operations.os_ops import OsOperations +from testgres.operations.local_ops import LocalOperations InternalError = pglib.InternalError ProgrammingError = pglib.ProgrammingError @@ -145,13 +144,13 @@ class PostgresNode(object): _port: typing.Optional[int] _should_free_port: bool _os_ops: OsOperations - _port_manager: PortManager + _port_manager: typing.Optional[PortManager] def __init__(self, name=None, base_dir=None, port: typing.Optional[int] = None, - conn_params: ConnectionParams = ConnectionParams(), + conn_params: ConnectionParams = None, bin_dir=None, prefix=None, os_ops: typing.Optional[OsOperations] = None, @@ -171,11 +170,15 @@ def __init__(self, assert os_ops is None or isinstance(os_ops, OsOperations) assert port_manager is None or isinstance(port_manager, PortManager) + if conn_params is not None: + assert type(conn_params) == ConnectionParams # noqa: E721 + + raise InvalidOperationException("conn_params is deprecated, please use os_ops parameter instead.") + # private if os_ops is None: - self._os_ops = __class__._get_os_ops(conn_params) + self._os_ops = __class__._get_os_ops() else: - assert conn_params is None assert isinstance(os_ops, OsOperations) self._os_ops = os_ops pass @@ -200,11 +203,14 @@ def __init__(self, self._should_free_port = False self._port_manager = None else: - if port_manager is not None: + if port_manager is None: + self._port_manager = __class__._get_port_manager(self._os_ops) + elif os_ops is None: + raise InvalidOperationException("When port_manager is not None you have to define os_ops, too.") + else: assert isinstance(port_manager, PortManager) + assert self._os_ops is os_ops self._port_manager = port_manager - else: - self._port_manager = __class__._get_port_manager(self._os_ops) assert self._port_manager is not None assert isinstance(self._port_manager, PortManager) @@ -231,8 +237,6 @@ def __enter__(self): return self def __exit__(self, type, value, traceback): - self.free_port() - # NOTE: Ctrl+C does not count! got_exception = type is not None and type != KeyboardInterrupt @@ -246,6 +250,8 @@ def __exit__(self, type, value, traceback): else: self._try_shutdown(attempts) + self._release_resources() + def __repr__(self): return "{}(name='{}', port={}, base_dir='{}')".format( self.__class__.__name__, @@ -255,24 +261,22 @@ def __repr__(self): ) @staticmethod - def _get_os_ops(conn_params: ConnectionParams) -> OsOperations: + def _get_os_ops() -> OsOperations: if testgres_config.os_ops: return testgres_config.os_ops - assert type(conn_params) == ConnectionParams # noqa: E721 - - if conn_params.ssh_key: - return RemoteOperations(conn_params) - - return LocalOperations(conn_params) + return LocalOperations.get_single_instance() @staticmethod def _get_port_manager(os_ops: OsOperations) -> PortManager: assert os_ops is not None assert isinstance(os_ops, OsOperations) - if isinstance(os_ops, LocalOperations): - return PortManager__ThisHost() + if os_ops is LocalOperations.get_single_instance(): + assert utils._old_port_manager is not None + assert type(utils._old_port_manager) == PortManager__Generic # noqa: E721 + assert utils._old_port_manager._os_ops is os_ops + return PortManager__ThisHost.get_single_instance() # TODO: Throw the exception "Please define a port manager." ? return PortManager__Generic(os_ops) @@ -294,7 +298,6 @@ def clone_with_new_name_and_base_dir(self, name: str, base_dir: str): node = PostgresNode( name=name, base_dir=base_dir, - conn_params=None, bin_dir=self._bin_dir, prefix=self._prefix, os_ops=self._os_ops, @@ -308,6 +311,11 @@ def os_ops(self) -> OsOperations: assert isinstance(self._os_ops, OsOperations) return self._os_ops + @property + def port_manager(self) -> typing.Optional[PortManager]: + assert self._port_manager is None or isinstance(self._port_manager, PortManager) + return self._port_manager + @property def name(self) -> str: if self._name is None: @@ -563,7 +571,11 @@ def bin_dir(self): @property def logs_dir(self): - path = os.path.join(self.base_dir, LOGS_DIR) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.base_dir, LOGS_DIR) + assert type(path) == str # noqa: E721 # NOTE: it's safe to create a new dir if not self.os_ops.path_exists(path): @@ -573,16 +585,31 @@ def logs_dir(self): @property def data_dir(self): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + # NOTE: we can't run initdb without user's args - return os.path.join(self.base_dir, DATA_DIR) + path = self._os_ops.build_path(self.base_dir, DATA_DIR) + assert type(path) == str # noqa: E721 + return path @property def utils_log_file(self): - return os.path.join(self.logs_dir, UTILS_LOG_FILE) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.logs_dir, UTILS_LOG_FILE) + assert type(path) == str # noqa: E721 + return path @property def pg_log_file(self): - return os.path.join(self.logs_dir, PG_LOG_FILE) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + path = self._os_ops.build_path(self.logs_dir, PG_LOG_FILE) + assert type(path) == str # noqa: E721 + return path @property def version(self): @@ -706,7 +733,11 @@ def _create_recovery_conf(self, username, slot=None): ).format(options_string(**conninfo)) # yapf: disable # Since 12 recovery.conf had disappeared if self.version >= PgVer('12'): - signal_name = os.path.join(self.data_dir, "standby.signal") + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + signal_name = self._os_ops.build_path(self.data_dir, "standby.signal") + assert type(signal_name) == str # noqa: E721 self.os_ops.touch(signal_name) else: line += "standby_mode=on\n" @@ -755,11 +786,14 @@ def _collect_special_files(self): result = [] # list of important files + last N lines + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + files = [ - (os.path.join(self.data_dir, PG_CONF_FILE), 0), - (os.path.join(self.data_dir, PG_AUTO_CONF_FILE), 0), - (os.path.join(self.data_dir, RECOVERY_CONF_FILE), 0), - (os.path.join(self.data_dir, HBA_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, PG_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, PG_AUTO_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, RECOVERY_CONF_FILE), 0), + (self._os_ops.build_path(self.data_dir, HBA_CONF_FILE), 0), (self.pg_log_file, testgres_config.error_log_lines) ] # yapf: disable @@ -776,28 +810,6 @@ def _collect_special_files(self): return result - def _collect_log_files(self): - # dictionary of log files + size in bytes - - files = [ - self.pg_log_file - ] # yapf: disable - - result = {} - - for f in files: - # skip missing files - if not self.os_ops.path_exists(f): - continue - - file_size = self.os_ops.get_file_size(f) - assert type(file_size) == int # noqa: E721 - assert file_size >= 0 - - result[f] = file_size - - return result - def init(self, initdb_params=None, cached=True, **kwargs): """ Perform initdb for this node. @@ -813,10 +825,13 @@ def init(self, initdb_params=None, cached=True, **kwargs): """ # initialize this PostgreSQL node + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + cached_initdb( data_dir=self.data_dir, logfile=self.utils_log_file, - os_ops=self.os_ops, + os_ops=self._os_ops, params=initdb_params, bin_path=self.bin_dir, cached=False) @@ -846,8 +861,11 @@ def default_conf(self, This instance of :class:`.PostgresNode`. """ - postgres_conf = os.path.join(self.data_dir, PG_CONF_FILE) - hba_conf = os.path.join(self.data_dir, HBA_CONF_FILE) + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + + postgres_conf = self._os_ops.build_path(self.data_dir, PG_CONF_FILE) + hba_conf = self._os_ops.build_path(self.data_dir, HBA_CONF_FILE) # filter lines in hba file # get rid of comments and blank lines @@ -962,7 +980,7 @@ def append_conf(self, line='', filename=PG_CONF_FILE, **kwargs): # format a new config line lines.append('{} = {}'.format(option, value)) - config_name = os.path.join(self.data_dir, filename) + config_name = self._os_ops.build_path(self.data_dir, filename) conf_text = '' for line in lines: conf_text += text_type(line) + '\n' @@ -1051,22 +1069,6 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem OperationalError}, max_attempts=max_attempts) - def _detect_port_conflict(self, log_files0, log_files1): - assert type(log_files0) == dict # noqa: E721 - assert type(log_files1) == dict # noqa: E721 - - for file in log_files1.keys(): - read_pos = 0 - - if file in log_files0.keys(): - read_pos = log_files0[file] # the previous size - - file_content = self.os_ops.read_binary(file, read_pos) - file_content_s = file_content.decode() - if 'Is another postmaster already running on port' in file_content_s: - return True - return False - def start(self, params=[], wait=True, exec_env=None): """ Starts the PostgreSQL node using pg_ctl if node has not been started. @@ -1126,8 +1128,7 @@ def LOCAL__raise_cannot_start_node__std(from_exception): assert isinstance(self._port_manager, PortManager) assert __class__._C_MAX_START_ATEMPTS > 1 - log_files0 = self._collect_log_files() - assert type(log_files0) == dict # noqa: E721 + log_reader = PostgresNodeLogReader(self, from_beginnig=False) nAttempt = 0 timeout = 1 @@ -1143,11 +1144,11 @@ def LOCAL__raise_cannot_start_node__std(from_exception): if nAttempt == __class__._C_MAX_START_ATEMPTS: LOCAL__raise_cannot_start_node(e, "Cannot start node after multiple attempts.") - log_files1 = self._collect_log_files() - if not self._detect_port_conflict(log_files0, log_files1): + is_it_port_conflict = PostgresNodeUtils.delect_port_conflict(log_reader) + + if not is_it_port_conflict: LOCAL__raise_cannot_start_node__std(e) - log_files0 = log_files1 logging.warning( "Detected a conflict with using the port {0}. Trying another port after a {1}-second sleep...".format(self._port, timeout) ) @@ -1320,27 +1321,20 @@ def pg_ctl(self, params): return execute_utility2(self.os_ops, _params, self.utils_log_file) + def release_resources(self): + """ + Release resorces owned by this node. + """ + return self._release_resources() + def free_port(self): """ Reclaim port owned by this node. NOTE: this method does not release manually defined port but reset it. """ - assert type(self._should_free_port) == bool # noqa: E721 - - if not self._should_free_port: - self._port = None - else: - assert type(self._port) == int # noqa: E721 - - assert self._port_manager is not None - assert isinstance(self._port_manager, PortManager) + return self._free_port() - port = self._port - self._should_free_port = False - self._port = None - self._port_manager.release_port(port) - - def cleanup(self, max_attempts=3, full=False): + def cleanup(self, max_attempts=3, full=False, release_resources=False): """ Stop node if needed and remove its data/logs directory. NOTE: take a look at TestgresConfig.node_cleanup_full. @@ -1363,6 +1357,9 @@ def cleanup(self, max_attempts=3, full=False): self.os_ops.rmdirs(rm_dir, ignore_errors=False) + if release_resources: + self._release_resources() + return self @method_decorator(positional_args_hack(['dbname', 'query'])) @@ -1372,6 +1369,8 @@ def psql(self, dbname=None, username=None, input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, **variables): """ Execute a query using psql. @@ -1382,6 +1381,8 @@ def psql(self, dbname: database name to connect to. username: database user name. input: raw input to be passed. + host: an explicit host of server. + port: an explicit port of server. **variables: vars to be set before execution. Returns: @@ -1393,6 +1394,10 @@ def psql(self, >>> psql(query='select 3', ON_ERROR_STOP=1) """ + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(variables) == dict # noqa: E721 + return self._psql( ignore_errors=True, query=query, @@ -1400,6 +1405,8 @@ def psql(self, dbname=dbname, username=username, input=input, + host=host, + port=port, **variables ) @@ -1411,7 +1418,11 @@ def _psql( dbname=None, username=None, input=None, + host: typing.Optional[str] = None, + port: typing.Optional[int] = None, **variables): + assert host is None or type(host) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 assert type(variables) == dict # noqa: E721 # @@ -1424,10 +1435,21 @@ def _psql( else: raise Exception("Input data must be None or bytes.") + if host is None: + host = self.host + + if port is None: + port = self.port + + assert host is not None + assert port is not None + assert type(host) == str # noqa: E721 + assert type(port) == int # noqa: E721 + psql_params = [ self._get_bin_path("psql"), - "-p", str(self.port), - "-h", self.host, + "-p", str(port), + "-h", host, "-U", username or self.os_ops.username, "-d", dbname or default_dbname(), "-X", # no .psqlrc @@ -2035,8 +2057,11 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}): rm_options (set, optional): A set containing the names of the options to remove. Defaults to an empty set. """ + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + # parse postgresql.auto.conf - path = os.path.join(self.data_dir, config) + path = self.os_ops.build_path(self.data_dir, config) lines = self.os_ops.readlines(path) current_options = {} @@ -2121,9 +2146,31 @@ def upgrade_from(self, old_node, options=None, expect_error=False): return self.os_ops.exec_command(upgrade_command, expect_error=expect_error) + def _release_resources(self): + self._free_port() + + def _free_port(self): + assert type(self._should_free_port) == bool # noqa: E721 + + if not self._should_free_port: + self._port = None + else: + assert type(self._port) == int # noqa: E721 + + assert self._port_manager is not None + assert isinstance(self._port_manager, PortManager) + + port = self._port + self._should_free_port = False + self._port = None + self._port_manager.release_port(port) + def _get_bin_path(self, filename): + assert self._os_ops is not None + assert isinstance(self._os_ops, OsOperations) + if self.bin_dir: - bin_path = os.path.join(self.bin_dir, filename) + bin_path = self._os_ops.build_path(self.bin_dir, filename) else: bin_path = get_bin_path2(self.os_ops, filename) return bin_path @@ -2153,156 +2200,162 @@ def _escape_config_value(value): return result -class NodeApp: +class PostgresNodeLogReader: + class LogInfo: + position: int - def __init__(self, test_path=None, nodes_to_cleanup=None, os_ops=LocalOperations()): - if test_path: - if os.path.isabs(test_path): - self.test_path = test_path - else: - self.test_path = os.path.join(os_ops.cwd(), test_path) - else: - self.test_path = os_ops.cwd() - self.nodes_to_cleanup = nodes_to_cleanup if nodes_to_cleanup else [] - self.os_ops = os_ops + def __init__(self, position: int): + self.position = position - def make_empty( + # -------------------------------------------------------------------- + class LogDataBlock: + _file_name: str + _position: int + _data: str + + def __init__( self, - base_dir=None, - port=None, - bin_dir=None): - real_base_dir = os.path.join(self.test_path, base_dir) - self.os_ops.rmdirs(real_base_dir, ignore_errors=True) - self.os_ops.makedirs(real_base_dir) + file_name: str, + position: int, + data: str + ): + assert type(file_name) == str # noqa: E721 + assert type(position) == int # noqa: E721 + assert type(data) == str # noqa: E721 + assert file_name != "" + assert position >= 0 + self._file_name = file_name + self._position = position + self._data = data + + @property + def file_name(self) -> str: + assert type(self._file_name) == str # noqa: E721 + assert self._file_name != "" + return self._file_name + + @property + def position(self) -> int: + assert type(self._position) == int # noqa: E721 + assert self._position >= 0 + return self._position + + @property + def data(self) -> str: + assert type(self._data) == str # noqa: E721 + return self._data + + # -------------------------------------------------------------------- + _node: PostgresNode + _logs: typing.Dict[str, LogInfo] + + # -------------------------------------------------------------------- + def __init__(self, node: PostgresNode, from_beginnig: bool): + assert node is not None + assert isinstance(node, PostgresNode) + assert type(from_beginnig) == bool # noqa: E721 + + self._node = node + + if from_beginnig: + self._logs = dict() + else: + self._logs = self._collect_logs() - node = PostgresNode(base_dir=real_base_dir, port=port, bin_dir=bin_dir) - node.should_rm_dirs = True - self.nodes_to_cleanup.append(node) + assert type(self._logs) == dict # noqa: E721 + return - return node + def read(self) -> typing.List[LogDataBlock]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) - def make_simple( - self, - base_dir=None, - port=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}, - checksum=True, - bin_dir=None): - assert type(pg_options) == dict # noqa: E721 - - if checksum and '--data-checksums' not in initdb_params: - initdb_params.append('--data-checksums') - node = self.make_empty(base_dir, port, bin_dir=bin_dir) - node.init( - initdb_params=initdb_params, allow_streaming=set_replication) - - # set major version - pg_version_file = self.os_ops.read(os.path.join(node.data_dir, 'PG_VERSION')) - node.major_version_str = str(pg_version_file.rstrip()) - node.major_version = float(node.major_version_str) - - # Set default parameters - options = { - 'max_connections': 100, - 'shared_buffers': '10MB', - 'fsync': 'off', - 'wal_level': 'logical', - 'hot_standby': 'off', - 'log_line_prefix': '%t [%p]: [%l-1] ', - 'log_statement': 'none', - 'log_duration': 'on', - 'log_min_duration_statement': 0, - 'log_connections': 'on', - 'log_disconnections': 'on', - 'restart_after_crash': 'off', - 'autovacuum': 'off', - # unix_socket_directories will be defined later - } - - # Allow replication in pg_hba.conf - if set_replication: - options['max_wal_senders'] = 10 - - if ptrack_enable: - options['ptrack.map_size'] = '1' - options['shared_preload_libraries'] = 'ptrack' - - if node.major_version >= 13: - options['wal_keep_size'] = '200MB' - else: - options['wal_keep_segments'] = '12' + cur_logs: typing.Dict[__class__.LogInfo] = self._collect_logs() + assert cur_logs is not None + assert type(cur_logs) == dict # noqa: E721 - # Apply given parameters - for option_name, option_value in iteritems(pg_options): - options[option_name] = option_value + assert type(self._logs) == dict # noqa: E721 - # Define delayed propertyes - if not ("unix_socket_directories" in options.keys()): - options["unix_socket_directories"] = __class__._gettempdir_for_socket() + result = list() - # Set config values - node.set_auto_conf(options) + for file_name, cur_log_info in cur_logs.items(): + assert type(file_name) == str # noqa: E721 + assert type(cur_log_info) == __class__.LogInfo # noqa: E721 - # kludge for testgres - # https://github.com/postgrespro/testgres/issues/54 - # for PG >= 13 remove 'wal_keep_segments' parameter - if node.major_version >= 13: - node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + read_pos = 0 - return node + if file_name in self._logs.keys(): + prev_log_info = self._logs[file_name] + assert type(prev_log_info) == __class__.LogInfo # noqa: E721 + read_pos = prev_log_info.position # the previous size - @staticmethod - def _gettempdir_for_socket(): - platform_system_name = platform.system().lower() + file_content_b = self._node.os_ops.read_binary(file_name, read_pos) + assert type(file_content_b) == bytes # noqa: E721 - if platform_system_name == "windows": - return __class__._gettempdir() + # + # A POTENTIAL PROBLEM: file_content_b may contain an incompleted UTF-8 symbol. + # + file_content_s = file_content_b.decode() + assert type(file_content_s) == str # noqa: E721 - # - # [2025-02-17] Hot fix. - # - # Let's use hard coded path as Postgres likes. - # - # pg_config_manual.h: - # - # #ifndef WIN32 - # #define DEFAULT_PGSOCKET_DIR "/tmp" - # #else - # #define DEFAULT_PGSOCKET_DIR "" - # #endif - # - # On the altlinux-10 tempfile.gettempdir() may return - # the path to "private" temp directiry - "/temp/.private//" - # - # But Postgres want to find a socket file in "/tmp" (see above). - # + next_read_pos = read_pos + len(file_content_b) - return "/tmp" + # It is a research/paranoja check. + # When we will process partial UTF-8 symbol, it must be adjusted. + assert cur_log_info.position <= next_read_pos - @staticmethod - def _gettempdir(): - v = tempfile.gettempdir() + cur_log_info.position = next_read_pos - # - # Paranoid checks - # - if type(v) != str: # noqa: E721 - __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + block = __class__.LogDataBlock( + file_name, + read_pos, + file_content_s + ) - if v == "": - __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + result.append(block) - if not os.path.exists(v): - __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + # A new check point + self._logs = cur_logs - # OK - return v + return result + + def _collect_logs(self) -> typing.Dict[LogInfo]: + assert self._node is not None + assert isinstance(self._node, PostgresNode) + + files = [ + self._node.pg_log_file + ] # yapf: disable + result = dict() + + for f in files: + assert type(f) == str # noqa: E721 + + # skip missing files + if not self._node.os_ops.path_exists(f): + continue + + file_size = self._node.os_ops.get_file_size(f) + assert type(file_size) == int # noqa: E721 + assert file_size >= 0 + + result[f] = __class__.LogInfo(file_size) + + return result + + +class PostgresNodeUtils: @staticmethod - def _raise_bugcheck(msg): - assert type(msg) == str # noqa: E721 - assert msg != "" - raise Exception("[BUG CHECK] " + msg) + def delect_port_conflict(log_reader: PostgresNodeLogReader) -> bool: + assert type(log_reader) == PostgresNodeLogReader # noqa: E721 + + blocks = log_reader.read() + assert type(blocks) == list # noqa: E721 + + for block in blocks: + assert type(block) == PostgresNodeLogReader.LogDataBlock # noqa: E721 + + if 'Is another postmaster already running on port' in block.data: + return True + + return False diff --git a/src/node_app.py b/src/node_app.py new file mode 100644 index 00000000..6e7b7c4f --- /dev/null +++ b/src/node_app.py @@ -0,0 +1,317 @@ +from .node import OsOperations +from .node import LocalOperations +from .node import PostgresNode +from .node import PortManager + +import os +import platform +import tempfile +import typing + + +T_DICT_STR_STR = typing.Dict[str, str] +T_LIST_STR = typing.List[str] + + +class NodeApp: + _test_path: str + _os_ops: OsOperations + _port_manager: PortManager + _nodes_to_cleanup: typing.List[PostgresNode] + + def __init__( + self, + test_path: typing.Optional[str] = None, + nodes_to_cleanup: typing.Optional[list] = None, + os_ops: typing.Optional[OsOperations] = None, + port_manager: typing.Optional[PortManager] = None, + ): + assert test_path is None or type(test_path) == str # noqa: E721 + assert os_ops is None or isinstance(os_ops, OsOperations) + assert port_manager is None or isinstance(port_manager, PortManager) + + if os_ops is None: + os_ops = LocalOperations.get_single_instance() + + assert isinstance(os_ops, OsOperations) + self._os_ops = os_ops + self._port_manager = port_manager + + if test_path is None: + self._test_path = os_ops.cwd() + elif os.path.isabs(test_path): + self._test_path = test_path + else: + self._test_path = os_ops.build_path(os_ops.cwd(), test_path) + + if nodes_to_cleanup is None: + self._nodes_to_cleanup = [] + else: + self._nodes_to_cleanup = nodes_to_cleanup + + @property + def test_path(self) -> str: + assert type(self._test_path) == str # noqa: E721 + return self._test_path + + @property + def os_ops(self) -> OsOperations: + assert isinstance(self._os_ops, OsOperations) + return self._os_ops + + @property + def port_manager(self) -> PortManager: + assert self._port_manager is None or isinstance(self._port_manager, PortManager) + return self._port_manager + + @property + def nodes_to_cleanup(self) -> typing.List[PostgresNode]: + assert type(self._nodes_to_cleanup) == list # noqa: E721 + return self._nodes_to_cleanup + + def make_empty( + self, + base_dir: str, + port: typing.Optional[int] = None, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert type(base_dir) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert bin_dir is None or type(bin_dir) == str # noqa: E721 + + assert isinstance(self._os_ops, OsOperations) + assert type(self._test_path) == str # noqa: E721 + + if base_dir is None: + raise ValueError("Argument 'base_dir' is not defined.") + + if base_dir == "": + raise ValueError("Argument 'base_dir' is empty.") + + real_base_dir = self._os_ops.build_path(self._test_path, base_dir) + self._os_ops.rmdirs(real_base_dir, ignore_errors=True) + self._os_ops.makedirs(real_base_dir) + + port_manager: PortManager = None + + if port is None: + port_manager = self._port_manager + + node = PostgresNode( + base_dir=real_base_dir, + port=port, + bin_dir=bin_dir, + os_ops=self._os_ops, + port_manager=port_manager + ) + + try: + assert type(self._nodes_to_cleanup) == list # noqa: E721 + self._nodes_to_cleanup.append(node) + except: # noqa: E722 + node.cleanup(release_resources=True) + raise + + return node + + def make_simple( + self, + base_dir: str, + port: typing.Optional[int] = None, + set_replication: bool = False, + ptrack_enable: bool = False, + initdb_params: typing.Optional[T_LIST_STR] = None, + pg_options: typing.Optional[T_DICT_STR_STR] = None, + checksum: bool = True, + bin_dir: typing.Optional[str] = None + ) -> PostgresNode: + assert type(base_dir) == str # noqa: E721 + assert port is None or type(port) == int # noqa: E721 + assert type(set_replication) == bool # noqa: E721 + assert type(ptrack_enable) == bool # noqa: E721 + assert initdb_params is None or type(initdb_params) == list # noqa: E721 + assert pg_options is None or type(pg_options) == dict # noqa: E721 + assert type(checksum) == bool # noqa: E721 + assert bin_dir is None or type(bin_dir) == str # noqa: E721 + + node = self.make_empty( + base_dir, + port, + bin_dir=bin_dir + ) + + final_initdb_params = initdb_params + + if checksum: + final_initdb_params = __class__._paramlist_append_is_not_exist( + initdb_params, + final_initdb_params, + '--data-checksums' + ) + assert final_initdb_params is not None + assert '--data-checksums' in final_initdb_params + + node.init( + initdb_params=final_initdb_params, + allow_streaming=set_replication + ) + + # set major version + pg_version_file = self._os_ops.read(self._os_ops.build_path(node.data_dir, 'PG_VERSION')) + node.major_version_str = str(pg_version_file.rstrip()) + node.major_version = float(node.major_version_str) + + # Set default parameters + options = { + 'max_connections': 100, + 'shared_buffers': '10MB', + 'fsync': 'off', + 'wal_level': 'logical', + 'hot_standby': 'off', + 'log_line_prefix': '%t [%p]: [%l-1] ', + 'log_statement': 'none', + 'log_duration': 'on', + 'log_min_duration_statement': 0, + 'log_connections': 'on', + 'log_disconnections': 'on', + 'restart_after_crash': 'off', + 'autovacuum': 'off', + # unix_socket_directories will be defined later + } + + # Allow replication in pg_hba.conf + if set_replication: + options['max_wal_senders'] = 10 + + if ptrack_enable: + options['ptrack.map_size'] = '1' + options['shared_preload_libraries'] = 'ptrack' + + if node.major_version >= 13: + options['wal_keep_size'] = '200MB' + else: + options['wal_keep_segments'] = '12' + + # Apply given parameters + if pg_options is not None: + assert type(pg_options) == dict # noqa: E721 + for option_name, option_value in pg_options.items(): + options[option_name] = option_value + + # Define delayed propertyes + if not ("unix_socket_directories" in options.keys()): + options["unix_socket_directories"] = __class__._gettempdir_for_socket() + + # Set config values + node.set_auto_conf(options) + + # kludge for testgres + # https://github.com/postgrespro/testgres/issues/54 + # for PG >= 13 remove 'wal_keep_segments' parameter + if node.major_version >= 13: + node.set_auto_conf({}, 'postgresql.conf', ['wal_keep_segments']) + + return node + + @staticmethod + def _paramlist_has_param( + params: typing.Optional[T_LIST_STR], + param: str + ) -> bool: + assert type(param) == str # noqa: E721 + + if params is None: + return False + + assert type(params) == list # noqa: E721 + + if param in params: + return True + + return False + + @staticmethod + def _paramlist_append( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> T_LIST_STR: + assert user_params is None or type(user_params) == list # noqa: E721 + assert updated_params is None or type(updated_params) == list # noqa: E721 + assert type(param) == str # noqa: E721 + + if updated_params is None: + if user_params is None: + return [param] + + return [*user_params, param] + + assert updated_params is not None + if updated_params is user_params: + return [*user_params, param] + + updated_params.append(param) + return updated_params + + @staticmethod + def _paramlist_append_is_not_exist( + user_params: typing.Optional[T_LIST_STR], + updated_params: typing.Optional[T_LIST_STR], + param: str, + ) -> typing.Optional[T_LIST_STR]: + if __class__._paramlist_has_param(updated_params, param): + return updated_params + return __class__._paramlist_append(user_params, updated_params, param) + + @staticmethod + def _gettempdir_for_socket() -> str: + platform_system_name = platform.system().lower() + + if platform_system_name == "windows": + return __class__._gettempdir() + + # + # [2025-02-17] Hot fix. + # + # Let's use hard coded path as Postgres likes. + # + # pg_config_manual.h: + # + # #ifndef WIN32 + # #define DEFAULT_PGSOCKET_DIR "/tmp" + # #else + # #define DEFAULT_PGSOCKET_DIR "" + # #endif + # + # On the altlinux-10 tempfile.gettempdir() may return + # the path to "private" temp directiry - "/temp/.private//" + # + # But Postgres want to find a socket file in "/tmp" (see above). + # + + return "/tmp" + + @staticmethod + def _gettempdir() -> str: + v = tempfile.gettempdir() + + # + # Paranoid checks + # + if type(v) != str: # noqa: E721 + __class__._raise_bugcheck("tempfile.gettempdir returned a value with type {0}.".format(type(v).__name__)) + + if v == "": + __class__._raise_bugcheck("tempfile.gettempdir returned an empty string.") + + if not os.path.exists(v): + __class__._raise_bugcheck("tempfile.gettempdir returned a not exist path [{0}].".format(v)) + + # OK + return v + + @staticmethod + def _raise_bugcheck(msg): + assert type(msg) == str # noqa: E721 + assert msg != "" + raise Exception("[BUG CHECK] " + msg) diff --git a/src/port_manager.py b/src/port_manager.py new file mode 100644 index 00000000..1ae696c8 --- /dev/null +++ b/src/port_manager.py @@ -0,0 +1,10 @@ +class PortManager: + def __init__(self): + super().__init__() + + def reserve_port(self) -> int: + raise NotImplementedError("PortManager::reserve_port is not implemented.") + + def release_port(self, number: int) -> None: + assert type(number) == int # noqa: E721 + raise NotImplementedError("PortManager::release_port is not implemented.") diff --git a/testgres/pubsub.py b/src/pubsub.py similarity index 100% rename from testgres/pubsub.py rename to src/pubsub.py diff --git a/testgres/standby.py b/src/standby.py similarity index 100% rename from testgres/standby.py rename to src/standby.py diff --git a/testgres/utils.py b/src/utils.py similarity index 84% rename from testgres/utils.py rename to src/utils.py index 2ff6f2a0..c04b4fd3 100644 --- a/testgres/utils.py +++ b/src/utils.py @@ -6,8 +6,6 @@ import os import sys -import socket -import random from contextlib import contextmanager from packaging.version import Version, InvalidVersion @@ -15,18 +13,25 @@ from six import iteritems -from .exceptions import PortForException from .exceptions import ExecUtilException from .config import testgres_config as tconf -from .operations.os_ops import OsOperations -from .operations.remote_ops import RemoteOperations -from .operations.helpers import Helpers as OsHelpers +from testgres.operations.os_ops import OsOperations +from testgres.operations.remote_ops import RemoteOperations +from testgres.operations.local_ops import LocalOperations +from testgres.operations.helpers import Helpers as OsHelpers + +from .impl.port_manager__generic import PortManager__Generic # rows returned by PG_CONFIG _pg_config_data = {} +# +# The old, global "port manager" always worked with LOCAL system +# +_old_port_manager = PortManager__Generic(LocalOperations.get_single_instance()) + # ports used by nodes -bound_ports = set() +bound_ports = _old_port_manager._reserved_ports # re-export version type @@ -43,28 +48,7 @@ def internal__reserve_port(): """ Generate a new port and add it to 'bound_ports'. """ - def LOCAL__is_port_free(port: int) -> bool: - """Check if a port is free to use.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("", port)) - return True - except OSError: - return False - - ports = set(range(1024, 65535)) - assert type(ports) == set # noqa: E721 - assert type(bound_ports) == set # noqa: E721 - ports.difference_update(bound_ports) - - sampled_ports = random.sample(tuple(ports), min(len(ports), 100)) - - for port in sampled_ports: - if LOCAL__is_port_free(port): - bound_ports.add(port) - return port - - raise PortForException("Can't select a port") + return _old_port_manager.reserve_port() def internal__release_port(port): @@ -73,9 +57,7 @@ def internal__release_port(port): """ assert type(port) == int # noqa: E721 - assert port in bound_ports - - bound_ports.discard(port) + return _old_port_manager.release_port(port) reserve_port = internal__reserve_port @@ -159,17 +141,17 @@ def get_bin_path2(os_ops: OsOperations, filename): if pg_config: bindir = get_pg_config(pg_config, os_ops)["BINDIR"] - return os.path.join(bindir, filename) + return os_ops.build_path(bindir, filename) # try PG_BIN pg_bin = os_ops.environ("PG_BIN") if pg_bin: - return os.path.join(pg_bin, filename) + return os_ops.build_path(pg_bin, filename) pg_config_path = os_ops.find_executable('pg_config') if pg_config_path: bindir = get_pg_config(pg_config_path)["BINDIR"] - return os.path.join(bindir, filename) + return os_ops.build_path(bindir, filename) return filename @@ -231,7 +213,7 @@ def cache_pg_config_data(cmd): # try PG_BIN pg_bin = os.environ.get("PG_BIN") if pg_bin: - cmd = os.path.join(pg_bin, "pg_config") + cmd = os_ops.build_path(pg_bin, "pg_config") return cache_pg_config_data(cmd) # try plain name @@ -245,8 +227,17 @@ def get_pg_version2(os_ops: OsOperations, bin_dir=None): assert os_ops is not None assert isinstance(os_ops, OsOperations) + C_POSTGRES_BINARY = "postgres" + # Get raw version (e.g., postgres (PostgreSQL) 9.5.7) - postgres_path = os.path.join(bin_dir, 'postgres') if bin_dir else get_bin_path2(os_ops, 'postgres') + if bin_dir is None: + postgres_path = get_bin_path2(os_ops, C_POSTGRES_BINARY) + else: + # [2025-06-25] OK ? + assert type(bin_dir) == str # noqa: E721 + assert bin_dir != "" + postgres_path = os_ops.build_path(bin_dir, 'postgres') + cmd = [postgres_path, '--version'] raw_ver = os_ops.exec_command(cmd, encoding='utf-8') diff --git a/testgres/exceptions.py b/testgres/exceptions.py deleted file mode 100644 index 20c1a8cf..00000000 --- a/testgres/exceptions.py +++ /dev/null @@ -1,113 +0,0 @@ -# coding: utf-8 - -import six - - -class TestgresException(Exception): - pass - - -class PortForException(TestgresException): - pass - - -@six.python_2_unicode_compatible -class ExecUtilException(TestgresException): - def __init__(self, message=None, command=None, exit_code=0, out=None, error=None): - super(ExecUtilException, self).__init__(message) - - self.message = message - self.command = command - self.exit_code = exit_code - self.out = out - self.error = error - - def __str__(self): - msg = [] - - if self.message: - msg.append(self.message) - - if self.command: - command_s = ' '.join(self.command) if isinstance(self.command, list) else self.command, - msg.append(u'Command: {}'.format(command_s)) - - if self.exit_code: - msg.append(u'Exit code: {}'.format(self.exit_code)) - - if self.error: - msg.append(u'---- Error:\n{}'.format(self.error)) - - if self.out: - msg.append(u'---- Out:\n{}'.format(self.out)) - - return self.convert_and_join(msg) - - @staticmethod - def convert_and_join(msg_list): - # Convert each byte element in the list to str - str_list = [six.text_type(item, 'utf-8') if isinstance(item, bytes) else six.text_type(item) for item in - msg_list] - - # Join the list into a single string with the specified delimiter - return six.text_type('\n').join(str_list) - - -@six.python_2_unicode_compatible -class QueryException(TestgresException): - def __init__(self, message=None, query=None): - super(QueryException, self).__init__(message) - - self.message = message - self.query = query - - def __str__(self): - msg = [] - - if self.message: - msg.append(self.message) - - if self.query: - msg.append(u'Query: {}'.format(self.query)) - - return six.text_type('\n').join(msg) - - -class TimeoutException(QueryException): - pass - - -class CatchUpException(QueryException): - pass - - -@six.python_2_unicode_compatible -class StartNodeException(TestgresException): - def __init__(self, message=None, files=None): - super(StartNodeException, self).__init__(message) - - self.message = message - self.files = files - - def __str__(self): - msg = [] - - if self.message: - msg.append(self.message) - - for f, lines in self.files or []: - msg.append(u'{}\n----\n{}\n'.format(f, lines)) - - return six.text_type('\n').join(msg) - - -class InitNodeException(TestgresException): - pass - - -class BackupException(TestgresException): - pass - - -class InvalidOperationException(TestgresException): - pass diff --git a/testgres/operations/__init__.py b/testgres/operations/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py deleted file mode 100644 index ebbf0f73..00000000 --- a/testgres/operations/helpers.py +++ /dev/null @@ -1,55 +0,0 @@ -import locale - - -class Helpers: - @staticmethod - def _make_get_default_encoding_func(): - # locale.getencoding is added in Python 3.11 - if hasattr(locale, 'getencoding'): - return locale.getencoding - - # It must exist - return locale.getpreferredencoding - - # Prepared pointer on function to get a name of system codepage - _get_default_encoding_func = _make_get_default_encoding_func.__func__() - - @staticmethod - def GetDefaultEncoding(): - # - # Original idea/source was: - # - # def os_ops.get_default_encoding(): - # if not hasattr(locale, 'getencoding'): - # locale.getencoding = locale.getpreferredencoding - # return locale.getencoding() or 'UTF-8' - # - - assert __class__._get_default_encoding_func is not None - - r = __class__._get_default_encoding_func() - - if r: - assert r is not None - assert type(r) == str # noqa: E721 - assert r != "" - return r - - # Is it an unexpected situation? - return 'UTF-8' - - @staticmethod - def PrepareProcessInput(input, encoding): - if not input: - return None - - if type(input) == str: # noqa: E721 - if encoding is None: - return input.encode(__class__.GetDefaultEncoding()) - - assert type(encoding) == str # noqa: E721 - return input.encode(encoding) - - # It is expected! - assert type(input) == bytes # noqa: E721 - return input diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py deleted file mode 100644 index 74323bb8..00000000 --- a/testgres/operations/local_ops.py +++ /dev/null @@ -1,502 +0,0 @@ -import getpass -import logging -import os -import shutil -import stat -import subprocess -import tempfile -import time -import socket - -import psutil -import typing - -from ..exceptions import ExecUtilException -from ..exceptions import InvalidOperationException -from .os_ops import ConnectionParams, OsOperations, get_default_encoding -from .raise_error import RaiseError -from .helpers import Helpers - -try: - from shutil import which as find_executable - from shutil import rmtree -except ImportError: - from distutils.spawn import find_executable - from distutils import rmtree - -CMD_TIMEOUT_SEC = 60 - - -class LocalOperations(OsOperations): - def __init__(self, conn_params=None): - if conn_params is None: - conn_params = ConnectionParams() - super(LocalOperations, self).__init__(conn_params.username) - self.conn_params = conn_params - self.host = conn_params.host - self.ssh_key = None - self.remote = False - self.username = conn_params.username or getpass.getuser() - - @staticmethod - def _process_output(encoding, temp_file_path): - """Process the output of a command from a temporary file.""" - with open(temp_file_path, 'rb') as temp_file: - output = temp_file.read() - if encoding: - output = output.decode(encoding) - return output, None # In Windows stderr writing in stdout - - def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): - assert exec_env is None or type(exec_env) == dict # noqa: E721 - - # TODO: why don't we use the data from input? - - extParams: typing.Dict[str, str] = dict() - - if exec_env is None: - pass - elif len(exec_env) == 0: - pass - else: - env = os.environ.copy() - assert type(env) == dict # noqa: E721 - for v in exec_env.items(): - assert type(v) == tuple # noqa: E721 - assert len(v) == 2 - assert type(v[0]) == str # noqa: E721 - assert v[0] != "" - - if v[1] is None: - env.pop(v[0], None) - else: - assert type(v[1]) == str # noqa: E721 - env[v[0]] = v[1] - - extParams["env"] = env - - with tempfile.NamedTemporaryFile(mode='w+b', delete=False) as temp_file: - stdout = temp_file - stderr = subprocess.STDOUT - process = subprocess.Popen( - cmd, - shell=shell, - stdin=stdin or subprocess.PIPE if input is not None else None, - stdout=stdout, - stderr=stderr, - **extParams, - ) - if get_process: - return process, None, None - temp_file_path = temp_file.name - - # Wait process finished - process.wait() - - output, error = self._process_output(encoding, temp_file_path) - return process, output, error - - def _run_command__generic(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): - assert exec_env is None or type(exec_env) == dict # noqa: E721 - - input_prepared = None - if not get_process: - input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw - - assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 - - extParams: typing.Dict[str, str] = dict() - - if exec_env is None: - pass - elif len(exec_env) == 0: - pass - else: - env = os.environ.copy() - assert type(env) == dict # noqa: E721 - for v in exec_env.items(): - assert type(v) == tuple # noqa: E721 - assert len(v) == 2 - assert type(v[0]) == str # noqa: E721 - assert v[0] != "" - - if v[1] is None: - env.pop(v[0], None) - else: - assert type(v[1]) == str # noqa: E721 - env[v[0]] = v[1] - - extParams["env"] = env - - process = subprocess.Popen( - cmd, - shell=shell, - stdin=stdin or subprocess.PIPE if input is not None else None, - stdout=stdout or subprocess.PIPE, - stderr=stderr or subprocess.PIPE, - **extParams - ) - assert not (process is None) - if get_process: - return process, None, None - try: - output, error = process.communicate(input=input_prepared, timeout=timeout) - except subprocess.TimeoutExpired: - process.kill() - raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - - assert type(output) == bytes # noqa: E721 - assert type(error) == bytes # noqa: E721 - - if encoding: - output = output.decode(encoding) - error = error.decode(encoding) - return process, output, error - - def _run_command(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=None): - """Execute a command and return the process and its output.""" - if os.name == 'nt' and stdout is None: # Windows - method = __class__._run_command__nt - else: # Other OS - method = __class__._run_command__generic - - return method(self, cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) - - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, encoding=None, shell=False, - text=False, input=None, stdin=None, stdout=None, stderr=None, get_process=False, timeout=None, - ignore_errors=False, exec_env=None): - """ - Execute a command in a subprocess and handle the output based on the provided parameters. - """ - assert type(expect_error) == bool # noqa: E721 - assert type(ignore_errors) == bool # noqa: E721 - assert exec_env is None or type(exec_env) == dict # noqa: E721 - - process, output, error = self._run_command(cmd, shell, input, stdin, stdout, stderr, get_process, timeout, encoding, exec_env=exec_env) - if get_process: - return process - - if expect_error: - if process.returncode == 0: - raise InvalidOperationException("We expected an execution error.") - elif ignore_errors: - pass - elif process.returncode == 0: - pass - else: - assert not expect_error - assert not ignore_errors - assert process.returncode != 0 - RaiseError.UtilityExitedWithNonZeroCode( - cmd=cmd, - exit_code=process.returncode, - msg_arg=error or output, - error=error, - out=output) - - if verbose: - return process.returncode, output, error - - return output - - # Environment setup - def environ(self, var_name): - return os.environ.get(var_name) - - def cwd(self): - return os.getcwd() - - def find_executable(self, executable): - return find_executable(executable) - - def is_executable(self, file): - # Check if the file is executable - assert stat.S_IXUSR != 0 - return (os.stat(file).st_mode & stat.S_IXUSR) == stat.S_IXUSR - - def set_env(self, var_name, var_val): - # Check if the directory is already in PATH - os.environ[var_name] = var_val - - def get_name(self): - return os.name - - # Work with dirs - def makedirs(self, path, remove_existing=False): - if remove_existing: - shutil.rmtree(path, ignore_errors=True) - try: - os.makedirs(path) - except FileExistsError: - pass - - # [2025-02-03] Old name of parameter attempts is "retries". - def rmdirs(self, path, ignore_errors=True, attempts=3, delay=1): - """ - Removes a directory and its contents, retrying on failure. - - :param path: Path to the directory. - :param ignore_errors: If True, ignore errors. - :param retries: Number of attempts to remove the directory. - :param delay: Delay between attempts in seconds. - """ - assert type(path) == str # noqa: E721 - assert type(ignore_errors) == bool # noqa: E721 - assert type(attempts) == int # noqa: E721 - assert type(delay) == int or type(delay) == float # noqa: E721 - assert attempts > 0 - assert delay >= 0 - - attempt = 0 - while True: - assert attempt < attempts - attempt += 1 - try: - rmtree(path) - except FileNotFoundError: - pass - except Exception as e: - if attempt < attempt: - errMsg = "Failed to remove directory {0} on attempt {1} ({2}): {3}".format( - path, attempt, type(e).__name__, e - ) - logging.warning(errMsg) - time.sleep(delay) - continue - - assert attempt == attempts - if not ignore_errors: - raise - - return False - - # OK! - return True - - def listdir(self, path): - return os.listdir(path) - - def path_exists(self, path): - return os.path.exists(path) - - @property - def pathsep(self): - os_name = self.get_name() - if os_name == "posix": - pathsep = ":" - elif os_name == "nt": - pathsep = ";" - else: - raise Exception("Unsupported operating system: {}".format(os_name)) - return pathsep - - def mkdtemp(self, prefix=None): - return tempfile.mkdtemp(prefix='{}'.format(prefix)) - - def mkstemp(self, prefix=None): - fd, filename = tempfile.mkstemp(prefix=prefix) - os.close(fd) # Close the file descriptor immediately after creating the file - return filename - - def copytree(self, src, dst): - return shutil.copytree(src, dst) - - # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False): - """ - Write data to a file locally - Args: - filename: The file path where the data will be written. - data: The data to be written to the file. - truncate: If True, the file will be truncated before writing ('w' option); - if False (default), data will be appended ('a' option). - binary: If True, the data will be written in binary mode ('b' option); - if False (default), the data will be written in text mode. - read_and_write: If True, the file will be opened with read and write permissions ('+' option); - if False (default), only write permission will be used. - """ - if isinstance(data, bytes) or isinstance(data, list) and all(isinstance(item, bytes) for item in data): - binary = True - - mode = "w" if truncate else "a" - - if read_and_write: - mode += "+" - - # If it is a bytes str or list - if binary: - mode += "b" - - assert type(mode) == str # noqa: E721 - assert mode != "" - - with open(filename, mode) as file: - if isinstance(data, list): - data2 = [__class__._prepare_line_to_write(s, binary) for s in data] - file.writelines(data2) - else: - data2 = __class__._prepare_data_to_write(data, binary) - file.write(data2) - - @staticmethod - def _prepare_line_to_write(data, binary): - data = __class__._prepare_data_to_write(data, binary) - - if binary: - assert type(data) == bytes # noqa: E721 - return data.rstrip(b'\n') + b'\n' - - assert type(data) == str # noqa: E721 - return data.rstrip('\n') + '\n' - - @staticmethod - def _prepare_data_to_write(data, binary): - if isinstance(data, bytes): - return data if binary else data.decode() - - if isinstance(data, str): - return data if not binary else data.encode() - - raise InvalidOperationException("Unknown type of data type [{0}].".format(type(data).__name__)) - - def touch(self, filename): - """ - Create a new file or update the access and modification times of an existing file. - Args: - filename (str): The name of the file to touch. - - This method behaves as the 'touch' command in Unix. It's equivalent to calling 'touch filename' in the shell. - """ - # cross-python touch(). It is vulnerable to races, but who cares? - with open(filename, "a"): - os.utime(filename, None) - - def read(self, filename, encoding=None, binary=False): - assert type(filename) == str # noqa: E721 - assert encoding is None or type(encoding) == str # noqa: E721 - assert type(binary) == bool # noqa: E721 - - if binary: - if encoding is not None: - raise InvalidOperationException("Enconding is not allowed for read binary operation") - - return self._read__binary(filename) - - # python behavior - assert (None or "abc") == "abc" - assert ("" or "abc") == "abc" - - return self._read__text_with_encoding(filename, encoding or get_default_encoding()) - - def _read__text_with_encoding(self, filename, encoding): - assert type(filename) == str # noqa: E721 - assert type(encoding) == str # noqa: E721 - with open(filename, mode='r', encoding=encoding) as file: # open in a text mode - content = file.read() - assert type(content) == str # noqa: E721 - return content - - def _read__binary(self, filename): - assert type(filename) == str # noqa: E721 - with open(filename, 'rb') as file: # open in a binary mode - content = file.read() - assert type(content) == bytes # noqa: E721 - return content - - def readlines(self, filename, num_lines=0, binary=False, encoding=None): - """ - Read lines from a local file. - If num_lines is greater than 0, only the last num_lines lines will be read. - """ - assert type(num_lines) == int # noqa: E721 - assert type(filename) == str # noqa: E721 - assert type(binary) == bool # noqa: E721 - assert encoding is None or type(encoding) == str # noqa: E721 - assert num_lines >= 0 - - if binary: - assert encoding is None - pass - elif encoding is None: - encoding = get_default_encoding() - assert type(encoding) == str # noqa: E721 - else: - assert type(encoding) == str # noqa: E721 - pass - - mode = 'rb' if binary else 'r' - if num_lines == 0: - with open(filename, mode, encoding=encoding) as file: # open in binary mode - return file.readlines() - else: - bufsize = 8192 - buffers = 1 - - with open(filename, mode, encoding=encoding) as file: # open in binary mode - file.seek(0, os.SEEK_END) - end_pos = file.tell() - - while True: - offset = max(0, end_pos - bufsize * buffers) - file.seek(offset, os.SEEK_SET) - pos = file.tell() - lines = file.readlines() - cur_lines = len(lines) - - if cur_lines >= num_lines or pos == 0: - return lines[-num_lines:] # get last num_lines from lines - - buffers = int( - buffers * max(2, int(num_lines / max(cur_lines, 1))) - ) # Adjust buffer size - - def read_binary(self, filename, offset): - assert type(filename) == str # noqa: E721 - assert type(offset) == int # noqa: E721 - - if offset < 0: - raise ValueError("Negative 'offset' is not supported.") - - with open(filename, 'rb') as file: # open in a binary mode - file.seek(offset, os.SEEK_SET) - r = file.read() - assert type(r) == bytes # noqa: E721 - return r - - def isfile(self, remote_file): - return os.path.isfile(remote_file) - - def isdir(self, dirname): - return os.path.isdir(dirname) - - def get_file_size(self, filename): - assert filename is not None - assert type(filename) == str # noqa: E721 - return os.path.getsize(filename) - - def remove_file(self, filename): - return os.remove(filename) - - # Processes control - def kill(self, pid, signal, expect_error=False): - # Kill the process - cmd = "kill -{} {}".format(signal, pid) - return self.exec_command(cmd, expect_error=expect_error) - - def get_pid(self): - # Get current process id - return os.getpid() - - def get_process_children(self, pid): - assert type(pid) == int # noqa: E721 - return psutil.Process(pid).children() - - def is_port_free(self, number: int) -> bool: - assert type(number) == int # noqa: E721 - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - try: - s.bind(("", number)) - return True - except OSError: - return False diff --git a/testgres/operations/os_ops.py b/testgres/operations/os_ops.py deleted file mode 100644 index d25e76bc..00000000 --- a/testgres/operations/os_ops.py +++ /dev/null @@ -1,124 +0,0 @@ -import getpass -import locale - - -class ConnectionParams: - def __init__(self, host='127.0.0.1', port=None, ssh_key=None, username=None): - self.host = host - self.port = port - self.ssh_key = ssh_key - self.username = username - - -def get_default_encoding(): - if not hasattr(locale, 'getencoding'): - locale.getencoding = locale.getpreferredencoding - return locale.getencoding() or 'UTF-8' - - -class OsOperations: - def __init__(self, username=None): - self.ssh_key = None - self.username = username or getpass.getuser() - - # Command execution - def exec_command(self, cmd, **kwargs): - raise NotImplementedError() - - # Environment setup - def environ(self, var_name): - raise NotImplementedError() - - def cwd(self): - raise NotImplementedError() - - def find_executable(self, executable): - raise NotImplementedError() - - def is_executable(self, file): - # Check if the file is executable - raise NotImplementedError() - - def set_env(self, var_name, var_val): - # Check if the directory is already in PATH - raise NotImplementedError() - - def get_user(self): - return self.username - - def get_name(self): - raise NotImplementedError() - - # Work with dirs - def makedirs(self, path, remove_existing=False): - raise NotImplementedError() - - def rmdirs(self, path, ignore_errors=True): - raise NotImplementedError() - - def listdir(self, path): - raise NotImplementedError() - - def path_exists(self, path): - raise NotImplementedError() - - @property - def pathsep(self): - raise NotImplementedError() - - def mkdtemp(self, prefix=None): - raise NotImplementedError() - - def mkstemp(self, prefix=None): - raise NotImplementedError() - - def copytree(self, src, dst): - raise NotImplementedError() - - # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False): - raise NotImplementedError() - - def touch(self, filename): - raise NotImplementedError() - - def read(self, filename, encoding, binary): - raise NotImplementedError() - - def readlines(self, filename): - raise NotImplementedError() - - def read_binary(self, filename, offset): - assert type(filename) == str # noqa: E721 - assert type(offset) == int # noqa: E721 - assert offset >= 0 - raise NotImplementedError() - - def isfile(self, remote_file): - raise NotImplementedError() - - def isdir(self, dirname): - raise NotImplementedError() - - def get_file_size(self, filename): - raise NotImplementedError() - - def remove_file(self, filename): - assert type(filename) == str # noqa: E721 - raise NotImplementedError() - - # Processes control - def kill(self, pid, signal): - # Kill the process - raise NotImplementedError() - - def get_pid(self): - # Get current process id - raise NotImplementedError() - - def get_process_children(self, pid): - raise NotImplementedError() - - def is_port_free(self, number: int): - assert type(number) == int # noqa: E721 - raise NotImplementedError() diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py deleted file mode 100644 index 0d14be5a..00000000 --- a/testgres/operations/raise_error.py +++ /dev/null @@ -1,57 +0,0 @@ -from ..exceptions import ExecUtilException -from .helpers import Helpers - - -class RaiseError: - @staticmethod - def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out): - assert type(exit_code) == int # noqa: E721 - - msg_arg_s = __class__._TranslateDataIntoString(msg_arg) - assert type(msg_arg_s) == str # noqa: E721 - - msg_arg_s = msg_arg_s.strip() - if msg_arg_s == "": - msg_arg_s = "#no_error_message" - - message = "Utility exited with non-zero code (" + str(exit_code) + "). Error: `" + msg_arg_s + "`" - raise ExecUtilException( - message=message, - command=cmd, - exit_code=exit_code, - out=out, - error=error) - - @staticmethod - def CommandExecutionError(cmd, exit_code, message, error, out): - assert type(exit_code) == int # noqa: E721 - assert type(message) == str # noqa: E721 - assert message != "" - - raise ExecUtilException( - message=message, - command=cmd, - exit_code=exit_code, - out=out, - error=error) - - @staticmethod - def _TranslateDataIntoString(data): - if data is None: - return "" - - if type(data) == bytes: # noqa: E721 - return __class__._TranslateDataIntoString__FromBinary(data) - - return str(data) - - @staticmethod - def _TranslateDataIntoString__FromBinary(data): - assert type(data) == bytes # noqa: E721 - - try: - return data.decode(Helpers.GetDefaultEncoding()) - except UnicodeDecodeError: - pass - - return "#cannot_decode_text" diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py deleted file mode 100644 index e722a2cb..00000000 --- a/testgres/operations/remote_ops.py +++ /dev/null @@ -1,741 +0,0 @@ -import getpass -import os -import platform -import subprocess -import tempfile -import io -import logging -import typing - -from ..exceptions import ExecUtilException -from ..exceptions import InvalidOperationException -from .os_ops import OsOperations, ConnectionParams, get_default_encoding -from .raise_error import RaiseError -from .helpers import Helpers - -error_markers = [b'error', b'Permission denied', b'fatal', b'No such file or directory'] - - -class PsUtilProcessProxy: - def __init__(self, ssh, pid): - assert isinstance(ssh, RemoteOperations) - assert type(pid) == int # noqa: E721 - self.ssh = ssh - self.pid = pid - - def kill(self): - assert isinstance(self.ssh, RemoteOperations) - assert type(self.pid) == int # noqa: E721 - command = ["kill", str(self.pid)] - self.ssh.exec_command(command, encoding=get_default_encoding()) - - def cmdline(self): - assert isinstance(self.ssh, RemoteOperations) - assert type(self.pid) == int # noqa: E721 - command = ["ps", "-p", str(self.pid), "-o", "cmd", "--no-headers"] - output = self.ssh.exec_command(command, encoding=get_default_encoding()) - assert type(output) == str # noqa: E721 - cmdline = output.strip() - # TODO: This code work wrong if command line contains quoted values. Yes? - return cmdline.split() - - -class RemoteOperations(OsOperations): - def __init__(self, conn_params: ConnectionParams): - if not platform.system().lower() == "linux": - raise EnvironmentError("Remote operations are supported only on Linux!") - - super().__init__(conn_params.username) - self.conn_params = conn_params - self.host = conn_params.host - self.port = conn_params.port - self.ssh_key = conn_params.ssh_key - self.ssh_args = [] - if self.ssh_key: - self.ssh_args += ["-i", self.ssh_key] - if self.port: - self.ssh_args += ["-p", self.port] - self.remote = True - self.username = conn_params.username or getpass.getuser() - self.ssh_dest = f"{self.username}@{self.host}" if conn_params.username else self.host - - def __enter__(self): - return self - - def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False, - encoding=None, shell=True, text=False, input=None, stdin=None, stdout=None, - stderr=None, get_process=None, timeout=None, ignore_errors=False, - exec_env=None): - """ - Execute a command in the SSH session. - Args: - - cmd (str): The command to be executed. - """ - assert type(expect_error) == bool # noqa: E721 - assert type(ignore_errors) == bool # noqa: E721 - assert exec_env is None or type(exec_env) == dict # noqa: E721 - - input_prepared = None - if not get_process: - input_prepared = Helpers.PrepareProcessInput(input, encoding) # throw - - assert input_prepared is None or (type(input_prepared) == bytes) # noqa: E721 - - if type(cmd) == str: # noqa: E721 - cmd_s = cmd - elif type(cmd) == list: # noqa: E721 - cmd_s = subprocess.list2cmdline(cmd) - else: - raise ValueError("Invalid 'cmd' argument type - {0}".format(type(cmd).__name__)) - - assert type(cmd_s) == str # noqa: E721 - - cmd_items = __class__._make_exec_env_list(exec_env=exec_env) - cmd_items.append(cmd_s) - - env_cmd_s = ';'.join(cmd_items) - - ssh_cmd = ['ssh', self.ssh_dest] + self.ssh_args + [env_cmd_s] - - process = subprocess.Popen(ssh_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert not (process is None) - if get_process: - return process - - try: - output, error = process.communicate(input=input_prepared, timeout=timeout) - except subprocess.TimeoutExpired: - process.kill() - raise ExecUtilException("Command timed out after {} seconds.".format(timeout)) - - assert type(output) == bytes # noqa: E721 - assert type(error) == bytes # noqa: E721 - - if encoding: - output = output.decode(encoding) - error = error.decode(encoding) - - if expect_error: - if process.returncode == 0: - raise InvalidOperationException("We expected an execution error.") - elif ignore_errors: - pass - elif process.returncode == 0: - pass - else: - assert not expect_error - assert not ignore_errors - assert process.returncode != 0 - RaiseError.UtilityExitedWithNonZeroCode( - cmd=cmd, - exit_code=process.returncode, - msg_arg=error, - error=error, - out=output) - - if verbose: - return process.returncode, output, error - - return output - - # Environment setup - def environ(self, var_name: str) -> str: - """ - Get the value of an environment variable. - Args: - - var_name (str): The name of the environment variable. - """ - cmd = "echo ${}".format(var_name) - return self.exec_command(cmd, encoding=get_default_encoding()).strip() - - def cwd(self): - cmd = 'pwd' - return self.exec_command(cmd, encoding=get_default_encoding()).rstrip() - - def find_executable(self, executable): - search_paths = self.environ("PATH") - if not search_paths: - return None - - search_paths = search_paths.split(self.pathsep) - for path in search_paths: - remote_file = os.path.join(path, executable) - if self.isfile(remote_file): - return remote_file - - return None - - def is_executable(self, file): - # Check if the file is executable - command = ["test", "-x", file] - - exit_status, output, error = self.exec_command(cmd=command, encoding=get_default_encoding(), ignore_errors=True, verbose=True) - - assert type(output) == str # noqa: E721 - assert type(error) == str # noqa: E721 - - if exit_status == 0: - return True - - if exit_status == 1: - return False - - errMsg = "Test operation returns an unknown result code: {0}. File name is [{1}].".format( - exit_status, - file) - - RaiseError.CommandExecutionError( - cmd=command, - exit_code=exit_status, - message=errMsg, - error=error, - out=output - ) - - def set_env(self, var_name: str, var_val: str): - """ - Set the value of an environment variable. - Args: - - var_name (str): The name of the environment variable. - - var_val (str): The value to be set for the environment variable. - """ - return self.exec_command("export {}={}".format(var_name, var_val)) - - def get_name(self): - cmd = 'python3 -c "import os; print(os.name)"' - return self.exec_command(cmd, encoding=get_default_encoding()).strip() - - # Work with dirs - def makedirs(self, path, remove_existing=False): - """ - Create a directory in the remote server. - Args: - - path (str): The path to the directory to be created. - - remove_existing (bool): If True, the existing directory at the path will be removed. - """ - if remove_existing: - cmd = "rm -rf {} && mkdir -p {}".format(path, path) - else: - cmd = "mkdir -p {}".format(path) - try: - exit_status, result, error = self.exec_command(cmd, verbose=True) - except ExecUtilException as e: - raise Exception("Couldn't create dir {} because of error {}".format(path, e.message)) - if exit_status != 0: - raise Exception("Couldn't create dir {} because of error {}".format(path, error)) - return result - - def rmdirs(self, path, ignore_errors=True): - """ - Remove a directory in the remote server. - Args: - - path (str): The path to the directory to be removed. - - ignore_errors (bool): If True, do not raise error if directory does not exist. - """ - assert type(path) == str # noqa: E721 - assert type(ignore_errors) == bool # noqa: E721 - - # ENOENT = 2 - No such file or directory - # ENOTDIR = 20 - Not a directory - - cmd1 = [ - "if", "[", "-d", path, "]", ";", - "then", "rm", "-rf", path, ";", - "elif", "[", "-e", path, "]", ";", - "then", "{", "echo", "cannot remove '" + path + "': it is not a directory", ">&2", ";", "exit", "20", ";", "}", ";", - "else", "{", "echo", "directory '" + path + "' does not exist", ">&2", ";", "exit", "2", ";", "}", ";", - "fi" - ] - - cmd2 = ["sh", "-c", subprocess.list2cmdline(cmd1)] - - try: - self.exec_command(cmd2, encoding=Helpers.GetDefaultEncoding()) - except ExecUtilException as e: - if e.exit_code == 2: # No such file or directory - return True - - if not ignore_errors: - raise - - errMsg = "Failed to remove directory {0} ({1}): {2}".format( - path, type(e).__name__, e - ) - logging.warning(errMsg) - return False - return True - - def listdir(self, path): - """ - List all files and directories in a directory. - Args: - path (str): The path to the directory. - """ - command = ["ls", path] - output = self.exec_command(cmd=command, encoding=get_default_encoding()) - assert type(output) == str # noqa: E721 - result = output.splitlines() - assert type(result) == list # noqa: E721 - return result - - def path_exists(self, path): - command = ["test", "-e", path] - - exit_status, output, error = self.exec_command(cmd=command, encoding=get_default_encoding(), ignore_errors=True, verbose=True) - - assert type(output) == str # noqa: E721 - assert type(error) == str # noqa: E721 - - if exit_status == 0: - return True - - if exit_status == 1: - return False - - errMsg = "Test operation returns an unknown result code: {0}. Path is [{1}].".format( - exit_status, - path) - - RaiseError.CommandExecutionError( - cmd=command, - exit_code=exit_status, - message=errMsg, - error=error, - out=output - ) - - @property - def pathsep(self): - os_name = self.get_name() - if os_name == "posix": - pathsep = ":" - elif os_name == "nt": - pathsep = ";" - else: - raise Exception("Unsupported operating system: {}".format(os_name)) - return pathsep - - def mkdtemp(self, prefix=None): - """ - Creates a temporary directory in the remote server. - Args: - - prefix (str): The prefix of the temporary directory name. - """ - if prefix: - command = ["mktemp", "-d", "-t", prefix + "XXXXXX"] - else: - command = ["mktemp", "-d"] - - exec_exitcode, exec_output, exec_error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) - - assert type(exec_exitcode) == int # noqa: E721 - assert type(exec_output) == str # noqa: E721 - assert type(exec_error) == str # noqa: E721 - - if exec_exitcode != 0: - RaiseError.CommandExecutionError( - cmd=command, - exit_code=exec_exitcode, - message="Could not create temporary directory.", - error=exec_error, - out=exec_output) - - temp_dir = exec_output.strip() - return temp_dir - - def mkstemp(self, prefix=None): - """ - Creates a temporary file in the remote server. - Args: - - prefix (str): The prefix of the temporary directory name. - """ - if prefix: - command = ["mktemp", "-t", prefix + "XXXXXX"] - else: - command = ["mktemp"] - - exec_exitcode, exec_output, exec_error = self.exec_command(command, verbose=True, encoding=get_default_encoding(), ignore_errors=True) - - assert type(exec_exitcode) == int # noqa: E721 - assert type(exec_output) == str # noqa: E721 - assert type(exec_error) == str # noqa: E721 - - if exec_exitcode != 0: - RaiseError.CommandExecutionError( - cmd=command, - exit_code=exec_exitcode, - message="Could not create temporary file.", - error=exec_error, - out=exec_output) - - temp_file = exec_output.strip() - return temp_file - - def copytree(self, src, dst): - if not os.path.isabs(dst): - dst = os.path.join('~', dst) - if self.isdir(dst): - raise FileExistsError("Directory {} already exists.".format(dst)) - return self.exec_command("cp -r {} {}".format(src, dst)) - - # Work with files - def write(self, filename, data, truncate=False, binary=False, read_and_write=False, encoding=None): - if not encoding: - encoding = get_default_encoding() - mode = "wb" if binary else "w" - - with tempfile.NamedTemporaryFile(mode=mode, delete=False) as tmp_file: - # For scp the port is specified by a "-P" option - scp_args = ['-P' if x == '-p' else x for x in self.ssh_args] - - if not truncate: - scp_cmd = ['scp'] + scp_args + [f"{self.ssh_dest}:{filename}", tmp_file.name] - subprocess.run(scp_cmd, check=False) # The file might not exist yet - tmp_file.seek(0, os.SEEK_END) - - if isinstance(data, list): - data2 = [__class__._prepare_line_to_write(s, binary, encoding) for s in data] - tmp_file.writelines(data2) - else: - data2 = __class__._prepare_data_to_write(data, binary, encoding) - tmp_file.write(data2) - - tmp_file.flush() - scp_cmd = ['scp'] + scp_args + [tmp_file.name, f"{self.ssh_dest}:{filename}"] - subprocess.run(scp_cmd, check=True) - - remote_directory = os.path.dirname(filename) - mkdir_cmd = ['ssh'] + self.ssh_args + [self.ssh_dest, f"mkdir -p {remote_directory}"] - subprocess.run(mkdir_cmd, check=True) - - os.remove(tmp_file.name) - - @staticmethod - def _prepare_line_to_write(data, binary, encoding): - data = __class__._prepare_data_to_write(data, binary, encoding) - - if binary: - assert type(data) == bytes # noqa: E721 - return data.rstrip(b'\n') + b'\n' - - assert type(data) == str # noqa: E721 - return data.rstrip('\n') + '\n' - - @staticmethod - def _prepare_data_to_write(data, binary, encoding): - if isinstance(data, bytes): - return data if binary else data.decode(encoding) - - if isinstance(data, str): - return data if not binary else data.encode(encoding) - - raise InvalidOperationException("Unknown type of data type [{0}].".format(type(data).__name__)) - - def touch(self, filename): - """ - Create a new file or update the access and modification times of an existing file on the remote server. - - Args: - filename (str): The name of the file to touch. - - This method behaves as the 'touch' command in Unix. It's equivalent to calling 'touch filename' in the shell. - """ - self.exec_command("touch {}".format(filename)) - - def read(self, filename, binary=False, encoding=None): - assert type(filename) == str # noqa: E721 - assert encoding is None or type(encoding) == str # noqa: E721 - assert type(binary) == bool # noqa: E721 - - if binary: - if encoding is not None: - raise InvalidOperationException("Enconding is not allowed for read binary operation") - - return self._read__binary(filename) - - # python behavior - assert (None or "abc") == "abc" - assert ("" or "abc") == "abc" - - return self._read__text_with_encoding(filename, encoding or get_default_encoding()) - - def _read__text_with_encoding(self, filename, encoding): - assert type(filename) == str # noqa: E721 - assert type(encoding) == str # noqa: E721 - content = self._read__binary(filename) - assert type(content) == bytes # noqa: E721 - buf0 = io.BytesIO(content) - buf1 = io.TextIOWrapper(buf0, encoding=encoding) - content_s = buf1.read() - assert type(content_s) == str # noqa: E721 - return content_s - - def _read__binary(self, filename): - assert type(filename) == str # noqa: E721 - cmd = ["cat", filename] - content = self.exec_command(cmd) - assert type(content) == bytes # noqa: E721 - return content - - def readlines(self, filename, num_lines=0, binary=False, encoding=None): - assert type(num_lines) == int # noqa: E721 - assert type(filename) == str # noqa: E721 - assert type(binary) == bool # noqa: E721 - assert encoding is None or type(encoding) == str # noqa: E721 - - if num_lines > 0: - cmd = ["tail", "-n", str(num_lines), filename] - else: - cmd = ["cat", filename] - - if binary: - assert encoding is None - pass - elif encoding is None: - encoding = get_default_encoding() - assert type(encoding) == str # noqa: E721 - else: - assert type(encoding) == str # noqa: E721 - pass - - result = self.exec_command(cmd, encoding=encoding) - assert result is not None - - if binary: - assert type(result) == bytes # noqa: E721 - lines = result.splitlines() - else: - assert type(result) == str # noqa: E721 - lines = result.splitlines() - - assert type(lines) == list # noqa: E721 - return lines - - def read_binary(self, filename, offset): - assert type(filename) == str # noqa: E721 - assert type(offset) == int # noqa: E721 - - if offset < 0: - raise ValueError("Negative 'offset' is not supported.") - - cmd = ["tail", "-c", "+{}".format(offset + 1), filename] - r = self.exec_command(cmd) - assert type(r) == bytes # noqa: E721 - return r - - def isfile(self, remote_file): - stdout = self.exec_command("test -f {}; echo $?".format(remote_file)) - result = int(stdout.strip()) - return result == 0 - - def isdir(self, dirname): - cmd = "if [ -d {} ]; then echo True; else echo False; fi".format(dirname) - response = self.exec_command(cmd) - return response.strip() == b"True" - - def get_file_size(self, filename): - C_ERR_SRC = "RemoteOpertions::get_file_size" - - assert filename is not None - assert type(filename) == str # noqa: E721 - cmd = ["du", "-b", filename] - - s = self.exec_command(cmd, encoding=get_default_encoding()) - assert type(s) == str # noqa: E721 - - if len(s) == 0: - raise Exception( - "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned an empty string. Check point [{0}][{1}].".format( - C_ERR_SRC, - "#001", - filename - ) - ) - - i = 0 - - while i < len(s) and s[i].isdigit(): - assert s[i] >= '0' - assert s[i] <= '9' - i += 1 - - if i == 0: - raise Exception( - "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( - C_ERR_SRC, - "#002", - filename - ) - ) - - if i == len(s): - raise Exception( - "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( - C_ERR_SRC, - "#003", - filename - ) - ) - - if not s[i].isspace(): - raise Exception( - "[BUG CHECK] Can't get size of file [{2}]. Remote operation returned a bad formatted string. Check point [{0}][{1}].".format( - C_ERR_SRC, - "#004", - filename - ) - ) - - r = 0 - - for i2 in range(0, i): - ch = s[i2] - assert ch >= '0' - assert ch <= '9' - # Here is needed to check overflow or that it is a human-valid result? - r = (r * 10) + ord(ch) - ord('0') - - return r - - def remove_file(self, filename): - cmd = "rm {}".format(filename) - return self.exec_command(cmd) - - # Processes control - def kill(self, pid, signal): - # Kill the process - cmd = "kill -{} {}".format(signal, pid) - return self.exec_command(cmd) - - def get_pid(self): - # Get current process id - return int(self.exec_command("echo $$", encoding=get_default_encoding())) - - def get_process_children(self, pid): - assert type(pid) == int # noqa: E721 - command = ["ssh"] + self.ssh_args + [self.ssh_dest, "pgrep", "-P", str(pid)] - - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - - if result.returncode == 0: - children = result.stdout.strip().splitlines() - return [PsUtilProcessProxy(self, int(child_pid.strip())) for child_pid in children] - - raise ExecUtilException(f"Error in getting process children. Error: {result.stderr}") - - def is_port_free(self, number: int) -> bool: - assert type(number) == int # noqa: E721 - - cmd = ["nc", "-w", "5", "-z", "-v", "localhost", str(number)] - - exit_status, output, error = self.exec_command(cmd=cmd, encoding=get_default_encoding(), ignore_errors=True, verbose=True) - - assert type(output) == str # noqa: E721 - assert type(error) == str # noqa: E721 - - if exit_status == 0: - return __class__._is_port_free__process_0(error) - - if exit_status == 1: - return __class__._is_port_free__process_1(error) - - errMsg = "nc returns an unknown result code: {0}".format(exit_status) - - RaiseError.CommandExecutionError( - cmd=cmd, - exit_code=exit_status, - message=errMsg, - error=error, - out=output - ) - - @staticmethod - def _is_port_free__process_0(error: str) -> bool: - assert type(error) == str # noqa: E721 - # - # Example of error text: - # "Connection to localhost (127.0.0.1) 1024 port [tcp/*] succeeded!\n" - # - # May be here is needed to check error message? - # - return False - - @staticmethod - def _is_port_free__process_1(error: str) -> bool: - assert type(error) == str # noqa: E721 - # - # Example of error text: - # "nc: connect to localhost (127.0.0.1) port 1024 (tcp) failed: Connection refused\n" - # - # May be here is needed to check error message? - # - return True - - @staticmethod - def _make_exec_env_list(exec_env: typing.Dict) -> typing.List[str]: - env: typing.Dict[str, str] = dict() - - # ---------------------------------- SYSTEM ENV - for envvar in os.environ.items(): - if __class__._does_put_envvar_into_exec_cmd(envvar[0]): - env[envvar[0]] = envvar[1] - - # ---------------------------------- EXEC (LOCAL) ENV - if exec_env is None: - pass - else: - for envvar in exec_env.items(): - assert type(envvar) == tuple # noqa: E721 - assert len(envvar) == 2 - assert type(envvar[0]) == str # noqa: E721 - env[envvar[0]] = envvar[1] - - # ---------------------------------- FINAL BUILD - result: typing.List[str] = list() - for envvar in env.items(): - assert type(envvar) == tuple # noqa: E721 - assert len(envvar) == 2 - assert type(envvar[0]) == str # noqa: E721 - - if envvar[1] is None: - result.append("unset " + envvar[0]) - else: - assert type(envvar[1]) == str # noqa: E721 - qvalue = __class__._quote_envvar(envvar[1]) - assert type(qvalue) == str # noqa: E721 - result.append(envvar[0] + "=" + qvalue) - continue - - return result - - sm_envs_for_exec_cmd = ["LANG", "LANGUAGE"] - - @staticmethod - def _does_put_envvar_into_exec_cmd(name: str) -> bool: - assert type(name) == str # noqa: E721 - name = name.upper() - if name.startswith("LC_"): - return True - if name in __class__.sm_envs_for_exec_cmd: - return True - return False - - @staticmethod - def _quote_envvar(value: str) -> str: - assert type(value) == str # noqa: E721 - result = "\"" - for ch in value: - if ch == "\"": - result += "\\\"" - elif ch == "\\": - result += "\\\\" - else: - result += ch - result += "\"" - return result - - -def normalize_error(error): - if isinstance(error, bytes): - return error.decode() - return error diff --git a/testgres/plugins/__init__.py b/testgres/plugins/__init__.py deleted file mode 100644 index 824eadc6..00000000 --- a/testgres/plugins/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .pg_probackup2.pg_probackup2.gdb import GDBobj -from .pg_probackup2.pg_probackup2.app import ProbackupApp, ProbackupException -from .pg_probackup2.pg_probackup2.init_helpers import init_params -from .pg_probackup2.pg_probackup2.storage.fs_backup import FSTestBackupDir - -__all__ = [ - "ProbackupApp", "ProbackupException", "init_params", "FSTestBackupDir", "GDBobj" -] diff --git a/testgres/plugins/pg_probackup2/README.md b/testgres/plugins/pg_probackup2/README.md deleted file mode 100644 index 5139ab0f..00000000 --- a/testgres/plugins/pg_probackup2/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# testgres - pg_probackup2 - -Ccontrol and testing utility for [pg_probackup2](https://github.com/postgrespro/pg_probackup). Python 3.5+ is supported. - - -## Installation - -To install `testgres`, run: - -``` -pip install testgres-pg_probackup -``` - -We encourage you to use `virtualenv` for your testing environment. -The package requires testgres~=1.9.3. - -## Usage - -### Environment variables - -| Variable | Required | Default value | Description | -| - | - | - | - | -| PGPROBACKUP_TMP_DIR | No | tests/tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the current working directory. | -| PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX | No | Temporary test hierarchy | Prefix of the test backup directories. Must be an absolute path. Use this variable to store test backups in a location other than the temporary test hierarchy. | - -See [Testgres](https://github.com/postgrespro/testgres/tree/master#environment) on how to configure a custom Postgres installation using `PG_CONFIG` and `PG_BIN` environment variables. - -### Examples - -Here is an example of what you can do with `testgres-pg_probackup2`: - -```python -# You can see full script here plugins/pg_probackup2/pg_probackup2/tests/basic_test.py -def test_full_backup(self): - # Setting up a simple test node - node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) - - # Initialize and configure Probackup - self.pb.init() - self.pb.add_instance('node', node) - self.pb.set_archiving('node', node) - - # Start the node and initialize pgbench - node.slow_start() - node.pgbench_init(scale=100, no_vacuum=True) - - # Perform backup and validation - backup_id = self.pb.backup_node('node', node) - out = self.pb.validate('node', backup_id) - - # Check if the backup is valid - self.assertIn(f"INFO: Backup {backup_id} is valid", out) -``` - -## Authors - -[Postgres Professional](https://postgrespro.ru/about) diff --git a/testgres/plugins/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/app.py b/testgres/plugins/pg_probackup2/pg_probackup2/app.py deleted file mode 100644 index 5166e9b8..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/app.py +++ /dev/null @@ -1,863 +0,0 @@ -import contextlib -import importlib -import json -import logging -import os -import re -import subprocess -import threading -import time -import unittest - -import testgres - -from .storage.fs_backup import TestBackupDir, FSTestBackupDir -from .gdb import GDBobj -from .init_helpers import init_params - -warning = """ -Wrong splint in show_pb -Original Header:f -{header} -Original Body: -{body} -Splitted Header -{header_split} -Splitted Body -{body_split} -""" - - -class ProbackupException(Exception): - def __init__(self, message, cmd): - self.message = message - self.cmd = cmd - - def __str__(self): - return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) - - -# Local backup control -fs_backup_class = FSTestBackupDir - - -class ProbackupApp: - - def __init__(self, test_class: unittest.TestCase, - pg_node, pb_log_path, test_env, auto_compress_alg, backup_dir, probackup_path=None): - self.process = None - self.test_class = test_class - self.pg_node = pg_node - self.pb_log_path = pb_log_path - self.test_env = test_env - self.auto_compress_alg = auto_compress_alg - self.backup_dir = backup_dir - self.probackup_path = probackup_path or init_params.probackup_path - self.probackup_old_path = init_params.probackup_old_path - self.remote = init_params.remote - self.wal_tree_enabled = init_params.wal_tree_enabled - self.verbose = init_params.verbose - self.archive_compress = init_params.archive_compress - self.test_class.output = None - self.execution_time = None - - def form_daemon_process(self, cmdline, env): - def stream_output(stream: subprocess.PIPE) -> None: - try: - for line in iter(stream.readline, ''): - print(line) - self.test_class.output += line - finally: - stream.close() - - self.process = subprocess.Popen( - cmdline, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env - ) - logging.info(f"Process started in background with PID: {self.process.pid}") - - if self.process.stdout and self.process.stderr: - stdout_thread = threading.Thread(target=stream_output, args=(self.process.stdout,), daemon=True) - stderr_thread = threading.Thread(target=stream_output, args=(self.process.stderr,), daemon=True) - - stdout_thread.start() - stderr_thread.start() - - return self.process.pid - - def run(self, command, gdb=False, old_binary=False, return_id=True, env=None, - skip_log_directory=False, expect_error=False, use_backup_dir=True, daemonize=False): - """ - Run pg_probackup - backup_dir: target directory for making backup - command: commandline options - expect_error: option for ignoring errors and getting error message as a result of running the function - gdb: when True it returns GDBObj(), when tuple('suspend', port) it runs probackup - in suspended gdb mode with attachable gdb port, for local debugging - """ - if isinstance(use_backup_dir, TestBackupDir): - command = [command[0], *use_backup_dir.pb_args, *command[1:]] - elif use_backup_dir: - command = [command[0], *self.backup_dir.pb_args, *command[1:]] - else: - command = [command[0], *self.backup_dir.pb_args[2:], *command[1:]] - - if not self.probackup_old_path and old_binary: - logging.error('PGPROBACKUPBIN_OLD is not set') - exit(1) - - if old_binary: - binary_path = self.probackup_old_path - else: - binary_path = self.probackup_path - - if not env: - env = self.test_env - - strcommand = ' '.join(str(p) for p in command) - if '--log-level-file' in strcommand and \ - '--log-directory' not in strcommand and \ - not skip_log_directory: - command += ['--log-directory=' + self.pb_log_path] - strcommand += ' ' + command[-1] - - if 'pglz' in strcommand and \ - ' -j' not in strcommand and '--thread' not in strcommand: - command += ['-j', '1'] - strcommand += ' -j 1' - - self.test_class.cmd = binary_path + ' ' + strcommand - if self.verbose: - print(self.test_class.cmd) - - cmdline = [binary_path, *command] - if gdb is True: - # general test flow for using GDBObj - return GDBobj(cmdline, self.test_class) - - try: - if type(gdb) is tuple and gdb[0] == 'suspend': - # special test flow for manually debug probackup - gdb_port = gdb[1] - cmdline = ['gdbserver'] + ['localhost:' + str(gdb_port)] + cmdline - logging.warning("pg_probackup gdb suspended, waiting gdb connection on localhost:{0}".format(gdb_port)) - - start_time = time.time() - if daemonize: - return self.form_daemon_process(cmdline, env) - else: - self.test_class.output = subprocess.check_output( - cmdline, - stderr=subprocess.STDOUT, - env=env - ).decode('utf-8', errors='replace') - end_time = time.time() - self.execution_time = end_time - start_time - - if command[0] == 'backup' and return_id: - result = self.get_backup_id() - else: - result = self.test_class.output - if expect_error is True: - assert False, f"Exception was expected, but run finished successful with result: `{result}`\n" \ - f"CMD: {self.test_class.cmd}" - elif expect_error: - assert False, f"Exception was expected {expect_error}, but run finished successful with result: `{result}`\n" \ - f"CMD: {self.test_class.cmd}" - return result - except subprocess.CalledProcessError as e: - self.test_class.output = e.output.decode('utf-8').replace("\r", "") - if expect_error: - return self.test_class.output - else: - raise ProbackupException(self.test_class.output, self.test_class.cmd) - - def get_backup_id(self): - if init_params.major_version > 2: - pattern = re.compile(r"Backup (.*) completed successfully.") - for line in self.test_class.output.splitlines(): - match = pattern.search(line) - if match: - return match.group(1) - else: - for line in self.test_class.output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - return line.split()[2] - return None - - def init(self, options=None, old_binary=False, skip_log_directory=False, expect_error=False, use_backup_dir=True): - if options is None: - options = [] - return self.run(['init'] + options, - old_binary=old_binary, - skip_log_directory=skip_log_directory, - expect_error=expect_error, - use_backup_dir=use_backup_dir - ) - - def add_instance(self, instance, node, old_binary=False, options=None, expect_error=False, datname=False): - if options is None: - options = [] - - if not datname: - datname = 'postgres' - - cmd = [ - 'add-instance', - '--instance={0}'.format(instance), - '-D', node.data_dir, - '--pgport', '%i' % node.port, - '--pgdatabase', datname - ] - - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: - options = options + [ - '--remote-proto=ssh', - '--remote-host=localhost'] - - if self.wal_tree_enabled: - options = options + ['--wal-tree'] - - return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) - - def set_config(self, instance, old_binary=False, options=None, expect_error=False): - if options is None: - options = [] - cmd = [ - 'set-config', - '--instance={0}'.format(instance), - ] - - return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) - - def set_backup(self, instance, backup_id=False, - old_binary=False, options=None, expect_error=False): - if options is None: - options = [] - cmd = [ - 'set-backup', - ] - - if instance: - cmd = cmd + ['--instance={0}'.format(instance)] - - if backup_id: - cmd = cmd + ['-i', backup_id] - - return self.run(cmd + options, old_binary=old_binary, expect_error=expect_error) - - def del_instance(self, instance, options=None, old_binary=False, expect_error=False): - if options is None: - options = [] - cmd = ['del-instance', '--instance={0}'.format(instance)] + options - return self.run(cmd, - old_binary=old_binary, - expect_error=expect_error) - - def backup_node( - self, instance, node, data_dir=False, - backup_type='full', datname=False, options=None, - gdb=False, - old_binary=False, return_id=True, no_remote=False, - env=None, - expect_error=False, - sync=False - ): - if options is None: - options = [] - if not node and not data_dir: - logging.error('You must provide ether node or data_dir for backup') - exit(1) - - if not datname: - datname = 'postgres' - - cmd_list = [ - 'backup', - '--instance={0}'.format(instance), - # "-D", pgdata, - '-p', '%i' % node.port, - '-d', datname - ] - - if data_dir: - cmd_list += ['-D', self._node_dir(data_dir)] - - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary and not no_remote: - options = options + [ - '--remote-proto=ssh', - '--remote-host=localhost'] - - if self.auto_compress_alg and '--compress' in options and \ - self.archive_compress and self.archive_compress != 'zlib': - options = [o if o != '--compress' else f'--compress-algorithm={self.archive_compress}' - for o in options] - - if backup_type: - cmd_list += ['-b', backup_type] - - if not (old_binary or sync): - cmd_list += ['--no-sync'] - - return self.run(cmd_list + options, gdb, old_binary, return_id, env=env, - expect_error=expect_error) - - def backup_replica_node(self, instance, node, data_dir=False, *, - master, backup_type='full', datname=False, - options=None, env=None): - """ - Try to reliably run backup on replica by switching wal at master - at the moment pg_probackup is waiting for archived wal segment - """ - if options is None: - options = [] - assert '--stream' not in options or backup_type == 'page', \ - "backup_replica_node should be used with one of archive-mode or " \ - "page-stream mode" - - options = options.copy() - if not any('--log-level-file' in x for x in options): - options.append('--log-level-file=INFO') - - gdb = self.backup_node( - instance, node, data_dir, - backup_type=backup_type, - datname=datname, - options=options, - env=env, - gdb=True) - gdb.set_breakpoint('wait_wal_lsn') - # we need to break on wait_wal_lsn in pg_stop_backup - gdb.run_until_break() - if backup_type == 'page': - self.switch_wal_segment(master) - if '--stream' not in options: - gdb.continue_execution_until_break() - self.switch_wal_segment(master) - gdb.continue_execution_until_exit() - - output = self.read_pb_log() - self.unlink_pg_log() - parsed_output = re.compile(r'Backup \S+ completed').search(output) - assert parsed_output, f"Expected: `Backup 'backup_id' completed`, but found `{output}`" - backup_id = parsed_output[0].split(' ')[1] - return (backup_id, output) - - def checkdb_node( - self, use_backup_dir=False, instance=False, data_dir=False, - options=None, gdb=False, old_binary=False, - skip_log_directory=False, - expect_error=False - ): - if options is None: - options = [] - cmd_list = ["checkdb"] - - if instance: - cmd_list += ["--instance={0}".format(instance)] - - if data_dir: - cmd_list += ["-D", self._node_dir(data_dir)] - - return self.run(cmd_list + options, gdb, old_binary, - skip_log_directory=skip_log_directory, expect_error=expect_error, - use_backup_dir=use_backup_dir) - - def merge_backup( - self, instance, backup_id, - gdb=False, old_binary=False, options=None, expect_error=False): - if options is None: - options = [] - cmd_list = [ - 'merge', - '--instance={0}'.format(instance), - '-i', backup_id - ] - - return self.run(cmd_list + options, gdb, old_binary, expect_error=expect_error) - - def restore_node( - self, instance, node=None, restore_dir=None, - backup_id=None, old_binary=False, options=None, - gdb=False, - expect_error=False, - sync=False - ): - if options is None: - options = [] - if node: - if isinstance(node, str): - data_dir = node - else: - data_dir = node.data_dir - elif restore_dir: - data_dir = self._node_dir(restore_dir) - else: - raise ValueError("You must provide ether node or base_dir for backup") - - cmd_list = [ - 'restore', - '-D', data_dir, - '--instance={0}'.format(instance) - ] - - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: - options = options + [ - '--remote-proto=ssh', - '--remote-host=localhost'] - - if backup_id: - cmd_list += ['-i', backup_id] - - if not (old_binary or sync): - cmd_list += ['--no-sync'] - - return self.run(cmd_list + options, gdb=gdb, old_binary=old_binary, expect_error=expect_error) - - def catchup_node( - self, - backup_mode, source_pgdata, destination_node, - options=None, - remote_host='localhost', - remote_port=None, - expect_error=False, - gdb=False - ): - - if options is None: - options = [] - cmd_list = [ - 'catchup', - '--backup-mode={0}'.format(backup_mode), - '--source-pgdata={0}'.format(source_pgdata), - '--destination-pgdata={0}'.format(destination_node.data_dir) - ] - if self.remote: - cmd_list += ['--remote-proto=ssh', f'--remote-host={remote_host}'] - if remote_port: - cmd_list.append(f'--remote-port={remote_port}') - if self.verbose: - cmd_list += [ - '--log-level-file=VERBOSE', - '--log-directory={0}'.format(destination_node.logs_dir) - ] - - return self.run(cmd_list + options, gdb=gdb, expect_error=expect_error, use_backup_dir=False) - - def show( - self, instance=None, backup_id=None, - options=None, as_text=False, as_json=True, old_binary=False, - env=None, - expect_error=False, - gdb=False - ): - - if options is None: - options = [] - backup_list = [] - specific_record = {} - cmd_list = [ - 'show', - ] - if instance: - cmd_list += ['--instance={0}'.format(instance)] - - if backup_id: - cmd_list += ['-i', backup_id] - - # AHTUNG, WARNING will break json parsing - if as_json: - cmd_list += ['--format=json', '--log-level-console=error'] - - if as_text: - # You should print it when calling as_text=true - return self.run(cmd_list + options, old_binary=old_binary, env=env, - expect_error=expect_error, gdb=gdb) - - # get show result as list of lines - if as_json: - text_json = str(self.run(cmd_list + options, old_binary=old_binary, env=env, - expect_error=expect_error, gdb=gdb)) - try: - if expect_error: - return text_json - data = json.loads(text_json) - except ValueError: - assert False, f"Couldn't parse {text_json} as json. " \ - f"Check that you don't have additional messages inside the log or use 'as_text=True'" - - for instance_data in data: - # find specific instance if requested - if instance and instance_data['instance'] != instance: - continue - - for backup in reversed(instance_data['backups']): - # find specific backup if requested - if backup_id: - if backup['id'] == backup_id: - return backup - else: - backup_list.append(backup) - - if backup_id is not None: - assert False, "Failed to find backup with ID: {0}".format(backup_id) - - return backup_list - else: - show_splitted = self.run(cmd_list + options, old_binary=old_binary, env=env, - expect_error=expect_error).splitlines() - if instance is not None and backup_id is None: - # cut header(ID, Mode, etc) from show as single string - header = show_splitted[1:2][0] - # cut backup records from show as single list - # with string for every backup record - body = show_splitted[3:] - # inverse list so oldest record come first - body = body[::-1] - # split string in list with string for every header element - header_split = re.split(' +', header) - # Remove empty items - for i in header_split: - if i == '': - header_split.remove(i) - continue - header_split = [ - header_element.rstrip() for header_element in header_split - ] - for backup_record in body: - backup_record = backup_record.rstrip() - # split list with str for every backup record element - backup_record_split = re.split(' +', backup_record) - # Remove empty items - for i in backup_record_split: - if i == '': - backup_record_split.remove(i) - if len(header_split) != len(backup_record_split): - logging.error(warning.format( - header=header, body=body, - header_split=header_split, - body_split=backup_record_split) - ) - exit(1) - new_dict = dict(zip(header_split, backup_record_split)) - backup_list.append(new_dict) - return backup_list - else: - # cut out empty lines and lines started with # - # and other garbage then reconstruct it as dictionary - # print show_splitted - sanitized_show = [item for item in show_splitted if item] - sanitized_show = [ - item for item in sanitized_show if not item.startswith('#') - ] - # print sanitized_show - for line in sanitized_show: - name, var = line.partition(' = ')[::2] - var = var.strip('"') - var = var.strip("'") - specific_record[name.strip()] = var - - if not specific_record: - assert False, "Failed to find backup with ID: {0}".format(backup_id) - - return specific_record - - def show_archive( - self, instance=None, options=None, - as_text=False, as_json=True, old_binary=False, - tli=0, - expect_error=False - ): - if options is None: - options = [] - cmd_list = [ - 'show', - '--archive', - ] - if instance: - cmd_list += ['--instance={0}'.format(instance)] - - # AHTUNG, WARNING will break json parsing - if as_json: - cmd_list += ['--format=json', '--log-level-console=error'] - - if as_text: - # You should print it when calling as_text=true - return self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) - - if as_json: - if as_text: - data = self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) - else: - data = json.loads(self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error)) - - if instance: - instance_timelines = None - for instance_name in data: - if instance_name['instance'] == instance: - instance_timelines = instance_name['timelines'] - break - - if tli > 0: - for timeline in instance_timelines: - if timeline['tli'] == tli: - return timeline - - return {} - - if instance_timelines: - return instance_timelines - - return data - else: - show_splitted = self.run(cmd_list + options, old_binary=old_binary, - expect_error=expect_error).splitlines() - logging.error(show_splitted) - exit(1) - - def validate( - self, instance=None, backup_id=None, - options=None, old_binary=False, gdb=False, expect_error=False - ): - if options is None: - options = [] - cmd_list = [ - 'validate', - ] - if instance: - cmd_list += ['--instance={0}'.format(instance)] - if backup_id: - cmd_list += ['-i', backup_id] - - return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, - expect_error=expect_error) - - def delete( - self, instance, backup_id=None, - options=None, old_binary=False, gdb=False, expect_error=False): - if options is None: - options = [] - cmd_list = [ - 'delete', - ] - - cmd_list += ['--instance={0}'.format(instance)] - if backup_id: - cmd_list += ['-i', backup_id] - - return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, - expect_error=expect_error) - - def delete_expired( - self, instance, options=None, old_binary=False, expect_error=False): - if options is None: - options = [] - cmd_list = [ - 'delete', - '--instance={0}'.format(instance) - ] - return self.run(cmd_list + options, old_binary=old_binary, expect_error=expect_error) - - def show_config(self, instance, old_binary=False, expect_error=False, gdb=False): - out_dict = {} - cmd_list = [ - 'show-config', - '--instance={0}'.format(instance) - ] - - res = self.run(cmd_list, old_binary=old_binary, expect_error=expect_error, gdb=gdb).splitlines() - for line in res: - if not line.startswith('#'): - name, var = line.partition(' = ')[::2] - out_dict[name] = var - return out_dict - - def run_binary(self, command, asynchronous=False, env=None): - - if not env: - env = self.test_env - - if self.verbose: - print([' '.join(map(str, command))]) - try: - if asynchronous: - return subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env - ) - else: - self.test_class.output = subprocess.check_output( - command, - stderr=subprocess.STDOUT, - env=env - ).decode('utf-8') - return self.test_class.output - except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode('utf-8'), command) - - def _node_dir(self, base_dir): - return os.path.join(self.pg_node.test_path, base_dir) - - def set_archiving( - self, instance, node, replica=False, - overwrite=False, compress=True, old_binary=False, - log_level=False, archive_timeout=False, - custom_archive_command=None): - - # parse postgresql.auto.conf - options = {} - if replica: - options['archive_mode'] = 'always' - options['hot_standby'] = 'on' - else: - options['archive_mode'] = 'on' - - if custom_archive_command is None: - archive_command = " ".join([f'"{init_params.probackup_path}"', - 'archive-push', *self.backup_dir.pb_args]) - if os.name == 'posix': - # Dash produces a core dump when it gets a SIGQUIT from its - # child process so replace the shell with pg_probackup - archive_command = 'exec ' + archive_command - elif os.name == "nt": - archive_command = archive_command.replace("\\", "\\\\") - archive_command += f' --instance={instance}' - - # don`t forget to kill old_binary after remote ssh release - if init_params.remote and not old_binary: - archive_command += ' --remote-proto=ssh --remote-host=localhost' - - if init_params.archive_compress and compress: - archive_command += ' --compress-algorithm=' + init_params.archive_compress - - if overwrite: - archive_command += ' --overwrite' - - if init_params.major_version > 2: - archive_command += ' --log-level-console=trace' - else: - archive_command += ' --log-level-console=VERBOSE' - archive_command += ' -j 5' - archive_command += ' --batch-size 10' - - archive_command += ' --no-sync' - - if archive_timeout: - archive_command += f' --archive-timeout={archive_timeout}' - - if os.name == 'posix': - archive_command += ' --wal-file-path=%p --wal-file-name=%f' - - elif os.name == 'nt': - archive_command += ' --wal-file-path="%p" --wal-file-name="%f"' - - if log_level: - archive_command += f' --log-level-console={log_level}' - else: # custom_archive_command is not None - archive_command = custom_archive_command - options['archive_command'] = archive_command - - node.set_auto_conf(options) - - def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): - """ - Execute pg_switch_wal() in given node - - Args: - node: an instance of PostgresNode or NodeConnection class - """ - if isinstance(node, testgres.PostgresNode): - with node.connect('postgres') as con: - if and_tx: - con.execute('select txid_current()') - lsn = con.execute('select pg_switch_wal()')[0][0] - else: - lsn = node.execute('select pg_switch_wal()')[0][0] - - if sleep_seconds > 0: - time.sleep(sleep_seconds) - return lsn - - @contextlib.contextmanager - def switch_wal_after(self, node, seconds, and_tx=True): - tm = threading.Timer(seconds, self.switch_wal_segment, [node, 0, and_tx]) - tm.start() - try: - yield - finally: - tm.cancel() - tm.join() - - def read_pb_log(self): - with open(os.path.join(self.pb_log_path, 'pg_probackup.log')) as fl: - return fl.read() - - def unlink_pg_log(self): - os.unlink(os.path.join(self.pb_log_path, 'pg_probackup.log')) - - def load_backup_class(fs_type): - fs_type = os.environ.get('PROBACKUP_FS_TYPE') - implementation = f"{__package__}.fs_backup.FSTestBackupDir" - if fs_type: - implementation = fs_type - - logging.info("Using ", implementation) - module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) - - module = importlib.import_module(module_name) - - return getattr(module, class_name) - - def archive_push(self, instance, node, wal_file_name, wal_file_path=None, options=None, expect_error=False): - if options is None: - options = [] - cmd = [ - 'archive-push', - '--instance={0}'.format(instance), - '--wal-file-name={0}'.format(wal_file_name), - ] - if wal_file_path is None: - cmd = cmd + ['--wal-file-path={0}'.format(os.path.join(node.data_dir, 'pg_wal'))] - else: - cmd = cmd + ['--wal-file-path={0}'.format(wal_file_path)] - return self.run(cmd + options, expect_error=expect_error) - - def archive_get(self, instance, wal_file_name, wal_file_path, options=None, expect_error=False): - if options is None: - options = [] - cmd = [ - 'archive-get', - '--instance={0}'.format(instance), - '--wal-file-name={0}'.format(wal_file_name), - '--wal-file-path={0}'.format(wal_file_path), - ] - return self.run(cmd + options, expect_error=expect_error) - - def maintain( - self, instance=None, backup_id=None, - options=None, old_binary=False, gdb=False, expect_error=False - ): - if options is None: - options = [] - cmd_list = [ - 'maintain', - ] - if instance: - cmd_list += ['--instance={0}'.format(instance)] - if backup_id: - cmd_list += ['-i', backup_id] - - return self.run(cmd_list + options, old_binary=old_binary, gdb=gdb, - expect_error=expect_error) - - def build_backup_dir(self, backup='backup'): - return fs_backup_class(rel_path=self.rel_path, backup=backup) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py b/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py deleted file mode 100644 index 2424c04d..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/gdb.py +++ /dev/null @@ -1,349 +0,0 @@ -import functools -import os -import re -import subprocess -import sys -import unittest -from time import sleep - - -class GdbException(Exception): - def __init__(self, message="False"): - self.message = message - - def __str__(self): - return '\n ERROR: {0}\n'.format(repr(self.message)) - - -class GDBobj: - _gdb_enabled = False - _gdb_ok = False - _gdb_ptrace_ok = False - - def __init__(self, cmd, env, attach=False): - self.verbose = env.verbose - self.output = '' - self._did_quit = False - self.has_breakpoint = False - - # Check gdb flag is set up - if not hasattr(env, "_gdb_decorated") or not env._gdb_decorated: - raise GdbException("Test should be decorated with @needs_gdb") - if not self._gdb_enabled: - raise GdbException("No `PGPROBACKUP_GDB=on` is set.") - if not self._gdb_ok: - if not self._gdb_ptrace_ok: - raise GdbException("set /proc/sys/kernel/yama/ptrace_scope to 0" - " to run GDB tests") - raise GdbException("No gdb usage possible.") - - # Check gdb presence - try: - gdb_version, _ = subprocess.Popen( - ['gdb', '--version'], - stdout=subprocess.PIPE - ).communicate() - except OSError: - raise GdbException("Couldn't find gdb on the path") - - self.base_cmd = [ - 'gdb', - '--interpreter', - 'mi2', - ] - - if attach: - self.cmd = self.base_cmd + ['--pid'] + cmd - else: - self.cmd = self.base_cmd + ['--args'] + cmd - - # Get version - gdb_version_number = re.search( - br"^GNU gdb [^\d]*(\d+)\.(\d)", - gdb_version) - self.major_version = int(gdb_version_number.group(1)) - self.minor_version = int(gdb_version_number.group(2)) - - if self.verbose: - print([' '.join(map(str, self.cmd))]) - - self.proc = subprocess.Popen( - self.cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - bufsize=0, - text=True, - errors='replace', - ) - self.gdb_pid = self.proc.pid - - while True: - line = self.get_line() - - if 'No such process' in line: - raise GdbException(line) - - if not line.startswith('(gdb)'): - pass - else: - break - - def __del__(self): - if not self._did_quit and hasattr(self, "proc"): - try: - self.quit() - except subprocess.TimeoutExpired: - self.kill() - - def get_line(self): - line = self.proc.stdout.readline() - self.output += line - return line - - def kill(self): - self._did_quit = True - self.proc.kill() - self.proc.wait(3) - self.proc.stdin.close() - self.proc.stdout.close() - - def terminate_subprocess(self): - self._execute('kill') - - def set_breakpoint(self, location): - - result = self._execute('break ' + location) - self.has_breakpoint = True - for line in result: - if line.startswith('~"Breakpoint'): - return - - elif line.startswith('=breakpoint-created'): - return - - elif line.startswith('^error'): # or line.startswith('(gdb)'): - break - - elif line.startswith('&"break'): - pass - - elif line.startswith('&"Function'): - raise GdbException(line) - - elif line.startswith('&"No line'): - raise GdbException(line) - - elif line.startswith('~"Make breakpoint pending on future shared'): - raise GdbException(line) - - raise GdbException( - 'Failed to set breakpoint.\n Output:\n {0}'.format(result) - ) - - def remove_all_breakpoints(self): - if not self.has_breakpoint: - return - - result = self._execute('delete') - self.has_breakpoint = False - for line in result: - - if line.startswith('^done'): - return - - raise GdbException( - 'Failed to remove breakpoints.\n Output:\n {0}'.format(result) - ) - - def run_until_break(self): - result = self._execute('run', False) - for line in result: - if line.startswith('*stopped,reason="breakpoint-hit"'): - return - raise GdbException( - 'Failed to run until breakpoint.\n' - ) - - def continue_execution_until_running(self): - result = self._execute('continue') - - for line in result: - if line.startswith('*running') or line.startswith('^running'): - return - if line.startswith('*stopped,reason="breakpoint-hit"'): - continue - if line.startswith('*stopped,reason="exited-normally"'): - continue - - raise GdbException( - 'Failed to continue execution until running.\n' - ) - - def signal(self, sig): - if 'KILL' in sig: - self.remove_all_breakpoints() - self._execute(f'signal {sig}') - - def continue_execution_until_exit(self): - self.remove_all_breakpoints() - result = self._execute('continue', False) - - for line in result: - if line.startswith('*running'): - continue - if line.startswith('*stopped,reason="breakpoint-hit"'): - continue - if line.startswith('*stopped,reason="exited') or line == '*stopped\n': - self.quit() - return - - raise GdbException( - 'Failed to continue execution until exit.\n' - ) - - def continue_execution_until_error(self): - self.remove_all_breakpoints() - result = self._execute('continue', False) - - for line in result: - if line.startswith('^error'): - return - if line.startswith('*stopped,reason="exited'): - return - if line.startswith( - '*stopped,reason="signal-received",signal-name="SIGABRT"'): - return - - raise GdbException( - 'Failed to continue execution until error.\n') - - def continue_execution_until_break(self, ignore_count=0): - if ignore_count > 0: - result = self._execute( - 'continue ' + str(ignore_count), - False - ) - else: - result = self._execute('continue', False) - - for line in result: - if line.startswith('*stopped,reason="breakpoint-hit"'): - return - if line.startswith('*stopped,reason="exited-normally"'): - break - - raise GdbException( - 'Failed to continue execution until break.\n') - - def show_backtrace(self): - return self._execute("backtrace", running=False) - - def stopped_in_breakpoint(self): - while True: - line = self.get_line() - if self.verbose: - print(line) - if line.startswith('*stopped,reason="breakpoint-hit"'): - return True - - def detach(self): - if not self._did_quit: - self._execute('detach') - - def quit(self): - if not self._did_quit: - self._did_quit = True - self.proc.terminate() - self.proc.wait(3) - self.proc.stdin.close() - self.proc.stdout.close() - - # use for breakpoint, run, continue - def _execute(self, cmd, running=True): - output = [] - self.proc.stdin.flush() - self.proc.stdin.write(cmd + '\n') - self.proc.stdin.flush() - sleep(1) - - # look for command we just send - while True: - line = self.get_line() - if self.verbose: - print(repr(line)) - - if cmd not in line: - continue - else: - break - - while True: - line = self.get_line() - output += [line] - if self.verbose: - print(repr(line)) - if line.startswith('^done') or line.startswith('*stopped'): - break - if line.startswith('^error'): - break - if running and (line.startswith('*running') or line.startswith('^running')): - # if running and line.startswith('*running'): - break - return output - - -def _set_gdb(self): - test_env = os.environ.copy() - self._gdb_enabled = test_env.get('PGPROBACKUP_GDB') == 'ON' - self._gdb_ok = self._gdb_enabled - if not self._gdb_enabled or sys.platform != 'linux': - return - try: - with open('/proc/sys/kernel/yama/ptrace_scope') as f: - ptrace = f.read() - except FileNotFoundError: - self._gdb_ptrace_ok = True - return - self._gdb_ptrace_ok = int(ptrace) == 0 - self._gdb_ok = self._gdb_ok and self._gdb_ptrace_ok - - -def _check_gdb_flag_or_skip_test(): - if not GDBobj._gdb_enabled: - return ("skip", - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) - if GDBobj._gdb_ok: - return None - if not GDBobj._gdb_ptrace_ok: - return ("fail", "set /proc/sys/kernel/yama/ptrace_scope to 0" - " to run GDB tests") - else: - return ("fail", "use of gdb is not possible") - - -def needs_gdb(func): - check = _check_gdb_flag_or_skip_test() - if not check: - @functools.wraps(func) - def ok_wrapped(self): - self._gdb_decorated = True - func(self) - - return ok_wrapped - reason = check[1] - if check[0] == "skip": - return unittest.skip(reason)(func) - elif check[0] == "fail": - @functools.wraps(func) - def fail_wrapper(self): - self.fail(reason) - - return fail_wrapper - else: - raise "Wrong action {0}".format(check) - - -_set_gdb(GDBobj) diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py b/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py deleted file mode 100644 index c4570a39..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/init_helpers.py +++ /dev/null @@ -1,226 +0,0 @@ -import logging -from functools import reduce -import getpass -import os -import re -import shutil -import subprocess -import sys -import testgres - -try: - import lz4.frame # noqa: F401 - - HAVE_LZ4 = True -except ImportError as e: - HAVE_LZ4 = False - LZ4_error = e - -try: - import zstd # noqa: F401 - - HAVE_ZSTD = True -except ImportError as e: - HAVE_ZSTD = False - ZSTD_error = e - -delete_logs = os.getenv('KEEP_LOGS') not in ['1', 'y', 'Y'] - -try: - testgres.configure_testgres( - cache_initdb=False, - cached_initdb_dir=False, - node_cleanup_full=delete_logs) -except Exception as e: - logging.warning("Can't configure testgres: {0}".format(e)) - - -class Init(object): - def __init__(self): - if '-v' in sys.argv or '--verbose' in sys.argv: - self.verbose = True - else: - self.verbose = False - - self._pg_config = testgres.get_pg_config() - self.is_enterprise = self._pg_config.get('PGPRO_EDITION', None) == 'enterprise' - self.is_shardman = self._pg_config.get('PGPRO_EDITION', None) == 'shardman' - self.is_pgpro = 'PGPRO_EDITION' in self._pg_config - self.is_nls_enabled = 'enable-nls' in self._pg_config['CONFIGURE'] - self.is_lz4_enabled = '-llz4' in self._pg_config['LIBS'] - version = self._pg_config['VERSION'].rstrip('develalphabetapre') - parts = [*version.split(' ')[1].split('.'), '0', '0'][:3] - parts[0] = re.match(r'\d+', parts[0]).group() - self.pg_config_version = reduce(lambda v, x: v * 100 + int(x), parts, 0) - - os.environ['LANGUAGE'] = 'en' # set default locale language to en. All messages will use this locale - test_env = os.environ.copy() - envs_list = [ - 'LANGUAGE', - 'LC_ALL', - 'PGCONNECT_TIMEOUT', - 'PGDATA', - 'PGDATABASE', - 'PGHOSTADDR', - 'PGREQUIRESSL', - 'PGSERVICE', - 'PGSSLMODE', - 'PGUSER', - 'PGPORT', - 'PGHOST' - ] - - for e in envs_list: - test_env.pop(e, None) - - test_env['LC_MESSAGES'] = 'C' - test_env['LC_TIME'] = 'C' - self._test_env = test_env - - # Get the directory from which the script was executed - self.source_path = os.getcwd() - tmp_path = test_env.get('PGPROBACKUP_TMP_DIR') - if tmp_path and os.path.isabs(tmp_path): - self.tmp_path = tmp_path - else: - self.tmp_path = os.path.abspath( - os.path.join(self.source_path, tmp_path or os.path.join('tests', 'tmp_dirs')) - ) - - os.makedirs(self.tmp_path, exist_ok=True) - - self.username = getpass.getuser() - - self.probackup_path = None - if 'PGPROBACKUPBIN' in test_env: - if shutil.which(test_env["PGPROBACKUPBIN"]): - self.probackup_path = test_env["PGPROBACKUPBIN"] - else: - if self.verbose: - print('PGPROBACKUPBIN is not an executable file') - - if not self.probackup_path: - probackup_path_tmp = os.path.join( - testgres.get_pg_config()['BINDIR'], 'pg_probackup') - - if os.path.isfile(probackup_path_tmp): - if not os.access(probackup_path_tmp, os.X_OK): - logging.warning('{0} is not an executable file'.format( - probackup_path_tmp)) - else: - self.probackup_path = probackup_path_tmp - - if not self.probackup_path: - probackup_path_tmp = self.source_path - - if os.path.isfile(probackup_path_tmp): - if not os.access(probackup_path_tmp, os.X_OK): - logging.warning('{0} is not an executable file'.format( - probackup_path_tmp)) - else: - self.probackup_path = probackup_path_tmp - - if not self.probackup_path: - raise Exception('pg_probackup binary is not found') - - if os.name == 'posix': - self.EXTERNAL_DIRECTORY_DELIMITER = ':' - os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ':' + os.environ['PATH'] - - elif os.name == 'nt': - self.EXTERNAL_DIRECTORY_DELIMITER = ';' - os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ';' + os.environ['PATH'] - - self.probackup_old_path = None - if 'PGPROBACKUPBIN_OLD' in test_env: - if (os.path.isfile(test_env['PGPROBACKUPBIN_OLD']) and os.access(test_env['PGPROBACKUPBIN_OLD'], os.X_OK)): - self.probackup_old_path = test_env['PGPROBACKUPBIN_OLD'] - else: - if self.verbose: - print('PGPROBACKUPBIN_OLD is not an executable file') - - self.probackup_version = None - self.old_probackup_version = None - - probackup_version_output = subprocess.check_output( - [self.probackup_path, "--version"], - stderr=subprocess.STDOUT, - ).decode('utf-8') - match = re.search(r"\d+\.\d+\.\d+", - probackup_version_output) - self.probackup_version = match.group(0) if match else None - match = re.search(r"\(compressions: ([^)]*)\)", probackup_version_output) - compressions = match.group(1) if match else None - if compressions: - self.probackup_compressions = {s.strip() for s in compressions.split(',')} - else: - self.probackup_compressions = [] - - if self.probackup_old_path: - old_probackup_version_output = subprocess.check_output( - [self.probackup_old_path, "--version"], - stderr=subprocess.STDOUT, - ).decode('utf-8') - match = re.search(r"\d+\.\d+\.\d+", - old_probackup_version_output) - self.old_probackup_version = match.group(0) if match else None - - self.remote = test_env.get('PGPROBACKUP_SSH_REMOTE', None) == 'ON' - self.ptrack = test_env.get('PG_PROBACKUP_PTRACK', None) == 'ON' and self.pg_config_version >= 110000 - self.wal_tree_enabled = test_env.get('PG_PROBACKUP_WAL_TREE_ENABLED', None) == 'ON' - - self.bckp_source = test_env.get('PG_PROBACKUP_SOURCE', 'pro').lower() - if self.bckp_source not in ('base', 'direct', 'pro'): - raise Exception("Wrong PG_PROBACKUP_SOURCE value. Available options: base|direct|pro") - - self.paranoia = test_env.get('PG_PROBACKUP_PARANOIA', None) == 'ON' - env_compress = test_env.get('ARCHIVE_COMPRESSION', None) - if env_compress: - env_compress = env_compress.lower() - if env_compress in ('on', 'zlib'): - self.compress_suffix = '.gz' - self.archive_compress = 'zlib' - elif env_compress == 'lz4': - if not HAVE_LZ4: - raise LZ4_error - if 'lz4' not in self.probackup_compressions: - raise Exception("pg_probackup is not compiled with lz4 support") - self.compress_suffix = '.lz4' - self.archive_compress = 'lz4' - elif env_compress == 'zstd': - if not HAVE_ZSTD: - raise ZSTD_error - if 'zstd' not in self.probackup_compressions: - raise Exception("pg_probackup is not compiled with zstd support") - self.compress_suffix = '.zst' - self.archive_compress = 'zstd' - else: - self.compress_suffix = '' - self.archive_compress = False - - cfs_compress = test_env.get('PG_PROBACKUP_CFS_COMPRESS', None) - if cfs_compress: - self.cfs_compress = cfs_compress.lower() - else: - self.cfs_compress = self.archive_compress - - os.environ["PGAPPNAME"] = "pg_probackup" - self.delete_logs = delete_logs - - if self.probackup_version.split('.')[0].isdigit(): - self.major_version = int(self.probackup_version.split('.')[0]) - else: - raise Exception('Can\'t process pg_probackup version \"{}\": the major version is expected to be a number'.format(self.probackup_version)) - - def test_env(self): - return self._test_env.copy() - - -try: - init_params = Init() -except Exception as e: - logging.error(str(e)) - logging.warning("testgres.plugins.probackup2.init_params is set to None.") - init_params = None diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py b/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py deleted file mode 100644 index 6c9d1463..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/storage/fs_backup.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -Utilities for accessing pg_probackup backup data on file system. -""" -import os -import shutil - -from ..init_helpers import init_params - - -class TestBackupDir: - - def list_instance_backups(self, instance): - raise NotImplementedError() - - def list_files(self, sub_dir, recursive=False): - raise NotImplementedError() - - def list_dirs(self, sub_dir): - raise NotImplementedError() - - def read_file(self, sub_path, *, text=True): - raise NotImplementedError() - - def write_file(self, sub_path, data, *, text=True): - raise NotImplementedError() - - def cleanup(self): - raise NotImplementedError() - - def remove_file(self, sub_path): - raise NotImplementedError() - - def remove_dir(self, sub_path): - raise NotImplementedError() - - def exists(self, sub_path): - raise NotImplementedError() - - -class FSTestBackupDir(TestBackupDir): - is_file_based = True - - """ Backup directory. Usually created by running pg_probackup init -B """ - - def __init__(self, *, rel_path, backup): - backup_prefix = os.environ.get('PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX') - if backup_prefix and not os.path.isabs(backup_prefix): - raise Exception(f"PG_PROBACKUP_TEST_BACKUP_DIR_PREFIX must be an absolute path, current value: {backup_prefix}") - self.path = os.path.join(backup_prefix or init_params.tmp_path, rel_path, backup) - self.pb_args = ('-B', self.path) - - def list_instance_backups(self, instance): - full_path = os.path.join(self.path, 'backups', instance) - return sorted((x for x in os.listdir(full_path) - if os.path.isfile(os.path.join(full_path, x, 'backup.control')))) - - def list_files(self, sub_dir, recursive=False): - full_path = os.path.join(self.path, sub_dir) - if not recursive: - return [f for f in os.listdir(full_path) - if os.path.isfile(os.path.join(full_path, f))] - files = [] - for rootdir, dirs, files_in_dir in os.walk(full_path): - rootdir = rootdir[len(self.path) + 1:] - files.extend(os.path.join(rootdir, file) for file in files_in_dir) - return files - - def list_dirs(self, sub_dir): - full_path = os.path.join(self.path, sub_dir) - return [f for f in os.listdir(full_path) - if os.path.isdir(os.path.join(full_path, f))] - - def read_file(self, sub_path, *, text=True): - full_path = os.path.join(self.path, sub_path) - with open(full_path, 'r' if text else 'rb') as fin: - return fin.read() - - def write_file(self, sub_path, data, *, text=True): - full_path = os.path.join(self.path, sub_path) - with open(full_path, 'w' if text else 'wb') as fout: - fout.write(data) - - def cleanup(self): - shutil.rmtree(self.path, ignore_errors=True) - - def remove_file(self, sub_path): - os.remove(os.path.join(self.path, sub_path)) - - def remove_dir(self, sub_path): - full_path = os.path.join(self.path, sub_path) - shutil.rmtree(full_path, ignore_errors=True) - - def exists(self, sub_path): - full_path = os.path.join(self.path, sub_path) - return os.path.exists(full_path) - - def __str__(self): - return self.path - - def __repr__(self): - return "FSTestBackupDir" + str(self.path) - - def __fspath__(self): - return self.path diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py b/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py deleted file mode 100644 index ba788623..00000000 --- a/testgres/plugins/pg_probackup2/pg_probackup2/tests/test_basic.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -import os -import shutil -import pytest - -from ...... import testgres -from ...pg_probackup2.app import ProbackupApp -from ...pg_probackup2.init_helpers import Init, init_params -from ..storage.fs_backup import FSTestBackupDir - - -class ProbackupTest: - pg_node: testgres.PostgresNode - - @staticmethod - def probackup_is_available() -> bool: - p = os.environ.get("PGPROBACKUPBIN") - - if p is None: - return False - - if not os.path.exists(p): - return False - - return True - - @pytest.fixture(autouse=True, scope="function") - def implicit_fixture(self, request: pytest.FixtureRequest): - assert isinstance(request, pytest.FixtureRequest) - self.helper__setUp(request) - yield - self.helper__tearDown() - - def helper__setUp(self, request: pytest.FixtureRequest): - assert isinstance(request, pytest.FixtureRequest) - - self.helper__setup_test_environment(request) - self.helper__setup_test_paths() - self.helper__setup_backup_dir() - self.helper__setup_probackup() - - def helper__setup_test_environment(self, request: pytest.FixtureRequest): - assert isinstance(request, pytest.FixtureRequest) - - self.output = None - self.cmd = None - self.nodes_to_cleanup = [] - self.module_name, self.fname = request.node.cls.__name__, request.node.name - self.test_env = Init().test_env() - - def helper__setup_test_paths(self): - self.rel_path = os.path.join(self.module_name, self.fname) - self.test_path = os.path.join(init_params.tmp_path, self.rel_path) - os.makedirs(self.test_path, exist_ok=True) - self.pb_log_path = os.path.join(self.test_path, "pb_log") - - def helper__setup_backup_dir(self): - self.backup_dir = self.helper__build_backup_dir('backup') - self.backup_dir.cleanup() - - def helper__setup_probackup(self): - self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) - self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, - auto_compress_alg='zlib', backup_dir=self.backup_dir) - - def helper__tearDown(self): - if os.path.exists(self.test_path): - shutil.rmtree(self.test_path) - - def helper__build_backup_dir(self, backup='backup'): - return FSTestBackupDir(rel_path=self.rel_path, backup=backup) - - -@pytest.mark.skipif(not ProbackupTest.probackup_is_available(), reason="Check that PGPROBACKUPBIN is defined and is valid.") -class TestBasic(ProbackupTest): - def test_full_backup(self): - # Setting up a simple test node - node = self.pg_node.make_simple('node', pg_options={"fsync": "off", "synchronous_commit": "off"}) - - # Initialize and configure Probackup - self.pb.init() - self.pb.add_instance('node', node) - self.pb.set_archiving('node', node) - - # Start the node and initialize pgbench - node.slow_start() - node.pgbench_init(scale=100, no_vacuum=True) - - # Perform backup and validation - backup_id = self.pb.backup_node('node', node) - out = self.pb.validate('node', backup_id) - - # Check if the backup is valid - assert f"INFO: Backup {backup_id} is valid" in out diff --git a/testgres/plugins/pg_probackup2/setup.py b/testgres/plugins/pg_probackup2/setup.py deleted file mode 100644 index 7a3212e4..00000000 --- a/testgres/plugins/pg_probackup2/setup.py +++ /dev/null @@ -1,18 +0,0 @@ -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -setup( - version='0.1.0', - name='testgres_pg_probackup2', - packages=['pg_probackup2', 'pg_probackup2.storage'], - description='Plugin for testgres that manages pg_probackup2', - url='https://github.com/postgrespro/testgres', - long_description_content_type='text/markdown', - license='PostgreSQL', - author='Postgres Professional', - author_email='testgres@postgrespro.ru', - keywords=['pg_probackup', 'testing', 'testgres'], - install_requires=['testgres>=1.9.2'] -) diff --git a/tests/conftest.py b/tests/conftest.py index 6f2f9e41..a1a00757 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,26 +9,68 @@ import math import datetime import typing +import enum import _pytest.outcomes import _pytest.unittest import _pytest.logging +from packaging.version import Version + # ///////////////////////////////////////////////////////////////////////////// C_ROOT_DIR__RELATIVE = ".." # ///////////////////////////////////////////////////////////////////////////// -# TestConfigPropNames +T_TUPLE__str_int = typing.Tuple[str, int] -class TestConfigPropNames: - TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" +# ///////////////////////////////////////////////////////////////////////////// +# T_PLUGGY_RESULT +if Version(pluggy.__version__) <= Version("1.2"): + T_PLUGGY_RESULT = pluggy._result._Result +else: + T_PLUGGY_RESULT = pluggy.Result # ///////////////////////////////////////////////////////////////////////////// -T_TUPLE__str_int = typing.Tuple[str, int] +g_error_msg_count_key = pytest.StashKey[int]() +g_warning_msg_count_key = pytest.StashKey[int]() +g_critical_msg_count_key = pytest.StashKey[int]() + +# ///////////////////////////////////////////////////////////////////////////// +# T_TEST_PROCESS_KIND + + +class T_TEST_PROCESS_KIND(enum.Enum): + Master = 1 + Worker = 2 + + +# ///////////////////////////////////////////////////////////////////////////// +# T_TEST_PROCESS_MODE + + +class T_TEST_PROCESS_MODE(enum.Enum): + Collect = 1 + ExecTests = 2 + + +# ///////////////////////////////////////////////////////////////////////////// + +g_test_process_kind: typing.Optional[T_TEST_PROCESS_KIND] = None +g_test_process_mode: typing.Optional[T_TEST_PROCESS_MODE] = None + +g_worker_log_is_created: typing.Optional[bool] = None + +# ///////////////////////////////////////////////////////////////////////////// +# TestConfigPropNames + + +class TestConfigPropNames: + TEST_CFG__LOG_DIR = "TEST_CFG__LOG_DIR" + # ///////////////////////////////////////////////////////////////////////////// # TestStartupData__Helper @@ -50,13 +92,24 @@ def CalcRootDir() -> str: r = os.path.abspath(r) return r + # -------------------------------------------------------------------- + def CalcRootLogDir() -> str: + if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: + resultPath = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] + else: + rootDir = __class__.CalcRootDir() + resultPath = os.path.join(rootDir, "logs") + + assert type(resultPath) == str # noqa: E721 + return resultPath + # -------------------------------------------------------------------- def CalcCurrentTestWorkerSignature() -> str: currentPID = os.getpid() - assert type(currentPID) + assert type(currentPID) == int # noqa: E721 startTS = __class__.sm_StartTS - assert type(startTS) + assert type(startTS) == datetime.datetime # noqa: E721 result = "pytest-{0:04d}{1:02d}{2:02d}_{3:02d}{4:02d}{5:02d}".format( startTS.year, @@ -86,11 +139,18 @@ class TestStartupData: TestStartupData__Helper.CalcCurrentTestWorkerSignature() ) + sm_RootLogDir: str = TestStartupData__Helper.CalcRootLogDir() + # -------------------------------------------------------------------- def GetRootDir() -> str: assert type(__class__.sm_RootDir) == str # noqa: E721 return __class__.sm_RootDir + # -------------------------------------------------------------------- + def GetRootLogDir() -> str: + assert type(__class__.sm_RootLogDir) == str # noqa: E721 + return __class__.sm_RootLogDir + # -------------------------------------------------------------------- def GetCurrentTestWorkerSignature() -> str: assert type(__class__.sm_CurrentTestWorkerSignature) == str # noqa: E721 @@ -318,14 +378,9 @@ def helper__build_test_id(item: pytest.Function) -> str: # ///////////////////////////////////////////////////////////////////////////// -g_error_msg_count_key = pytest.StashKey[int]() -g_warning_msg_count_key = pytest.StashKey[int]() - -# ///////////////////////////////////////////////////////////////////////////// - def helper__makereport__setup( - item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result + item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT ): assert item is not None assert call is not None @@ -333,7 +388,7 @@ def helper__makereport__setup( # it may be pytest.Function or _pytest.unittest.TestCaseFunction assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - assert type(outcome) == pluggy.Result # noqa: E721 + assert type(outcome) == T_PLUGGY_RESULT # noqa: E721 C_LINE1 = "******************************************************" @@ -382,9 +437,19 @@ def helper__makereport__setup( return +# ------------------------------------------------------------------------ +class ExitStatusNames: + FAILED = "FAILED" + PASSED = "PASSED" + XFAILED = "XFAILED" + NOT_XFAILED = "NOT XFAILED" + SKIPPED = "SKIPPED" + UNEXPECTED = "UNEXPECTED" + + # ------------------------------------------------------------------------ def helper__makereport__call( - item: pytest.Function, call: pytest.CallInfo, outcome: pluggy.Result + item: pytest.Function, call: pytest.CallInfo, outcome: T_PLUGGY_RESULT ): assert item is not None assert call is not None @@ -392,13 +457,20 @@ def helper__makereport__call( # it may be pytest.Function or _pytest.unittest.TestCaseFunction assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - assert type(outcome) == pluggy.Result # noqa: E721 + assert type(outcome) == T_PLUGGY_RESULT # noqa: E721 # -------- - item_error_msg_count = item.stash.get(g_error_msg_count_key, 0) - assert type(item_error_msg_count) == int # noqa: E721 - assert item_error_msg_count >= 0 + item_error_msg_count1 = item.stash.get(g_error_msg_count_key, 0) + assert type(item_error_msg_count1) == int # noqa: E721 + assert item_error_msg_count1 >= 0 + + item_error_msg_count2 = item.stash.get(g_critical_msg_count_key, 0) + assert type(item_error_msg_count2) == int # noqa: E721 + assert item_error_msg_count2 >= 0 + + item_error_msg_count = item_error_msg_count1 + item_error_msg_count2 + # -------- item_warning_msg_count = item.stash.get(g_warning_msg_count_key, 0) assert type(item_warning_msg_count) == int # noqa: E721 assert item_warning_msg_count >= 0 @@ -424,6 +496,7 @@ def helper__makereport__call( # -------- exitStatus = None + exitStatusInfo = None if rep.outcome == "skipped": assert call.excinfo is not None # research assert call.excinfo.value is not None # research @@ -431,21 +504,21 @@ def helper__makereport__call( if type(call.excinfo.value) == _pytest.outcomes.Skipped: # noqa: E721 assert not hasattr(rep, "wasxfail") - exitStatus = "SKIPPED" + exitStatus = ExitStatusNames.SKIPPED reasonText = str(call.excinfo.value) reasonMsgTempl = "SKIP REASON: {0}" TEST_PROCESS_STATS.incrementSkippedTestCount() - elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 - exitStatus = "XFAILED" + elif type(call.excinfo.value) == _pytest.outcomes.XFailed: # noqa: E721 E501 + exitStatus = ExitStatusNames.XFAILED reasonText = str(call.excinfo.value) reasonMsgTempl = "XFAIL REASON: {0}" TEST_PROCESS_STATS.incrementXFailedTestCount(testID, item_error_msg_count) else: - exitStatus = "XFAILED" + exitStatus = ExitStatusNames.XFAILED assert hasattr(rep, "wasxfail") assert rep.wasxfail is not None assert type(rep.wasxfail) == str # noqa: E721 @@ -482,7 +555,7 @@ def helper__makereport__call( assert item_error_msg_count > 0 TEST_PROCESS_STATS.incrementFailedTestCount(testID, item_error_msg_count) - exitStatus = "FAILED" + exitStatus = ExitStatusNames.FAILED elif rep.outcome == "passed": assert call.excinfo is None @@ -497,15 +570,16 @@ def helper__makereport__call( warnMsg += " [" + rep.wasxfail + "]" logging.info(warnMsg) - exitStatus = "NOT XFAILED" + exitStatus = ExitStatusNames.NOT_XFAILED else: assert not hasattr(rep, "wasxfail") TEST_PROCESS_STATS.incrementPassedTestCount() - exitStatus = "PASSED" + exitStatus = ExitStatusNames.PASSED else: TEST_PROCESS_STATS.incrementUnexpectedTests() - exitStatus = "UNEXPECTED [{0}]".format(rep.outcome) + exitStatus = ExitStatusNames.UNEXPECTED + exitStatusInfo = rep.outcome # [2025-03-28] It may create a useless problem in new environment. # assert False @@ -513,6 +587,14 @@ def helper__makereport__call( if item_warning_msg_count > 0: TEST_PROCESS_STATS.incrementWarningTestCount(testID, item_warning_msg_count) + # -------- + assert exitStatus is not None + assert type(exitStatus) == str # noqa: E721 + + if exitStatus == ExitStatusNames.FAILED: + assert item_error_msg_count > 0 + pass + # -------- assert type(TEST_PROCESS_STATS.cTotalDuration) == datetime.timedelta # noqa: E721 assert type(testDurration) == datetime.timedelta # noqa: E721 @@ -521,11 +603,17 @@ def helper__makereport__call( assert testDurration <= TEST_PROCESS_STATS.cTotalDuration + # -------- + exitStatusLineData = exitStatus + + if exitStatusInfo is not None: + exitStatusLineData += " [{}]".format(exitStatusInfo) + # -------- logging.info("*") logging.info("* DURATION : {0}".format(timedelta_to_human_text(testDurration))) logging.info("*") - logging.info("* EXIT STATUS : {0}".format(exitStatus)) + logging.info("* EXIT STATUS : {0}".format(exitStatusLineData)) logging.info("* ERROR COUNT : {0}".format(item_error_msg_count)) logging.info("* WARNING COUNT: {0}".format(item_warning_msg_count)) logging.info("*") @@ -552,9 +640,9 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): assert isinstance(item, pytest.Function) assert type(call) == pytest.CallInfo # noqa: E721 - outcome: pluggy.Result = yield + outcome = yield assert outcome is not None - assert type(outcome) == pluggy.Result # noqa: E721 + assert type(outcome) == T_PLUGGY_RESULT # noqa: E721 assert type(call.when) == str # noqa: E721 @@ -582,103 +670,87 @@ def pytest_runtest_makereport(item: pytest.Function, call: pytest.CallInfo): # ///////////////////////////////////////////////////////////////////////////// -class LogErrorWrapper2: +class LogWrapper2: _old_method: any - _counter: typing.Optional[int] + _err_counter: typing.Optional[int] + _warn_counter: typing.Optional[int] + + _critical_counter: typing.Optional[int] # -------------------------------------------------------------------- def __init__(self): self._old_method = None - self._counter = None + self._err_counter = None + self._warn_counter = None + + self._critical_counter = None # -------------------------------------------------------------------- def __enter__(self): assert self._old_method is None - assert self._counter is None - - self._old_method = logging.error - self._counter = 0 - - logging.error = self - return self + assert self._err_counter is None + assert self._warn_counter is None - # -------------------------------------------------------------------- - def __exit__(self, exc_type, exc_val, exc_tb): - assert self._old_method is not None - assert self._counter is not None + assert self._critical_counter is None - assert logging.error is self + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) - logging.error = self._old_method - - self._old_method = None - self._counter = None - return False - - # -------------------------------------------------------------------- - def __call__(self, *args, **kwargs): - assert self._old_method is not None - assert self._counter is not None - - assert type(self._counter) == int # noqa: E721 - assert self._counter >= 0 - - r = self._old_method(*args, **kwargs) - - self._counter += 1 - assert self._counter > 0 - - return r + self._old_method = logging.root.handle + self._err_counter = 0 + self._warn_counter = 0 + self._critical_counter = 0 -# ///////////////////////////////////////////////////////////////////////////// - - -class LogWarningWrapper2: - _old_method: any - _counter: typing.Optional[int] - - # -------------------------------------------------------------------- - def __init__(self): - self._old_method = None - self._counter = None - - # -------------------------------------------------------------------- - def __enter__(self): - assert self._old_method is None - assert self._counter is None - - self._old_method = logging.warning - self._counter = 0 - - logging.warning = self + logging.root.handle = self return self # -------------------------------------------------------------------- def __exit__(self, exc_type, exc_val, exc_tb): assert self._old_method is not None - assert self._counter is not None + assert self._err_counter is not None + assert self._warn_counter is not None + + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) - assert logging.warning is self + assert logging.root.handle is self - logging.warning = self._old_method + logging.root.handle = self._old_method self._old_method = None - self._counter = None + self._err_counter = None + self._warn_counter = None + self._critical_counter = None return False # -------------------------------------------------------------------- - def __call__(self, *args, **kwargs): + def __call__(self, record: logging.LogRecord): + assert record is not None + assert isinstance(record, logging.LogRecord) assert self._old_method is not None - assert self._counter is not None - - assert type(self._counter) == int # noqa: E721 - assert self._counter >= 0 - - r = self._old_method(*args, **kwargs) - - self._counter += 1 - assert self._counter > 0 + assert self._err_counter is not None + assert self._warn_counter is not None + assert self._critical_counter is not None + + assert type(self._err_counter) == int # noqa: E721 + assert self._err_counter >= 0 + assert type(self._warn_counter) == int # noqa: E721 + assert self._warn_counter >= 0 + assert type(self._critical_counter) == int # noqa: E721 + assert self._critical_counter >= 0 + + r = self._old_method(record) + + if record.levelno == logging.ERROR: + self._err_counter += 1 + assert self._err_counter > 0 + elif record.levelno == logging.WARNING: + self._warn_counter += 1 + assert self._warn_counter > 0 + elif record.levelno == logging.CRITICAL: + self._critical_counter += 1 + assert self._critical_counter > 0 return r @@ -699,6 +771,13 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function): assert pyfuncitem is not None assert isinstance(pyfuncitem, pytest.Function) + assert logging.root is not None + assert isinstance(logging.root, logging.RootLogger) + assert logging.root.handle is not None + + debug__log_handle_method = logging.root.handle + assert debug__log_handle_method is not None + debug__log_error_method = logging.error assert debug__log_error_method is not None @@ -707,55 +786,56 @@ def pytest_pyfunc_call(pyfuncitem: pytest.Function): pyfuncitem.stash[g_error_msg_count_key] = 0 pyfuncitem.stash[g_warning_msg_count_key] = 0 + pyfuncitem.stash[g_critical_msg_count_key] = 0 try: - with LogErrorWrapper2() as logErrorWrapper, LogWarningWrapper2() as logWarningWrapper: - assert type(logErrorWrapper) == LogErrorWrapper2 # noqa: E721 - assert logErrorWrapper._old_method is not None - assert type(logErrorWrapper._counter) == int # noqa: E721 - assert logErrorWrapper._counter == 0 - assert logging.error is logErrorWrapper - - assert type(logWarningWrapper) == LogWarningWrapper2 # noqa: E721 - assert logWarningWrapper._old_method is not None - assert type(logWarningWrapper._counter) == int # noqa: E721 - assert logWarningWrapper._counter == 0 - assert logging.warning is logWarningWrapper - - r: pluggy.Result = yield + with LogWrapper2() as logWrapper: + assert type(logWrapper) == LogWrapper2 # noqa: E721 + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter == 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter == 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter == 0 + assert logging.root.handle is logWrapper + + r = yield assert r is not None - assert type(r) == pluggy.Result # noqa: E721 + assert type(r) == T_PLUGGY_RESULT # noqa: E721 - assert logErrorWrapper._old_method is not None - assert type(logErrorWrapper._counter) == int # noqa: E721 - assert logErrorWrapper._counter >= 0 - assert logging.error is logErrorWrapper - - assert logWarningWrapper._old_method is not None - assert type(logWarningWrapper._counter) == int # noqa: E721 - assert logWarningWrapper._counter >= 0 - assert logging.warning is logWarningWrapper + assert logWrapper._old_method is not None + assert type(logWrapper._err_counter) == int # noqa: E721 + assert logWrapper._err_counter >= 0 + assert type(logWrapper._warn_counter) == int # noqa: E721 + assert logWrapper._warn_counter >= 0 + assert type(logWrapper._critical_counter) == int # noqa: E721 + assert logWrapper._critical_counter >= 0 + assert logging.root.handle is logWrapper assert g_error_msg_count_key in pyfuncitem.stash assert g_warning_msg_count_key in pyfuncitem.stash + assert g_critical_msg_count_key in pyfuncitem.stash assert pyfuncitem.stash[g_error_msg_count_key] == 0 assert pyfuncitem.stash[g_warning_msg_count_key] == 0 + assert pyfuncitem.stash[g_critical_msg_count_key] == 0 - pyfuncitem.stash[g_error_msg_count_key] = logErrorWrapper._counter - pyfuncitem.stash[g_warning_msg_count_key] = logWarningWrapper._counter + pyfuncitem.stash[g_error_msg_count_key] = logWrapper._err_counter + pyfuncitem.stash[g_warning_msg_count_key] = logWrapper._warn_counter + pyfuncitem.stash[g_critical_msg_count_key] = logWrapper._critical_counter if r.exception is not None: pass - elif logErrorWrapper._counter == 0: - pass - else: - assert logErrorWrapper._counter > 0 + elif logWrapper._err_counter > 0: + r.force_exception(SIGNAL_EXCEPTION()) + elif logWrapper._critical_counter > 0: r.force_exception(SIGNAL_EXCEPTION()) finally: assert logging.error is debug__log_error_method assert logging.warning is debug__log_warning_method + assert logging.root.handle == debug__log_handle_method pass @@ -831,13 +911,37 @@ def helper__print_test_list2(tests: typing.List[T_TUPLE__str_int]) -> None: # ///////////////////////////////////////////////////////////////////////////// +# SUMMARY BUILDER + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(): + # + # NOTE: It should execute after logging.pytest_sessionfinish + # + + global g_test_process_kind # noqa: F824 + global g_test_process_mode # noqa: F824 + global g_worker_log_is_created # noqa: F824 + + assert g_test_process_kind is not None + assert type(g_test_process_kind) == T_TEST_PROCESS_KIND # noqa: E721 + + if g_test_process_kind == T_TEST_PROCESS_KIND.Master: + return + + assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker + + assert g_test_process_mode is not None + assert type(g_test_process_mode) == T_TEST_PROCESS_MODE # noqa: E721 + + if g_test_process_mode == T_TEST_PROCESS_MODE.Collect: + return -@pytest.fixture(autouse=True, scope="session") -def run_after_tests(request: pytest.FixtureRequest): - assert isinstance(request, pytest.FixtureRequest) + assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests - yield + assert type(g_worker_log_is_created) == bool # noqa: E721 + assert g_worker_log_is_created C_LINE1 = "---------------------------" @@ -847,7 +951,9 @@ def LOCAL__print_line1_with_header(header: str): assert header != "" logging.info(C_LINE1 + " [" + header + "]") - def LOCAL__print_test_list(header: str, test_count: int, test_list: typing.List[str]): + def LOCAL__print_test_list( + header: str, test_count: int, test_list: typing.List[str] + ): assert type(header) == str # noqa: E721 assert type(test_count) == int # noqa: E721 assert type(test_list) == list # noqa: E721 @@ -947,20 +1053,41 @@ def LOCAL__print_test_list2( # ///////////////////////////////////////////////////////////////////////////// +def helper__detect_test_process_kind(config: pytest.Config) -> T_TEST_PROCESS_KIND: + assert isinstance(config, pytest.Config) + + # + # xdist' master process registers DSession plugin. + # + p = config.pluginmanager.get_plugin("dsession") + + if p is not None: + return T_TEST_PROCESS_KIND.Master + + return T_TEST_PROCESS_KIND.Worker + + +# ------------------------------------------------------------------------ +def helper__detect_test_process_mode(config: pytest.Config) -> T_TEST_PROCESS_MODE: + assert isinstance(config, pytest.Config) + + if config.getvalue("collectonly"): + return T_TEST_PROCESS_MODE.Collect + + return T_TEST_PROCESS_MODE.ExecTests + + +# ------------------------------------------------------------------------ @pytest.hookimpl(trylast=True) -def pytest_configure(config: pytest.Config) -> None: +def helper__pytest_configure__logging(config: pytest.Config) -> None: assert isinstance(config, pytest.Config) log_name = TestStartupData.GetCurrentTestWorkerSignature() log_name += ".log" - if TestConfigPropNames.TEST_CFG__LOG_DIR in os.environ: - log_path_v = os.environ[TestConfigPropNames.TEST_CFG__LOG_DIR] - log_path = pathlib.Path(log_path_v) - else: - log_path = config.rootpath.joinpath("logs") + log_dir = TestStartupData.GetRootLogDir() - log_path.mkdir(exist_ok=True) + pathlib.Path(log_dir).mkdir(exist_ok=True) logging_plugin: _pytest.logging.LoggingPlugin = config.pluginmanager.get_plugin( "logging-plugin" @@ -969,7 +1096,46 @@ def pytest_configure(config: pytest.Config) -> None: assert logging_plugin is not None assert isinstance(logging_plugin, _pytest.logging.LoggingPlugin) - logging_plugin.set_log_path(str(log_path / log_name)) + log_file_path = os.path.join(log_dir, log_name) + assert log_file_path is not None + assert type(log_file_path) == str # noqa: E721 + + logging_plugin.set_log_path(log_file_path) + return + + +# ------------------------------------------------------------------------ +@pytest.hookimpl(trylast=True) +def pytest_configure(config: pytest.Config) -> None: + assert isinstance(config, pytest.Config) + + global g_test_process_kind + global g_test_process_mode + global g_worker_log_is_created + + assert g_test_process_kind is None + assert g_test_process_mode is None + assert g_worker_log_is_created is None + + g_test_process_mode = helper__detect_test_process_mode(config) + g_test_process_kind = helper__detect_test_process_kind(config) + + assert type(g_test_process_kind) == T_TEST_PROCESS_KIND # noqa: E721 + assert type(g_test_process_mode) == T_TEST_PROCESS_MODE # noqa: E721 + + if g_test_process_kind == T_TEST_PROCESS_KIND.Master: + pass + else: + assert g_test_process_kind == T_TEST_PROCESS_KIND.Worker + + if g_test_process_mode == T_TEST_PROCESS_MODE.Collect: + g_worker_log_is_created = False + else: + assert g_test_process_mode == T_TEST_PROCESS_MODE.ExecTests + helper__pytest_configure__logging(config) + g_worker_log_is_created = True + + return # ///////////////////////////////////////////////////////////////////////////// diff --git a/tests/helpers/global_data.py b/tests/helpers/global_data.py index c21d7dd8..5c3f7a46 100644 --- a/tests/helpers/global_data.py +++ b/tests/helpers/global_data.py @@ -1,11 +1,11 @@ -from ...testgres.operations.os_ops import OsOperations -from ...testgres.operations.os_ops import ConnectionParams -from ...testgres.operations.local_ops import LocalOperations -from ...testgres.operations.remote_ops import RemoteOperations +from testgres.operations.os_ops import OsOperations +from testgres.operations.os_ops import ConnectionParams +from testgres.operations.local_ops import LocalOperations +from testgres.operations.remote_ops import RemoteOperations -from ...testgres.node import PortManager -from ...testgres.node import PortManager__ThisHost -from ...testgres.node import PortManager__Generic +from src.node import PortManager +from src.node import PortManager__ThisHost +from src.node import PortManager__Generic import os @@ -31,7 +31,7 @@ class OsOpsDescrs: sm_remote_os_ops_descr = OsOpsDescr("remote_ops", sm_remote_os_ops) - sm_local_os_ops = LocalOperations() + sm_local_os_ops = LocalOperations.get_single_instance() sm_local_os_ops_descr = OsOpsDescr("local_ops", sm_local_os_ops) @@ -39,7 +39,7 @@ class OsOpsDescrs: class PortManagers: sm_remote_port_manager = PortManager__Generic(OsOpsDescrs.sm_remote_os_ops) - sm_local_port_manager = PortManager__ThisHost() + sm_local_port_manager = PortManager__ThisHost.get_single_instance() sm_local2_port_manager = PortManager__Generic(OsOpsDescrs.sm_local_os_ops) diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 00000000..407279b3 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,6 @@ +psutil +pytest +pytest-xdist +psycopg2 +six +testgres.os_ops>=0.0.2,<1.0.0 diff --git a/tests/test_config.py b/tests/test_config.py index 05702e9a..3969b2c2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,9 +1,9 @@ -from ..testgres import TestgresConfig -from ..testgres import configure_testgres -from ..testgres import scoped_config -from ..testgres import pop_config +from src import TestgresConfig +from src import configure_testgres +from src import scoped_config +from src import pop_config -from .. import testgres +import src as testgres import pytest diff --git a/tests/test_os_ops_common.py b/tests/test_os_ops_common.py index 17c3151c..bf2dce76 100644 --- a/tests/test_os_ops_common.py +++ b/tests/test_os_ops_common.py @@ -13,9 +13,13 @@ import socket import threading import typing +import uuid -from ..testgres import InvalidOperationException -from ..testgres import ExecUtilException +from src import InvalidOperationException +from src import ExecUtilException + +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future as ThreadFuture class TestOsOpsCommon: @@ -33,6 +37,13 @@ def os_ops(self, request: pytest.FixtureRequest) -> OsOperations: assert isinstance(request.param, OsOperations) return request.param + def test_create_clone(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + clone = os_ops.create_clone() + assert clone is not None + assert clone is not os_ops + assert type(clone) == type(os_ops) # noqa: E721 + def test_exec_command_success(self, os_ops: OsOperations): """ Test exec_command for successful command execution. @@ -114,6 +125,23 @@ def test_exec_command_with_exec_env(self, os_ops: OsOperations): assert type(response) == bytes # noqa: E721 assert response == b'\n' + def test_exec_command_with_cwd(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + RunConditions.skip_if_windows() + + cmd = ["pwd"] + + response = os_ops.exec_command(cmd, cwd="/tmp") + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response == b'/tmp\n' + + response = os_ops.exec_command(cmd) + assert response is not None + assert type(response) == bytes # noqa: E721 + assert response != b'/tmp\n' + def test_exec_command__test_unset(self, os_ops: OsOperations): assert isinstance(os_ops, OsOperations) @@ -812,3 +840,300 @@ def LOCAL_server(s: socket.socket): if ok_count == 0: raise RuntimeError("No one free port was found.") + + def test_get_tmpdir(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + dir = os_ops.get_tempdir() + assert type(dir) == str # noqa: E721 + assert os_ops.path_exists(dir) + assert os.path.exists(dir) + + file_path = os.path.join(dir, "testgres--" + uuid.uuid4().hex + ".tmp") + + os_ops.write(file_path, "1234", binary=False) + + assert os_ops.path_exists(file_path) + assert os.path.exists(file_path) + + d = os_ops.read(file_path, binary=False) + + assert d == "1234" + + os_ops.remove_file(file_path) + + assert not os_ops.path_exists(file_path) + assert not os.path.exists(file_path) + + def test_get_tmpdir__compare_with_py_info(self, os_ops: OsOperations): + assert isinstance(os_ops, OsOperations) + + actual_dir = os_ops.get_tempdir() + assert actual_dir is not None + assert type(actual_dir) == str # noqa: E721 + expected_dir = str(tempfile.tempdir) + assert actual_dir == expected_dir + + class tagData_OS_OPS__NUMS: + os_ops_descr: OsOpsDescr + nums: int + + def __init__(self, os_ops_descr: OsOpsDescr, nums: int): + assert isinstance(os_ops_descr, OsOpsDescr) + assert type(nums) == int # noqa: E721 + + self.os_ops_descr = os_ops_descr + self.nums = nums + + sm_test_exclusive_creation__mt__data = [ + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_local_os_ops_descr, 100000), + tagData_OS_OPS__NUMS(OsOpsDescrs.sm_remote_os_ops_descr, 120), + ] + + @pytest.fixture( + params=sm_test_exclusive_creation__mt__data, + ids=[x.os_ops_descr.sign for x in sm_test_exclusive_creation__mt__data] + ) + def data001(self, request: pytest.FixtureRequest) -> tagData_OS_OPS__NUMS: + assert isinstance(request, pytest.FixtureRequest) + return request.param + + def test_mkdir__mt(self, data001: tagData_OS_OPS__NUMS): + assert type(data001) == __class__.tagData_OS_OPS__NUMS # noqa: E721 + + N_WORKERS = 4 + N_NUMBERS = data001.nums + assert type(N_NUMBERS) == int # noqa: E721 + + os_ops = data001.os_ops_descr.os_ops + assert isinstance(os_ops, OsOperations) + + lock_dir_prefix = "test_mkdir_mt--" + uuid.uuid4().hex + + lock_dir = os_ops.mkdtemp(prefix=lock_dir_prefix) + + logging.info("A lock file [{}] is creating ...".format(lock_dir)) + + assert os.path.exists(lock_dir) + + def MAKE_PATH(lock_dir: str, num: int) -> str: + assert type(lock_dir) == str # noqa: E721 + assert type(num) == int # noqa: E721 + return os.path.join(lock_dir, str(num) + ".lock") + + def LOCAL_WORKER(os_ops: OsOperations, + workerID: int, + lock_dir: str, + cNumbers: int, + reservedNumbers: typing.Set[int]) -> None: + assert isinstance(os_ops, OsOperations) + assert type(workerID) == int # noqa: E721 + assert type(lock_dir) == str # noqa: E721 + assert type(cNumbers) == int # noqa: E721 + assert type(reservedNumbers) == set # noqa: E721 + assert cNumbers > 0 + assert len(reservedNumbers) == 0 + + assert os.path.exists(lock_dir) + + def LOG_INFO(template: str, *args: list) -> None: + assert type(template) == str # noqa: E721 + assert type(args) == tuple # noqa: E721 + + msg = template.format(*args) + assert type(msg) == str # noqa: E721 + + logging.info("[Worker #{}] {}".format(workerID, msg)) + return + + LOG_INFO("HELLO! I am here!") + + for num in range(cNumbers): + assert not (num in reservedNumbers) + + file_path = MAKE_PATH(lock_dir, num) + + try: + os_ops.makedir(file_path) + except Exception as e: + LOG_INFO( + "Can't reserve {}. Error ({}): {}", + num, + type(e).__name__, + str(e) + ) + continue + + LOG_INFO("Number {} is reserved!", num) + assert os_ops.path_exists(file_path) + reservedNumbers.add(num) + continue + + n_total = cNumbers + n_ok = len(reservedNumbers) + assert n_ok <= n_total + + LOG_INFO("Finish! OK: {}. FAILED: {}.", n_ok, n_total - n_ok) + return + + # ----------------------- + logging.info("Worker are creating ...") + + threadPool = ThreadPoolExecutor( + max_workers=N_WORKERS, + thread_name_prefix="ex_creator" + ) + + class tadWorkerData: + future: ThreadFuture + reservedNumbers: typing.Set[int] + + workerDatas: typing.List[tadWorkerData] = list() + + nErrors = 0 + + try: + for n in range(N_WORKERS): + logging.info("worker #{} is creating ...".format(n)) + + workerDatas.append(tadWorkerData()) + + workerDatas[n].reservedNumbers = set() + + workerDatas[n].future = threadPool.submit( + LOCAL_WORKER, + os_ops, + n, + lock_dir, + N_NUMBERS, + workerDatas[n].reservedNumbers + ) + + assert workerDatas[n].future is not None + + logging.info("OK. All the workers were created!") + except Exception as e: + nErrors += 1 + logging.error("A problem is detected ({}): {}".format(type(e).__name__, str(e))) + + logging.info("Will wait for stop of all the workers...") + + nWorkers = 0 + + assert type(workerDatas) == list # noqa: E721 + + for i in range(len(workerDatas)): + worker = workerDatas[i].future + + if worker is None: + continue + + nWorkers += 1 + + assert isinstance(worker, ThreadFuture) + + try: + logging.info("Wait for worker #{}".format(i)) + worker.result() + except Exception as e: + nErrors += 1 + logging.error("Worker #{} finished with error ({}): {}".format( + i, + type(e).__name__, + str(e), + )) + continue + + assert nWorkers == N_WORKERS + + if nErrors != 0: + raise RuntimeError("Some problems were detected. Please examine the log messages.") + + logging.info("OK. Let's check worker results!") + + reservedNumbers: typing.Dict[int, int] = dict() + + for i in range(N_WORKERS): + logging.info("Worker #{} is checked ...".format(i)) + + workerNumbers = workerDatas[i].reservedNumbers + assert type(workerNumbers) == set # noqa: E721 + + for n in workerNumbers: + if n < 0 or n >= N_NUMBERS: + nErrors += 1 + logging.error("Unexpected number {}".format(n)) + continue + + if n in reservedNumbers.keys(): + nErrors += 1 + logging.error("Number {} was already reserved by worker #{}".format( + n, + reservedNumbers[n] + )) + else: + reservedNumbers[n] = i + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + continue + + logging.info("OK. Let's check reservedNumbers!") + + for n in range(N_NUMBERS): + if not (n in reservedNumbers.keys()): + nErrors += 1 + logging.error("Number {} is not reserved!".format(n)) + continue + + file_path = MAKE_PATH(lock_dir, n) + if not os_ops.path_exists(file_path): + nErrors += 1 + logging.error("File {} is not found!".format(file_path)) + continue + + # OK! + continue + + logging.info("Verification is finished! Total error count is {}.".format(nErrors)) + + if nErrors == 0: + logging.info("Root lock-directory [{}] will be deleted.".format( + lock_dir + )) + + for n in range(N_NUMBERS): + file_path = MAKE_PATH(lock_dir, n) + try: + os_ops.rmdir(file_path) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + file_path, + type(e).__name__, + str(e) + )) + continue + + if os_ops.path_exists(file_path): + nErrors += 1 + logging.error("Directory {} is not deleted!".format(file_path)) + continue + + if nErrors == 0: + try: + os_ops.rmdir(lock_dir) + except Exception as e: + nErrors += 1 + logging.error("Cannot delete directory [{}]. Error ({}): {}".format( + lock_dir, + type(e).__name__, + str(e) + )) + + logging.info("Test is finished! Total error count is {}.".format(nErrors)) + return diff --git a/tests/test_os_ops_remote.py b/tests/test_os_ops_remote.py index 338e49f3..8be98da5 100755 --- a/tests/test_os_ops_remote.py +++ b/tests/test_os_ops_remote.py @@ -3,7 +3,7 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres import ExecUtilException +from src import ExecUtilException import os import pytest diff --git a/tests/test_testgres_common.py b/tests/test_testgres_common.py index e1252de2..e308a95e 100644 --- a/tests/test_testgres_common.py +++ b/tests/test_testgres_common.py @@ -1,31 +1,36 @@ +from __future__ import annotations + from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices from .helpers.global_data import OsOperations from .helpers.global_data import PortManager -from ..testgres.node import PgVer -from ..testgres.node import PostgresNode -from ..testgres.utils import get_pg_version2 -from ..testgres.utils import file_tail -from ..testgres.utils import get_bin_path2 -from ..testgres import ProcessType -from ..testgres import NodeStatus -from ..testgres import IsolationLevel +from src.node import PgVer +from src.node import PostgresNode +from src.node import PostgresNodeLogReader +from src.node import PostgresNodeUtils +from src.utils import get_pg_version2 +from src.utils import file_tail +from src.utils import get_bin_path2 +from src import ProcessType +from src import NodeStatus +from src import IsolationLevel +from src import NodeApp # New name prevents to collect test-functions in TestgresException and fixes # the problem with pytest warning. -from ..testgres import TestgresException as testgres_TestgresException - -from ..testgres import InitNodeException -from ..testgres import StartNodeException -from ..testgres import QueryException -from ..testgres import ExecUtilException -from ..testgres import TimeoutException -from ..testgres import InvalidOperationException -from ..testgres import BackupException -from ..testgres import ProgrammingError -from ..testgres import scoped_config -from ..testgres import First, Any +from src import TestgresException as testgres_TestgresException + +from src import InitNodeException +from src import StartNodeException +from src import QueryException +from src import ExecUtilException +from src import TimeoutException +from src import InvalidOperationException +from src import BackupException +from src import ProgrammingError +from src import scoped_config +from src import First, Any from contextlib import contextmanager @@ -620,13 +625,12 @@ def LOCAL__test_lines(): assert (master._logger.is_alive()) finally: # It is a hack code to logging cleanup - logging._acquireLock() - assert logging.Logger.manager is not None - assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() - logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) - assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) - assert not (handler in logging._handlers.values()) - logging._releaseLock() + with logging._lock: + assert logging.Logger.manager is not None + assert C_NODE_NAME in logging.Logger.manager.loggerDict.keys() + logging.Logger.manager.loggerDict.pop(C_NODE_NAME, None) + assert not (C_NODE_NAME in logging.Logger.manager.loggerDict.keys()) + assert not (handler in logging._handlers.values()) # GO HOME! return @@ -678,6 +682,89 @@ def test_psql(self, node_svc: PostgresNodeService): r = node.safe_psql('select 1') # raises! logging.error("node.safe_psql returns [{}]".format(r)) + def test_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == (0, b'1\n', b'')) + + def test_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + res = node.psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + res2 = __class__.helper__rm_carriage_returns(res) + + assert res2[0] != 0 + assert b"DUMMY_HOST_NAME" in res[2] + + def test_safe_psql__another_port(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node1: + with __class__.helper__get_node(node_svc).init() as node2: + node1.start() + node2.start() + assert node1.port != node2.port + assert node1.host == node2.host + + node1.stop() + + logging.info("test table in node2 is creating ...") + node2.safe_psql( + dbname="postgres", + query="create table test (id integer);" + ) + + logging.info("try to find test table through node1.psql ...") + res = node1.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host=node2.host, + port=node2.port, + ) + assert (__class__.helper__rm_carriage_returns(res) == b'1\n') + + def test_safe_psql__another_bad_host(self, node_svc: PostgresNodeService): + assert isinstance(node_svc, PostgresNodeService) + with __class__.helper__get_node(node_svc).init() as node: + logging.info("try to execute node1.psql ...") + + with pytest.raises(expected_exception=Exception) as x: + node.safe_psql( + dbname="postgres", + query="select count(*) from pg_class where relname='test'", + host="DUMMY_HOST_NAME", + port=node.port, + ) + + assert "DUMMY_HOST_NAME" in str(x.value) + def test_safe_psql__expect_error(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) with __class__.helper__get_node(node_svc).init().start() as node: @@ -800,15 +887,55 @@ def test_backup_wrong_xlog_method(self, node_svc: PostgresNodeService): def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) - C_MAX_ATTEMPTS = 50 - node = __class__.helper__get_node(node_svc) + C_MAX_ATTEMPT = 5 + + nAttempt = 0 + + while True: + if nAttempt == C_MAX_ATTEMPT: + raise Exception("PostgresSQL did not start.") + + nAttempt += 1 + logging.info("------------------------ NODE #{}".format( + nAttempt + )) + + with __class__.helper__get_node(node_svc, port=12345) as node: + if self.impl__test_pg_ctl_wait_option(node_svc, node): + break + continue + + logging.info("OK. Test is passed. Number of attempts is {}".format( + nAttempt + )) + return + + def impl__test_pg_ctl_wait_option( + self, + node_svc: PostgresNodeService, + node: PostgresNode + ) -> None: + assert isinstance(node_svc, PostgresNodeService) + assert isinstance(node, PostgresNode) assert node.status() == NodeStatus.Uninitialized + + C_MAX_ATTEMPTS = 50 + node.init() assert node.status() == NodeStatus.Stopped + + node_log_reader = PostgresNodeLogReader(node, from_beginnig=True) + node.start(wait=False) nAttempt = 0 while True: + if PostgresNodeUtils.delect_port_conflict(node_log_reader): + logging.info("Node port {} conflicted with another PostgreSQL instance.".format( + node.port + )) + return False + if nAttempt == C_MAX_ATTEMPTS: # # [2025-03-11] @@ -867,7 +994,7 @@ def test_pg_ctl_wait_option(self, node_svc: PostgresNodeService): raise Exception("Unexpected node status: {0}.".format(s1)) logging.info("OK. Node is stopped.") - node.cleanup() + return True def test_replicate(self, node_svc: PostgresNodeService): assert isinstance(node_svc, PostgresNodeService) @@ -1378,6 +1505,333 @@ def test_try_to_start_node_after_free_manual_port(self, node_svc: PostgresNodeSe ): node2.start() + def test_node__os_ops(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.os_ops is not None + assert isinstance(node_svc.os_ops, OsOperations) + + with PostgresNode(name="node", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node: + # retest + assert node_svc.os_ops is not None + assert isinstance(node_svc.os_ops, OsOperations) + + assert node.os_ops is node_svc.os_ops + # one more time + assert node.os_ops is node_svc.os_ops + + def test_node__port_manager(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + with PostgresNode(name="node", os_ops=node_svc.os_ops, port_manager=node_svc.port_manager) as node: + # retest + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is node_svc.port_manager + # one more time + assert node.port_manager is node_svc.port_manager + + def test_node__port_manager_and_explicit_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + port = node_svc.port_manager.reserve_port() + assert type(port) == int # noqa: E721 + + try: + with PostgresNode(name="node", port=port, os_ops=node_svc.os_ops) as node: + # retest + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + + # one more time + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + finally: + node_svc.port_manager.release_port(port) + + def test_node__no_port_manager(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + port = node_svc.port_manager.reserve_port() + assert type(port) == int # noqa: E721 + + try: + with PostgresNode(name="node", port=port, os_ops=node_svc.os_ops, port_manager=None) as node: + # retest + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + + # one more time + assert node.port_manager is None + assert node.os_ops is node_svc.os_ops + finally: + node_svc.port_manager.release_port(port) + + class tag_rmdirs_protector: + _os_ops: OsOperations + _cwd: str + _old_rmdirs: any + _cwd: str + + def __init__(self, os_ops: OsOperations): + self._os_ops = os_ops + self._cwd = os.path.abspath(os_ops.cwd()) + self._old_rmdirs = os_ops.rmdirs + + def __enter__(self): + assert self._os_ops.rmdirs == self._old_rmdirs + self._os_ops.rmdirs = self.proxy__rmdirs + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._os_ops.rmdirs == self.proxy__rmdirs + self._os_ops.rmdirs = self._old_rmdirs + return False + + def proxy__rmdirs(self, path, ignore_errors=True): + raise Exception("Call of rmdirs is not expected!") + + def test_node_app__make_empty__base_dir_is_None(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops + + with pytest.raises(expected_exception=BaseException) as x: + node_app.make_empty(base_dir=None) + + if type(x.value) == AssertionError: # noqa: E721 + pass + else: + assert type(x.value) == ValueError # noqa: E721 + assert str(x.value) == "Argument 'base_dir' is not defined." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty__base_dir_is_Empty(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + os_ops = node_svc.os_ops.create_clone() + assert os_ops is not node_svc.os_ops + + # ----------- + with __class__.tag_rmdirs_protector(os_ops): + node_app = NodeApp(test_path=tmp_dir, os_ops=os_ops) + assert node_app.os_ops is os_ops + + with pytest.raises(expected_exception=ValueError) as x: + node_app.make_empty(base_dir="") + + assert str(x.value) == "Argument 'base_dir' is empty." + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 0 + + node: PostgresNode = None + try: + node = node_app.make_simple("node") + assert node is not None + assert isinstance(node, PostgresNode) + assert node.os_ops is node_svc.os_ops + assert node.port_manager is node_svc.port_manager + + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + + node.slow_start() + finally: + if node is not None: + node.stop() + node.release_resources() + + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_simple__checksum(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + + logging.info("temp directory is [{}]".format(tmp_dir)) + node_app = NodeApp(test_path=tmp_dir, os_ops=node_svc.os_ops) + + C_NODE = "node" + + # ----------- + def LOCAL__test(checksum: bool, initdb_params: typing.Optional[list]): + initdb_params0 = initdb_params + initdb_params0_copy = initdb_params0.copy() if initdb_params0 is not None else None + + with node_app.make_simple(C_NODE, checksum=checksum, initdb_params=initdb_params): + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + assert initdb_params is initdb_params0 + if initdb_params0 is not None: + assert initdb_params0 == initdb_params0_copy + + # ----------- + LOCAL__test(checksum=False, initdb_params=None) + LOCAL__test(checksum=True, initdb_params=None) + + # ----------- + params = [] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + params = ["--no-sync"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + params = ["--data-checksums"] + LOCAL__test(checksum=False, initdb_params=params) + LOCAL__test(checksum=True, initdb_params=params) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + + def test_node_app__make_empty_with_explicit_port(self, node_svc: PostgresNodeService): + assert type(node_svc) == PostgresNodeService # noqa: E721 + + assert isinstance(node_svc.os_ops, OsOperations) + assert node_svc.port_manager is not None + assert isinstance(node_svc.port_manager, PortManager) + + tmp_dir = node_svc.os_ops.mkdtemp() + assert tmp_dir is not None + assert type(tmp_dir) == str # noqa: E721 + logging.info("temp directory is [{}]".format(tmp_dir)) + + # ----------- + node_app = NodeApp( + test_path=tmp_dir, + os_ops=node_svc.os_ops, + port_manager=node_svc.port_manager + ) + + assert node_app.os_ops is node_svc.os_ops + assert node_app.port_manager is node_svc.port_manager + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 0 + + port = node_app.port_manager.reserve_port() + assert type(port) == int # noqa: E721 + + node: PostgresNode = None + try: + node = node_app.make_simple("node", port=port) + assert node is not None + assert isinstance(node, PostgresNode) + assert node.os_ops is node_svc.os_ops + assert node.port_manager is None # <--------- + assert node.port == port + assert node._should_free_port == False # noqa: E712 + + assert type(node_app.nodes_to_cleanup) == list # noqa: E721 + assert len(node_app.nodes_to_cleanup) == 1 + assert node_app.nodes_to_cleanup[0] is node + + node.slow_start() + finally: + if node is not None: + node.stop() + node.free_port() + + assert node._port is None + assert not node._should_free_port + + node.cleanup(release_resources=True) + + # ----------- + logging.info("temp directory [{}] is deleting".format(tmp_dir)) + node_svc.os_ops.rmdir(tmp_dir) + @staticmethod def helper__get_node( node_svc: PostgresNodeService, @@ -1395,7 +1849,6 @@ def helper__get_node( return PostgresNode( name, port=port, - conn_params=None, os_ops=node_svc.os_ops, port_manager=port_manager if port is None else None ) diff --git a/tests/test_testgres_local.py b/tests/test_testgres_local.py index 9dbd455b..6018188e 100644 --- a/tests/test_testgres_local.py +++ b/tests/test_testgres_local.py @@ -7,21 +7,21 @@ import platform import logging -from .. import testgres +import src as testgres -from ..testgres import StartNodeException -from ..testgres import ExecUtilException -from ..testgres import NodeApp -from ..testgres import scoped_config -from ..testgres import get_new_node -from ..testgres import get_bin_path -from ..testgres import get_pg_config -from ..testgres import get_pg_version +from src import StartNodeException +from src import ExecUtilException +from src import NodeApp +from src import scoped_config +from src import get_new_node +from src import get_bin_path +from src import get_pg_config +from src import get_pg_version # NOTE: those are ugly imports -from ..testgres.utils import bound_ports -from ..testgres.utils import PgVer -from ..testgres.node import ProcessProxy +from src.utils import bound_ports +from src.utils import PgVer +from src.node import ProcessProxy def pg_version_ge(version): @@ -158,15 +158,15 @@ def test_child_process_dies(self): def test_upgrade_node(self): old_bin_dir = os.path.dirname(get_bin_path("pg_config")) new_bin_dir = os.path.dirname(get_bin_path("pg_config")) - node_old = get_new_node(prefix='node_old', bin_dir=old_bin_dir) - node_old.init() - node_old.start() - node_old.stop() - node_new = get_new_node(prefix='node_new', bin_dir=new_bin_dir) - node_new.init(cached=False) - res = node_new.upgrade_from(old_node=node_old) - node_new.start() - assert (b'Upgrade Complete' in res) + with get_new_node(prefix='node_old', bin_dir=old_bin_dir) as node_old: + node_old.init() + node_old.start() + node_old.stop() + with get_new_node(prefix='node_new', bin_dir=new_bin_dir) as node_new: + node_new.init(cached=False) + res = node_new.upgrade_from(old_node=node_old) + node_new.start() + assert (b'Upgrade Complete' in res) class tagPortManagerProxy: sm_prev_testgres_reserve_port = None @@ -341,10 +341,10 @@ def test_simple_with_bin_dir(self): bin_dir = node.bin_dir app = NodeApp() - correct_bin_dir = app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) - correct_bin_dir.slow_start() - correct_bin_dir.safe_psql("SELECT 1;") - correct_bin_dir.stop() + with app.make_simple(base_dir=node.base_dir, bin_dir=bin_dir) as correct_bin_dir: + correct_bin_dir.slow_start() + correct_bin_dir.safe_psql("SELECT 1;") + correct_bin_dir.stop() while True: try: diff --git a/tests/test_testgres_remote.py b/tests/test_testgres_remote.py index e38099b7..fc533559 100755 --- a/tests/test_testgres_remote.py +++ b/tests/test_testgres_remote.py @@ -7,16 +7,16 @@ from .helpers.global_data import PostgresNodeService from .helpers.global_data import PostgresNodeServices -from .. import testgres +import src as testgres -from ..testgres.exceptions import InitNodeException -from ..testgres.exceptions import ExecUtilException +from src.exceptions import InitNodeException +from src.exceptions import ExecUtilException -from ..testgres.config import scoped_config -from ..testgres.config import testgres_config +from src.config import scoped_config +from src.config import testgres_config -from ..testgres import get_bin_path -from ..testgres import get_pg_config +from src import get_bin_path +from src import get_pg_config # NOTE: those are ugly imports @@ -173,7 +173,6 @@ def helper__get_node(name=None): return testgres.PostgresNode( name, - conn_params=None, os_ops=svc.os_ops, port_manager=svc.port_manager) diff --git a/tests/test_utils.py b/tests/test_utils.py index c05bd2fe..9bb233b2 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,9 +2,9 @@ from .helpers.global_data import OsOpsDescrs from .helpers.global_data import OsOperations -from ..testgres.utils import parse_pg_version -from ..testgres.utils import get_pg_config2 -from ..testgres import scoped_config +from src.utils import parse_pg_version +from src.utils import get_pg_config2 +from src import scoped_config import pytest import typing