From 63adcf1b514df7df6994c5fb6abe16ce0a23c18e Mon Sep 17 00:00:00 2001 From: Philipp Herzog Date: Fri, 18 Oct 2024 13:14:19 +0200 Subject: [PATCH 1/5] run the test suite as part of the nix build --- flake.nix | 5 +- nix/batou.nix | 65 +++++++++++++------ .../secrets/tests/fixture/gnupg/S.gpg-agent | 2 + .../tests/fixture/gnupg/S.gpg-agent.ssh | 2 + .../tests/fixture/gnupg/gpg-agent.conf | 2 + 5 files changed, 54 insertions(+), 22 deletions(-) create mode 100644 src/batou/secrets/tests/fixture/gnupg/S.gpg-agent create mode 100644 src/batou/secrets/tests/fixture/gnupg/S.gpg-agent.ssh create mode 100644 src/batou/secrets/tests/fixture/gnupg/gpg-agent.conf diff --git a/flake.nix b/flake.nix index 0e0cf4bc8..50e8dbceb 100644 --- a/flake.nix +++ b/flake.nix @@ -56,7 +56,10 @@ devShells.default = pkgs.mkShell { packages = [ - (pkgs.python3.withPackages (ps: [batou ps.tox])) + (pkgs.python3.withPackages (ps: [ps.tox ps.setuptools ps.pytest])) + pkgs.mercurial + pkgs.age + pkgs.subversion ]; shellHook = '' diff --git a/nix/batou.nix b/nix/batou.nix index 09fb7568a..91023c7fd 100644 --- a/nix/batou.nix +++ b/nix/batou.nix @@ -1,7 +1,13 @@ { + gnupg, + rsync, + unzip, + git, + subversion, + python3, + age, + mercurial, buildPythonPackage, - fetchPypi, - markupsafe, requests, pyyaml, execnet, @@ -16,23 +22,40 @@ jinja2, src, }: - buildPythonPackage { - propagatedBuildInputs = [ - requests - pyyaml - execnet - importlib-metadata - importlib-resources - remote-pdb - py - configupdater - mock - pytest - setuptools - jinja2 - ]; +buildPythonPackage { + propagatedBuildInputs = [ + requests + pyyaml + execnet + importlib-metadata + importlib-resources + remote-pdb + py + configupdater + mock + pytest + setuptools + jinja2 + ]; - pname = "batou"; - version = "latest"; - inherit src; - } + checkPhase = '' + tox -e py + cp report.xml $out + cp -r htmlcov $out + ''; + + nativeCheckInputs = [ + (python3.withPackages (ps: [ps.tox])) + mercurial + age + git + subversion + unzip + rsync + gnupg + ]; + + pname = "batou"; + version = "latest"; + inherit src; +} diff --git a/src/batou/secrets/tests/fixture/gnupg/S.gpg-agent b/src/batou/secrets/tests/fixture/gnupg/S.gpg-agent new file mode 100644 index 000000000..63837678b --- /dev/null +++ b/src/batou/secrets/tests/fixture/gnupg/S.gpg-agent @@ -0,0 +1,2 @@ +%Assuan% +socket=/tmp/S.gpg-agent-batou diff --git a/src/batou/secrets/tests/fixture/gnupg/S.gpg-agent.ssh b/src/batou/secrets/tests/fixture/gnupg/S.gpg-agent.ssh new file mode 100644 index 000000000..6e28ed81f --- /dev/null +++ b/src/batou/secrets/tests/fixture/gnupg/S.gpg-agent.ssh @@ -0,0 +1,2 @@ +%Assuan% +socket=/tmp/S.gpg-agent-batou.ssh diff --git a/src/batou/secrets/tests/fixture/gnupg/gpg-agent.conf b/src/batou/secrets/tests/fixture/gnupg/gpg-agent.conf new file mode 100644 index 000000000..5c1feb980 --- /dev/null +++ b/src/batou/secrets/tests/fixture/gnupg/gpg-agent.conf @@ -0,0 +1,2 @@ +extra-socket /tmp/S.gpg-agent-batou.extra +browser-socket /tmp/S.gpg-agent-batou.browser From 11198f556276e41ee77b3fb10f26e11249d5b041 Mon Sep 17 00:00:00 2001 From: Philipp Herzog Date: Tue, 22 Oct 2024 08:34:54 +0200 Subject: [PATCH 2/5] wip: use pytest over tox for nix build checks --- MANIFEST.in | 2 ++ flake.nix | 20 +++++++++++++++++- nix/batou.nix | 57 +++++++++++++++++++++++++++++++-------------------- 3 files changed, 56 insertions(+), 23 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index d7802a952..813b9c472 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,7 @@ include src/batou/insecure-private.key recursive-include src/batou/init-template * recursive-include src/batou/tests/fixture * recursive-include src/batou/lib/resources * +recursive-include src/batou/lib/tests * +recursive-include src/batou/secrets/tests/fixture * global-include *.txt global-include *.in diff --git a/flake.nix b/flake.nix index 50e8dbceb..92c5b13bb 100644 --- a/flake.nix +++ b/flake.nix @@ -56,7 +56,25 @@ devShells.default = pkgs.mkShell { packages = [ - (pkgs.python3.withPackages (ps: [ps.tox ps.setuptools ps.pytest])) + #(pkgs.python3.withPackages (ps: [ps.tox ps.setuptools ps.pytest])) + (pkgs.python3.withPackages (ps: [ + ps.mock + ps.pytest + ps.pytest-cov + ps.pytest-instafail + ps.pytest-timeout + + ps.requests + ps.pyyaml + ps.execnet + ps.importlib-metadata + ps.importlib-resources + ps.remote-pdb + ps.py + ps.configupdater + ps.setuptools + ps.jinja2 + ])) pkgs.mercurial pkgs.age pkgs.subversion diff --git a/nix/batou.nix b/nix/batou.nix index 91023c7fd..73608ccff 100644 --- a/nix/batou.nix +++ b/nix/batou.nix @@ -4,7 +4,6 @@ unzip, git, subversion, - python3, age, mercurial, buildPythonPackage, @@ -16,43 +15,57 @@ remote-pdb, py, configupdater, - mock, - pytest, setuptools, jinja2, src, + pytestCheckHook, + mock, + pytest-cov, + pytest-instafail, + pytest-timeout, }: buildPythonPackage { - propagatedBuildInputs = [ - requests - pyyaml + build-system = [setuptools]; + dependencies = [ + configupdater execnet importlib-metadata importlib-resources - remote-pdb - py - configupdater - mock - pytest - setuptools jinja2 + py + pyyaml + remote-pdb + requests ]; - checkPhase = '' - tox -e py - cp report.xml $out - cp -r htmlcov $out - ''; - nativeCheckInputs = [ - (python3.withPackages (ps: [ps.tox])) - mercurial + pytestCheckHook + + py + mock + pytest-cov + pytest-instafail + pytest-timeout + requests + age git + gnupg + mercurial + rsync subversion unzip - rsync - gnupg + ]; + + PY_IGNORE_IMPORTMISMATCH = 1; + + disabledTests = [ + "test_runs_buildout_successfully" + "test_runs_buildout3_successfully" + ]; + + disabledTestPaths = [ + "src/batou/lib/tests/test_supervisor.py" ]; pname = "batou"; From 62a7b7854d8fb9c0ea742bf4c4a3aa4486144e2f Mon Sep 17 00:00:00 2001 From: Philipp Herzog Date: Tue, 22 Oct 2024 09:01:19 +0200 Subject: [PATCH 3/5] wip: disable a few more tests that dont work in the nix sandbox --- .github/workflows/nix.yml | 2 +- nix/batou.nix | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index c360292fb..680fcb457 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -16,5 +16,5 @@ jobs: steps: - uses: actions/checkout@v3 - uses: cachix/install-nix-action@v20 - - run: nix build + - run: nix build -L - run: nix flake check diff --git a/nix/batou.nix b/nix/batou.nix index 73608ccff..d7e53cf7a 100644 --- a/nix/batou.nix +++ b/nix/batou.nix @@ -41,8 +41,8 @@ buildPythonPackage { nativeCheckInputs = [ pytestCheckHook - py mock + py pytest-cov pytest-instafail pytest-timeout @@ -60,12 +60,27 @@ buildPythonPackage { PY_IGNORE_IMPORTMISMATCH = 1; disabledTests = [ - "test_runs_buildout_successfully" - "test_runs_buildout3_successfully" + # requires internet access + "test_manage__2_age" + "test_manage__reencrypt__1" + + # requires access to /usr/bin/hdiutil + "test_dmg_extracts_archive_to_target_directory" + + # cannot resolve hosts in sandbox + "test_address_netloc_attributes" + "test_address_resolves_listen_address" ]; disabledTestPaths = [ + # requires internet access for download / pip install + "src/batou/lib/tests/test_appenv.py" + "src/batou/lib/tests/test_buildout.py" + "src/batou/lib/tests/test_download.py" "src/batou/lib/tests/test_supervisor.py" + + # bad interpreter `/usr/bin/env python3` in appenv.py + "src/batou/tests/test_endtoend.py" ]; pname = "batou"; From 20ea39b99f3280ffc2185a72e9f475dbe99659e0 Mon Sep 17 00:00:00 2001 From: Philipp Herzog Date: Tue, 12 Nov 2024 12:49:32 +0100 Subject: [PATCH 4/5] wip: fix pylib import order Due to the way the PYTHONPATH is constructed in the buildPythonPackage builder, pylib was shadowed by Pytest's vendored pylib which however did not include all of pylib's modules. Inserting pylib right at the beginning of PYTHONPATH resolves that issue --- nix/batou.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nix/batou.nix b/nix/batou.nix index d7e53cf7a..d3bc9e4de 100644 --- a/nix/batou.nix +++ b/nix/batou.nix @@ -23,6 +23,7 @@ pytest-cov, pytest-instafail, pytest-timeout, + python, }: buildPythonPackage { build-system = [setuptools]; @@ -59,6 +60,10 @@ buildPythonPackage { PY_IGNORE_IMPORTMISMATCH = 1; + preCheck = '' + export PYTHONPATH=${py}/${python.sitePackages}:$PYTHONPATH + ''; + disabledTests = [ # requires internet access "test_manage__2_age" From 8b79e1418c528611011f6a3b8b252e822fab3f71 Mon Sep 17 00:00:00 2001 From: Philipp Herzog Date: Tue, 12 Nov 2024 12:57:04 +0100 Subject: [PATCH 5/5] wip: disable last failing check that can probably be fixed --- nix/batou.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nix/batou.nix b/nix/batou.nix index d3bc9e4de..1c3c3716a 100644 --- a/nix/batou.nix +++ b/nix/batou.nix @@ -75,6 +75,10 @@ buildPythonPackage { # cannot resolve hosts in sandbox "test_address_netloc_attributes" "test_address_resolves_listen_address" + + # does not return relative but absolute paths + # can probably be fixed, then reenabled + "test_edit_command_loop" ]; disabledTestPaths = [