Update CI files for branch 2.22

.ci/ansible/Containerfile.j2

@@ -44,4 +44,8 @@ RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
 RUN mkdir /var/lib/pulp/.config
 USER root:root
 
+# Lots of plugins try to use this path, and throw warnings if they cannot access it.
+RUN mkdir /.pytest_cache
+RUN chown pulp:pulp /.pytest_cache
+
 ENTRYPOINT ["/init"]

.ci/ansible/build_container.yaml

@@ -1,6 +1,7 @@
 # Ansible playbook to create the pulp service containers image
 ---
-- hosts: "localhost"
+- name: "Build CI Container Image"
+  hosts: "localhost"
   gather_facts: false
   vars_files:
     - "vars/main.yaml"
@@ -9,6 +10,7 @@
       ansible.builtin.template:
         src: "Containerfile.j2"
         dest: "Containerfile"
+
     - name: "Build pulp image"
       # We build from the ../.. (parent dir of pulpcore git repo) Docker build
       # "context" so that repos like pulp-smash are accessible to Docker
@@ -18,9 +20,20 @@
       # 1-off-builds and CI purposes (which has no cache across CI runs.)
       # Run build.yaml with -e cache=false if your builds are using outdated
       # layers.
-      command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.."
+      ansible.builtin.command:
+        argv:
+          - "docker"
+          - "build"
+          - "--network"
+          - "host"
+          - "--no-cache={{ not cache | default(true) | bool }}"
+          - "-t"
+          - "{{ image.name }}:{{ image.tag }}"
+          - "-f"
+          - "{{ playbook_dir }}/Containerfile"
+          - "../../.."
 
     - name: "Clean image cache"
-      docker_prune:
-        images : true
+      community.docker.docker_prune:
+        images: true
 ...

.ci/ansible/filter/repr.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import, division, print_function
+
 from packaging.version import parse as parse_version
 
 __metaclass__ = type

.ci/ansible/settings.py.j2

@@ -10,10 +10,6 @@ REDIS_HOST = "localhost"
 REDIS_PORT = 6379
 ANALYTICS = False
 
-{% if api_root is defined %}
-API_ROOT = {{ api_root | repr }}
-{% endif %}
-
 {% if pulp_settings %}
 {% for key, value in pulp_settings.items() %}
 {{ key | upper }} = {{ value | repr }}

.ci/ansible/start_container.yaml

@@ -1,6 +1,7 @@
 # Ansible playbook to start the pulp service container and its supporting services
 ---
-- hosts: "localhost"
+- name: "Start CI Containers"
+  hosts: "localhost"
   gather_facts: false
   vars_files:
     - "vars/main.yaml"
@@ -14,16 +15,16 @@
         - "settings"
 
     - name: "Generate Pulp Settings"
-      template:
+      ansible.builtin.template:
         src: "settings.py.j2"
         dest: "settings/settings.py"
 
     - name: "Setup docker networking"
-      docker_network:
+      community.docker.docker_network:
         name: "pulp_ci_bridge"
 
     - name: "Start Service Containers"
-      docker_container:
+      community.docker.docker_container:
         name: "{{ item.name }}"
         image: "{{ item.image }}"
         auto_remove: true
@@ -39,12 +40,12 @@
       loop: "{{ services | default([]) }}"
 
     - name: "Retrieve Docker Network Info"
-      docker_network_info:
+      community.docker.docker_network_info:
         name: "pulp_ci_bridge"
       register: "pulp_ci_bridge_info"
 
     - name: "Update /etc/hosts"
-      lineinfile:
+      ansible.builtin.lineinfile:
         path: "/etc/hosts"
         regexp: "\\s{{ item.value.Name }}\\s*$"
         line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}"
@@ -55,16 +56,27 @@
       amazon.aws.s3_bucket:
         aws_access_key: "{{ minio_access_key }}"
         aws_secret_key: "{{ minio_secret_key }}"
-        s3_url: "http://minio:9000"
+        endpoint_url: "http://minio:9000"
         region: "eu-central-1"
         name: "pulp3"
         state: "present"
       when: "s3_test | default(false)"
 
-    - block:
+    - name: "Wait on Services"
+      block:
+        - name: "Wait for azurite"
+          ansible.builtin.uri:
+            url: "http://ci-azurite:10000/"
+            status_code:
+              - 200
+              - 400
+          when: "azure_test | default(false)"
+          retries: 2
+          delay: 5
+
         - name: "Wait for Pulp"
-          uri:
-            url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/"
+          ansible.builtin.uri:
+            url: "http://pulp{{ pulp_scenario_settings.api_root | default(pulp_settings.api_root | default('\/pulp\/', True), True) }}api/v3/status/"
             follow_redirects: "all"
             validate_certs: "no"
           register: "result"
@@ -73,11 +85,12 @@
           delay: 5
       rescue:
         - name: "Output pulp container log"
-          command: "docker logs pulp"
+          ansible.builtin.command:
+            cmd: "docker logs pulp"
           failed_when: true
 
     - name: "Check version of component being tested"
-      assert:
+      ansible.builtin.assert:
         that:
           - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)"
         fail_msg: |
@@ -86,14 +99,15 @@
       loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}"
 
     - name: "Set pulp password in .netrc"
-      copy:
+      ansible.builtin.copy:
         dest: "~/.netrc"
         content: |
           machine pulp
           login admin
           password password
 
-- hosts: "pulp"
+- name: "Prepare Pulp Application Container"
+  hosts: "pulp"
   gather_facts: false
   tasks:
     - name: "Create directory for pulp-smash config"
@@ -108,6 +122,6 @@
         dest: "/var/lib/pulp/.config/pulp_smash/settings.json"
 
     - name: "Set pulp admin password"
-      command:
+      ansible.builtin.command:
         cmd: "pulpcore-manager reset-admin-password --password password"
 ...

.ci/scripts/calc_constraints.py

@@ -7,11 +7,12 @@
 
 import argparse
 import fileinput
-import urllib.request
 import sys
+import urllib.request
+
+import yaml
 from packaging.requirements import Requirement
 from packaging.version import Version
-import yaml
 
 try:
     import tomllib

.ci/scripts/check_pulpcore_imports.sh

@@ -10,10 +10,10 @@
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../..
 
-set -uv
+set -u
 
 # check for imports not from pulpcore.plugin. exclude tests
-MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin")
+MATCHES="$(grep -n -r --include \*.py "from pulpcore.*import" pulp_container | grep -v "tests\|plugin")"
 
 if [ $? -ne 1 ]; then
   printf "\nERROR: Detected bad imports from pulpcore:\n"

.ci/scripts/check_release.py

@@ -9,16 +9,16 @@
 # ///
 
 import argparse
-import re
 import os
+import re
 import sys
-import tomllib
 import typing as t
 from pathlib import Path
 
+import tomllib
 import yaml
-from packaging.version import Version
 from git import Repo
+from packaging.version import Version
 
 RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$"
 Y_CHANGELOG_EXTS = [".feature"]
@@ -157,9 +157,9 @@ def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int:
 
             if reasons:
                 curr_version = Version(last_tag)
-                assert curr_version.base_version.startswith(
-                    branch
-                ), "Current-version has to belong to the current branch!"
+                assert curr_version.base_version.startswith(branch), (
+                    "Current-version has to belong to the current branch!"
+                )
                 next_version = Version(f"{branch}.{curr_version.micro + 1}")
                 print(
                     f"A Z-release is needed for {branch}, "

.ci/scripts/check_requirements.py

@@ -5,14 +5,14 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
-import tomllib
 import warnings
+
+import tomllib
 from packaging.requirements import Requirement
 
 CHECK_MATRIX = [
     ("pyproject.toml", True, True, True),
     ("requirements.txt", True, True, True),
-    ("dev_requirements.txt", False, True, False),
     ("ci_requirements.txt", False, True, True),
     ("doc_requirements.txt", False, True, False),
     ("lint_requirements.txt", False, True, True),

.ci/scripts/collect_changes.py

@@ -18,10 +18,10 @@
 import json
 import os
 import re
-import tomllib
 import urllib.request
 from pathlib import Path
 
+import tomllib
 from git import GitCommandError, Repo
 from packaging.version import parse as parse_version
 

.ci/scripts/pr_labels.py

@@ -4,9 +4,9 @@
 
 import re
 import sys
-import tomllib
 from pathlib import Path
 
+import tomllib
 from git import Repo
 
 

.ci/scripts/schema.py

@@ -9,6 +9,7 @@
 """
 
 import json
+
 from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH
 
 with open(JSON_SCHEMA_SPEC_PATH) as fh:

.ci/scripts/skip_tests.py

@@ -15,12 +15,13 @@
     *: Error
 """
 
-import sys
+import argparse
 import os
 import re
-import git
+import sys
 import textwrap
-import argparse
+
+import git
 
 DOC_PATTERNS = [
     r"^docs/",

.ci/scripts/update_github.py

@@ -6,6 +6,7 @@
 # For more info visit https://github.com/pulp/plugin_template
 
 import os
+
 from github import Github
 
 g = Github(os.environ.get("GITHUB_TOKEN"))

.ci/scripts/validate_commit_message.py

@@ -5,10 +5,10 @@
 import re
 import subprocess
 import sys
-import tomllib
-import yaml
 from pathlib import Path
 
+import tomllib
+import yaml
 from github import Github