diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index fb9ed4a1c..a446f3394 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -44,4 +44,8 @@ RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ RUN mkdir /var/lib/pulp/.config USER root:root +# Lots of plugins try to use this path, and throw warnings if they cannot access it. +RUN mkdir /.pytest_cache +RUN chown pulp:pulp /.pytest_cache + ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index 0a188cba9..0ffd21d4f 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,6 +1,7 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: "localhost" +- name: "Build CI Container Image" + hosts: "localhost" gather_facts: false vars_files: - "vars/main.yaml" @@ -9,6 +10,7 @@ ansible.builtin.template: src: "Containerfile.j2" dest: "Containerfile" + - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker @@ -18,9 +20,20 @@ # 1-off-builds and CI purposes (which has no cache across CI runs.) # Run build.yaml with -e cache=false if your builds are using outdated # layers. - command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.." + ansible.builtin.command: + argv: + - "docker" + - "build" + - "--network" + - "host" + - "--no-cache={{ not cache | default(true) | bool }}" + - "-t" + - "{{ image.name }}:{{ image.tag }}" + - "-f" + - "{{ playbook_dir }}/Containerfile" + - "../../.." - name: "Clean image cache" - docker_prune: - images : true + community.docker.docker_prune: + images: true ... diff --git a/.ci/ansible/filter/repr.py b/.ci/ansible/filter/repr.py index 8455c3442..c8c1678dd 100644 --- a/.ci/ansible/filter/repr.py +++ b/.ci/ansible/filter/repr.py @@ -1,4 +1,5 @@ from __future__ import absolute_import, division, print_function + from packaging.version import parse as parse_version __metaclass__ = type diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index dfe2851de..ab8ebb3dc 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -10,10 +10,6 @@ REDIS_HOST = "localhost" REDIS_PORT = 6379 ANALYTICS = False -{% if api_root is defined %} -API_ROOT = {{ api_root | repr }} -{% endif %} - {% if pulp_settings %} {% for key, value in pulp_settings.items() %} {{ key | upper }} = {{ value | repr }} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index acdc22ad3..e0891b7ab 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,6 +1,7 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: "localhost" +- name: "Start CI Containers" + hosts: "localhost" gather_facts: false vars_files: - "vars/main.yaml" @@ -14,16 +15,16 @@ - "settings" - name: "Generate Pulp Settings" - template: + ansible.builtin.template: src: "settings.py.j2" dest: "settings/settings.py" - name: "Setup docker networking" - docker_network: + community.docker.docker_network: name: "pulp_ci_bridge" - name: "Start Service Containers" - docker_container: + community.docker.docker_container: name: "{{ item.name }}" image: "{{ item.image }}" auto_remove: true @@ -39,12 +40,12 @@ loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" - docker_network_info: + community.docker.docker_network_info: name: "pulp_ci_bridge" register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" - lineinfile: + ansible.builtin.lineinfile: path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" @@ -55,16 +56,27 @@ amazon.aws.s3_bucket: aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" - s3_url: "http://minio:9000" + endpoint_url: "http://minio:9000" region: "eu-central-1" name: "pulp3" state: "present" when: "s3_test | default(false)" - - block: + - name: "Wait on Services" + block: + - name: "Wait for azurite" + ansible.builtin.uri: + url: "http://ci-azurite:10000/" + status_code: + - 200 + - 400 + when: "azure_test | default(false)" + retries: 2 + delay: 5 + - name: "Wait for Pulp" - uri: - url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" + ansible.builtin.uri: + url: "http://pulp{{ pulp_scenario_settings.api_root | default(pulp_settings.api_root | default('\/pulp\/', True), True) }}api/v3/status/" follow_redirects: "all" validate_certs: "no" register: "result" @@ -73,11 +85,12 @@ delay: 5 rescue: - name: "Output pulp container log" - command: "docker logs pulp" + ansible.builtin.command: + cmd: "docker logs pulp" failed_when: true - name: "Check version of component being tested" - assert: + ansible.builtin.assert: that: - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" fail_msg: | @@ -86,14 +99,15 @@ loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}" - name: "Set pulp password in .netrc" - copy: + ansible.builtin.copy: dest: "~/.netrc" content: | machine pulp login admin password password -- hosts: "pulp" +- name: "Prepare Pulp Application Container" + hosts: "pulp" gather_facts: false tasks: - name: "Create directory for pulp-smash config" @@ -108,6 +122,6 @@ dest: "/var/lib/pulp/.config/pulp_smash/settings.json" - name: "Set pulp admin password" - command: + ansible.builtin.command: cmd: "pulpcore-manager reset-admin-password --password password" ... diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 792f30d24..0b05b5a84 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -7,11 +7,12 @@ import argparse import fileinput -import urllib.request import sys +import urllib.request + +import yaml from packaging.requirements import Requirement from packaging.version import Version -import yaml try: import tomllib diff --git a/.ci/scripts/check_gettext.sh b/.ci/scripts/check_gettext.sh deleted file mode 100755 index 94459f171..000000000 --- a/.ci/scripts/check_gettext.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../.. - -set -uv - -MATCHES=$(grep -n -r --include \*.py "_(f") - -if [ $? -ne 1 ]; then - printf "\nERROR: Detected mix of f-strings and gettext:\n" - echo "$MATCHES" - exit 1 -fi diff --git a/.ci/scripts/check_pulpcore_imports.sh b/.ci/scripts/check_pulpcore_imports.sh index 58900f0b2..4852dc779 100755 --- a/.ci/scripts/check_pulpcore_imports.sh +++ b/.ci/scripts/check_pulpcore_imports.sh @@ -10,10 +10,10 @@ # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../.. -set -uv +set -u # check for imports not from pulpcore.plugin. exclude tests -MATCHES=$(grep -n -r --include \*.py "from pulpcore.*import" . | grep -v "tests\|plugin") +MATCHES="$(grep -n -r --include \*.py "from pulpcore.*import" pulp_container | grep -v "tests\|plugin")" if [ $? -ne 1 ]; then printf "\nERROR: Detected bad imports from pulpcore:\n" diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 6e0799952..32e2a2ffd 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -9,16 +9,16 @@ # /// import argparse -import re import os +import re import sys -import tomllib import typing as t from pathlib import Path +import tomllib import yaml -from packaging.version import Version from git import Repo +from packaging.version import Version RELEASE_BRANCH_REGEX = r"^([0-9]+)\.([0-9]+)$" Y_CHANGELOG_EXTS = [".feature"] @@ -157,9 +157,9 @@ def main(options: argparse.Namespace, template_config: dict[str, t.Any]) -> int: if reasons: curr_version = Version(last_tag) - assert curr_version.base_version.startswith( - branch - ), "Current-version has to belong to the current branch!" + assert curr_version.base_version.startswith(branch), ( + "Current-version has to belong to the current branch!" + ) next_version = Version(f"{branch}.{curr_version.micro + 1}") print( f"A Z-release is needed for {branch}, " diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index 0a309798e..48bfa45bd 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,14 +5,14 @@ # # For more info visit https://github.com/pulp/plugin_template -import tomllib import warnings + +import tomllib from packaging.requirements import Requirement CHECK_MATRIX = [ ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), - ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), ("doc_requirements.txt", False, True, False), ("lint_requirements.txt", False, True, True), diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index f5c6b71a2..a4328e95a 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -18,10 +18,10 @@ import json import os import re -import tomllib import urllib.request from pathlib import Path +import tomllib from git import GitCommandError, Repo from packaging.version import parse as parse_version diff --git a/.ci/scripts/pr_labels.py b/.ci/scripts/pr_labels.py index 0c478a212..4f801c39e 100755 --- a/.ci/scripts/pr_labels.py +++ b/.ci/scripts/pr_labels.py @@ -4,9 +4,9 @@ import re import sys -import tomllib from pathlib import Path +import tomllib from git import Repo diff --git a/.ci/scripts/schema.py b/.ci/scripts/schema.py index 91191c5ed..9c8e11b2e 100644 --- a/.ci/scripts/schema.py +++ b/.ci/scripts/schema.py @@ -9,6 +9,7 @@ """ import json + from drf_spectacular.validation import JSON_SCHEMA_SPEC_PATH with open(JSON_SCHEMA_SPEC_PATH) as fh: diff --git a/.ci/scripts/skip_tests.py b/.ci/scripts/skip_tests.py index a68d000d6..380a3da9f 100755 --- a/.ci/scripts/skip_tests.py +++ b/.ci/scripts/skip_tests.py @@ -15,12 +15,13 @@ *: Error """ -import sys +import argparse import os import re -import git +import sys import textwrap -import argparse + +import git DOC_PATTERNS = [ r"^docs/", diff --git a/.ci/scripts/update_github.py b/.ci/scripts/update_github.py index 91113a652..5ed88d7c8 100755 --- a/.ci/scripts/update_github.py +++ b/.ci/scripts/update_github.py @@ -6,6 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template import os + from github import Github g = Github(os.environ.get("GITHUB_TOKEN")) diff --git a/.ci/scripts/validate_commit_message.py b/.ci/scripts/validate_commit_message.py index a4dc9004a..96b84172f 100644 --- a/.ci/scripts/validate_commit_message.py +++ b/.ci/scripts/validate_commit_message.py @@ -5,10 +5,10 @@ import re import subprocess import sys -import tomllib -import yaml from pathlib import Path +import tomllib +import yaml from github import Github diff --git a/.flake8 b/.flake8 deleted file mode 100644 index bccbff9f8..000000000 --- a/.flake8 +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_container' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template -[flake8] -exclude = ./docs/*,*/migrations/* -per-file-ignores = */__init__.py: F401 - -ignore = E203,W503,Q000,Q003,D100,D104,D106,D200,D205,D400,D401,D402,F824 -max-line-length = 100 - -# Flake8 builtin codes -# -------------------- -# E203: no whitespace around ':'. disabled until https://github.com/PyCQA/pycodestyle/issues/373 is fixed -# W503: This enforces operators before line breaks which is not pep8 or black compatible. -# F824: 'nonlocal' is unused: name is never assigned in scope - -# Flake8-quotes extension codes -# ----------------------------- -# Q000: double or single quotes only, default is double (don't want to enforce this) -# Q003: Change outer quotes to avoid escaping inner quotes - -# Flake8-docstring extension codes -# -------------------------------- -# D100: missing docstring in public module -# D104: missing docstring in public package -# D106: missing docstring in public nested class (complains about "class Meta:" and documenting those is silly) -# D200: one-line docstring should fit on one line with quotes -# D205: 1 blank line required between summary line and description -# D400: First line should end with a period -# D401: first line should be imperative (nitpicky) -# D402: first line should not be the function’s “signature” (false positives) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3f60c9354..01237fdae 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -36,20 +36,13 @@ jobs: run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows - # run black separately from flake8 to get a diff - - name: "Run black" + - name: "Check formating" run: | - black --version - black --check --diff . + ruff format --check --diff - # Lint code. - - name: "Run flake8" + - name: "Lint code" run: | - flake8 - - - name: "Check for common gettext problems" - run: | - sh .ci/scripts/check_gettext.sh + ruff check - name: "Run extra lint checks" run: | diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index d768965bb..7386fc912 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -29,7 +29,7 @@ if [ "$TEST" = "s3" ]; then COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" fi if [ "$TEST" = "azure" ]; then - COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure,uvloop]" fi if [[ "$TEST" = "pulp" ]]; then @@ -39,10 +39,6 @@ if [[ "$TEST" = "lowerbounds" ]]; then python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi -export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") - -echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" - # Compose the scenario definition. mkdir -p .ci/ansible/vars @@ -54,9 +50,8 @@ legacy_component_name: "pulp_container" component_name: "container" component_version: "${COMPONENT_VERSION}" pulp_env: {} -pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "flatpak_index": true} +pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "api_root": "/pulp/", "flatpak_index": true} pulp_scheme: "https" -api_root: "${PULP_API_ROOT}" image: name: "pulp" tag: "ci_build" @@ -93,7 +88,7 @@ if [ "$TEST" = "s3" ]; then s3_test: true minio_access_key: "${MINIO_ACCESS_KEY}" minio_secret_key: "${MINIO_SECRET_KEY}" -pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "flatpak_index": false, "token_auth_disabled": true} +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "api_root": "/rerouted/djnd/", "flatpak_index": false, "token_auth_disabled": true} pulp_scenario_env: {} VARSYAML fi diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 188ac6103..ba210fe05 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,7 +15,7 @@ REPO_ROOT="$PWD" source .github/workflows/scripts/utils.sh -PIP_REQUIREMENTS=("pulp-cli") +PIP_REQUIREMENTS=("pulp-cli" "yq") # This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} @@ -23,7 +23,7 @@ pip install ${PIP_REQUIREMENTS[*]} if [[ "$TEST" = "s3" ]]; then for i in {1..3} do - ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 + ansible-galaxy collection install "amazon.aws:11.1.0" && s=0 && break || s=$? && sleep 3 done if [[ $s -gt 0 ]] then @@ -36,18 +36,15 @@ fi PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -cd .ci/ansible/ +PULP_API_ROOT="$(yq -r '.pulp_scenario_settings.api_root // .pulp_settings.api_root // "/pulp/"' < .ci/ansible/vars/main.yaml)" pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password" -cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" +cp ~/.config/pulp/cli.toml "../pulp-cli/tests/cli.toml" + +cd .ci/ansible/ ansible-playbook build_container.yaml ansible-playbook start_container.yaml - -# Plugins often write to ~/.config/pulp/cli.toml from the host -chmod 777 ~/.config/pulp -chmod 666 ~/.config/pulp/cli.toml -sudo chown -R 700:700 ~/.config echo ::group::SSL # Copy pulp CA sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt @@ -72,7 +69,3 @@ fi # Needed for some functional tests cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" - -# Lots of plugins try to use this path, and throw warnings if they cannot access it. -cmd_prefix mkdir /.pytest_cache -cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh index a08353cdb..40bbcb198 100755 --- a/.github/workflows/scripts/release.sh +++ b/.github/workflows/scripts/release.sh @@ -24,4 +24,5 @@ towncrier build --yes --version "${NEW_VERSION}" bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty bump-my-version bump patch --commit -git push origin "${BRANCH}" "${NEW_VERSION}" +# Git push is not atomic by default! +git push --atomic origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/stage-changelog-for-default-branch.py b/.github/workflows/scripts/stage-changelog-for-default-branch.py index 515ef60fc..34ba7d736 100755 --- a/.github/workflows/scripts/stage-changelog-for-default-branch.py +++ b/.github/workflows/scripts/stage-changelog-for-default-branch.py @@ -12,13 +12,15 @@ from git import Repo from git.exc import GitCommandError -helper = textwrap.dedent("""\ +helper = textwrap.dedent( + """\ Stage the changelog for a release on main branch. Example: $ python .github/workflows/scripts/stage-changelog-for-default-branch.py 3.4.0 - """) + """ +) parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index bff8b77ac..290401d78 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -5,10 +5,11 @@ # # For more info visit https://github.com/pulp/plugin_template +import os +import random + import requests import yaml -import random -import os def random_color(): diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 662b3038f..7f3385267 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,7 +134,7 @@ jobs: if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + pulp status || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/.gitleaks.toml b/.gitleaks.toml index bda71d9be..6d2f9481d 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -1,10 +1,7 @@ [allowlist] - description = "Our test install exports a test only MINIO ACCESS KEY" + description = "Allow specific test-keys." paths = [ ".github/workflows/scripts/install.sh", ".travis/install.sh", ] - regexes = [ - "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkdNMkQ6SU9CVDpHQVpEOk1aUlE6RzQyVzpDWkJaOkdWUlQ6R00zRzpNRTJUOlFNSlk6R1JURDpNTUpRIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoicHVscDMtc291cmNlLWZlZG9yYTMxLmxvY2FsaG9zdC5leGFtcGxlLmNvbSIsImV4cCI6MTU5NDYzNDU0NSwiaWF0IjoxNTk0NjM0MjQ1LCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjI0ODE3L3Rva2VuLyIsImp0aSI6ImU4ZTUyYzVhLWYxMzAtNGJlMi1iNjFhLTUwNzVhMjhkMTA0YSIsIm5iZiI6MTU5NDYzNDI0NSwic3ViIjoiIn0.ySDUHooaURbsyKLkHoXqA1JJPwlcDtpz_u6GgcqA8fmFGmSWJFlAGYtA2GLXDzPioH-bh1JkMJdBDs61c5JnFw", - "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkhBM1Q6SVlSUjpHUTNUOklPTEM6TVE0RzpFT0xDOkdGUVQ6QVpURTpHQlNXOkNaUlY6TUlZVzpLTkpWIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoibG9jYWxob3N0OjI0ODE2IiwiZXhwIjoxNTcxMDcxOTUzLCJpYXQiOjE1NzEwNzE2NTMsImlzcyI6ImxvY2FsaG9zdDoyNDgxNi90b2tlbiIsImp0aSI6IjRmYTliYTYwLTY0ZTUtNDA3MC1hMzMyLWZmZTRlMTk2YzVjNyIsIm5iZiI6MTU3MTA3MTY1Mywic3ViIjoiIn0.pirj8yhbjYnldxmZ-jIZ72VJrzxkAnwLXLu1ND9QAL-kl3gZrvPbp98w2xdhEoQ_7WEka4veb6uU5ZzmD87X1Q", - ] + regexes = ['''AKIAIT2Z5TDYPX3ARJBA''', '''Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw''', "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkdNMkQ6SU9CVDpHQVpEOk1aUlE6RzQyVzpDWkJaOkdWUlQ6R00zRzpNRTJUOlFNSlk6R1JURDpNTUpRIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoicHVscDMtc291cmNlLWZlZG9yYTMxLmxvY2FsaG9zdC5leGFtcGxlLmNvbSIsImV4cCI6MTU5NDYzNDU0NSwiaWF0IjoxNTk0NjM0MjQ1LCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjI0ODE3L3Rva2VuLyIsImp0aSI6ImU4ZTUyYzVhLWYxMzAtNGJlMi1iNjFhLTUwNzVhMjhkMTA0YSIsIm5iZiI6MTU5NDYzNDI0NSwic3ViIjoiIn0.ySDUHooaURbsyKLkHoXqA1JJPwlcDtpz_u6GgcqA8fmFGmSWJFlAGYtA2GLXDzPioH-bh1JkMJdBDs61c5JnFw", "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkhBM1Q6SVlSUjpHUTNUOklPTEM6TVE0RzpFT0xDOkdGUVQ6QVpURTpHQlNXOkNaUlY6TUlZVzpLTkpWIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoibG9jYWxob3N0OjI0ODE2IiwiZXhwIjoxNTcxMDcxOTUzLCJpYXQiOjE1NzEwNzE2NTMsImlzcyI6ImxvY2FsaG9zdDoyNDgxNi90b2tlbiIsImp0aSI6IjRmYTliYTYwLTY0ZTUtNDA3MC1hMzMyLWZmZTRlMTk2YzVjNyIsIm5iZiI6MTU3MTA3MTY1Mywic3ViIjoiIn0.pirj8yhbjYnldxmZ-jIZ72VJrzxkAnwLXLu1ND9QAL-kl3gZrvPbp98w2xdhEoQ_7WEka4veb6uU5ZzmD87X1Q", '''fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS''', '''qR\+vjWPU50fCqQuUWbj9Fain/j2pV\+ZtBCiDiieS'''] diff --git a/dev_requirements.txt b/dev_requirements.txt deleted file mode 100644 index 4e043dcbf..000000000 --- a/dev_requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -check-manifest -coverage -flake8 -flake8-black -flake8-docstrings -flake8-tuple -flake8-quotes -pydocstyle -requests diff --git a/lint_requirements.txt b/lint_requirements.txt index 06aee86c5..a24786418 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,10 +5,8 @@ # # For more info visit https://github.com/pulp/plugin_template -black~=26.3 # Pin style to the year. https://black.readthedocs.io/en/stable/faq.html#how-stable-is-black-s-style bump-my-version check-manifest -flake8 -flake8-black packaging +ruff yamllint diff --git a/pulp_container/app/__init__.py b/pulp_container/app/__init__.py index 8fd0f990d..b3ae06ac8 100644 --- a/pulp_container/app/__init__.py +++ b/pulp_container/app/__init__.py @@ -1,7 +1,8 @@ -from pulpcore.plugin import PulpPluginAppConfig from django.db import connection from django.db.models.signals import post_migrate +from pulpcore.plugin import PulpPluginAppConfig + update_sequences_to_bigint = """ ALTER TABLE container_blobmanifest ALTER COLUMN id TYPE bigint; ALTER TABLE container_manifestlistmanifest ALTER COLUMN id TYPE bigint; @@ -34,6 +35,6 @@ def update_sequences(sender, **kwargs): def ready(self): super().ready() - from . import checks + from . import checks # noqa: F401 post_migrate.connect(PulpContainerPluginAppConfig.update_sequences, sender=self) diff --git a/pulp_container/app/authorization.py b/pulp_container/app/authorization.py index e79fff151..9d719674e 100644 --- a/pulp_container/app/authorization.py +++ b/pulp_container/app/authorization.py @@ -2,27 +2,24 @@ import hashlib import random import uuid - -import jwt - from collections import defaultdict, namedtuple from datetime import datetime from functools import partial +import jwt +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization from django.conf import settings -from django.http import HttpRequest from django.db.models import F, Value +from django.http import HttpRequest from rest_framework.request import Request -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization - +from pulp_container.app.access_policy import RegistryAccessPolicy from pulp_container.app.models import ( ContainerDistribution, ContainerNamespace, ContainerPullThroughDistribution, ) -from pulp_container.app.access_policy import RegistryAccessPolicy TOKEN_EXPIRATION_TIME = settings.get("TOKEN_EXPIRATION_TIME", 300) diff --git a/pulp_container/app/cache.py b/pulp_container/app/cache.py index 38ed477e4..0e6c35f16 100644 --- a/pulp_container/app/cache.py +++ b/pulp_container/app/cache.py @@ -1,10 +1,10 @@ from django.core.exceptions import ObjectDoesNotExist from django.db.models import F, Value -from pulpcore.plugin.cache import CacheKeys, AsyncContentCache, SyncContentCache +from pulpcore.plugin.cache import AsyncContentCache, CacheKeys, SyncContentCache -from pulp_container.app.models import ContainerDistribution, ContainerPullThroughDistribution from pulp_container.app.exceptions import RepositoryNotFound +from pulp_container.app.models import ContainerDistribution, ContainerPullThroughDistribution ACCEPT_HEADER_KEY = "accept_header" QUERY_KEY = "query" diff --git a/pulp_container/app/checks.py b/pulp_container/app/checks.py index 43d7b8753..dd88ff9d7 100644 --- a/pulp_container/app/checks.py +++ b/pulp_container/app/checks.py @@ -1,5 +1,6 @@ from django.conf import settings -from django.core.checks import Error as CheckError, register +from django.core.checks import Error as CheckError +from django.core.checks import register @register(deploy=True) diff --git a/pulp_container/app/content.py b/pulp_container/app/content.py index fcdc7786f..b8f07e0eb 100644 --- a/pulp_container/app/content.py +++ b/pulp_container/app/content.py @@ -1,6 +1,7 @@ from aiohttp import web from pulpcore.plugin.content import app + from pulp_container.app.registry import Registry registry = Registry() diff --git a/pulp_container/app/downloaders.py b/pulp_container/app/downloaders.py index ce51f63f4..9dae12021 100644 --- a/pulp_container/app/downloaders.py +++ b/pulp_container/app/downloaders.py @@ -1,13 +1,13 @@ -import aiohttp import asyncio import json import re - -from aiohttp.client_exceptions import ClientResponseError from collections import namedtuple from logging import getLogger from urllib import parse +import aiohttp +from aiohttp.client_exceptions import ClientResponseError + from pulpcore.plugin.download import DownloaderFactory, HttpDownloader from pulp_container.constants import V2_ACCEPT_HEADERS diff --git a/pulp_container/app/fields.py b/pulp_container/app/fields.py index 110d2c55c..b124ea31a 100644 --- a/pulp_container/app/fields.py +++ b/pulp_container/app/fields.py @@ -1,5 +1,5 @@ -from drf_spectacular.utils import extend_schema_field from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from rest_framework import serializers diff --git a/pulp_container/app/management/commands/container-handle-image-data.py b/pulp_container/app/management/commands/container-handle-image-data.py index facdbee22..20869a41c 100644 --- a/pulp_container/app/management/commands/container-handle-image-data.py +++ b/pulp_container/app/management/commands/container-handle-image-data.py @@ -1,8 +1,6 @@ -from json.decoder import JSONDecodeError - -from gettext import gettext as _ - from contextlib import suppress +from gettext import gettext as _ +from json.decoder import JSONDecodeError from django.conf import settings from django.core.exceptions import ObjectDoesNotExist @@ -12,9 +10,7 @@ from pulpcore.plugin.cache import SyncContentCache from pulp_container.app.models import ContainerDistribution, Manifest - from pulp_container.app.utils import get_content_data - from pulp_container.constants import MEDIA_TYPE diff --git a/pulp_container/app/management/commands/container-repair-media-type.py b/pulp_container/app/management/commands/container-repair-media-type.py index abe258e11..90e8e1909 100644 --- a/pulp_container/app/management/commands/container-repair-media-type.py +++ b/pulp_container/app/management/commands/container-repair-media-type.py @@ -1,5 +1,4 @@ import json - from gettext import gettext as _ from django.conf import settings @@ -7,9 +6,8 @@ from pulpcore.plugin.cache import SyncContentCache -from pulp_container.app.models import Manifest, ContainerDistribution +from pulp_container.app.models import ContainerDistribution, Manifest from pulp_container.app.utils import determine_media_type_from_json - from pulp_container.constants import MEDIA_TYPE diff --git a/pulp_container/app/modelresource.py b/pulp_container/app/modelresource.py index b83dec1ba..8866b9e0f 100644 --- a/pulp_container/app/modelresource.py +++ b/pulp_container/app/modelresource.py @@ -1,11 +1,12 @@ from import_export import fields, widgets -from pulpcore.plugin.importexport import QueryModelResource, BaseContentResource + +from pulpcore.plugin.importexport import BaseContentResource, QueryModelResource from pulpcore.plugin.modelresources import RepositoryResource from pulp_container.app.models import ( Blob, - ContainerRepository, ContainerPushRepository, + ContainerRepository, Manifest, ManifestListManifest, ManifestSignature, diff --git a/pulp_container/app/models.py b/pulp_container/app/models.py index 0f6b9108c..6e97dbdbc 100644 --- a/pulp_container/app/models.py +++ b/pulp_container/app/models.py @@ -1,17 +1,16 @@ -from gettext import gettext as _ - import json import os import re import tempfile import time +from gettext import gettext as _ from logging import getLogger -from django.db import models from django.conf import settings from django.contrib.postgres import fields +from django.db import models from django.shortcuts import redirect -from django_lifecycle import hook, AFTER_CREATE, AFTER_DELETE, AFTER_UPDATE +from django_lifecycle import AFTER_CREATE, AFTER_DELETE, AFTER_UPDATE, hook from pulpcore.plugin.cache import SyncContentCache from pulpcore.plugin.download import DownloaderFactory @@ -20,17 +19,17 @@ AutoAddObjPermsMixin, BaseModel, Content, + Distribution, Remote, Repository, - Distribution, SigningService, +) +from pulpcore.plugin.models import ( Upload as CoreUpload, ) from pulpcore.plugin.repo_version_utils import remove_duplicates, validate_repo_version from pulpcore.plugin.util import gpg_verify - -from . import downloaders from pulp_container.app.utils import get_content_data, integer_overflow_safe from pulp_container.constants import ( COSIGN_MEDIA_TYPES, @@ -41,6 +40,8 @@ SIGNATURE_TYPE, ) +from . import downloaders + logger = getLogger(__name__) diff --git a/pulp_container/app/redirects.py b/pulp_container/app/redirects.py index b938a562a..afce3fe98 100644 --- a/pulp_container/app/redirects.py +++ b/pulp_container/app/redirects.py @@ -1,7 +1,7 @@ from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.shortcuts import redirect from django.http import Http404 +from django.shortcuts import redirect from pulp_container.app.exceptions import ManifestNotFound from pulp_container.app.utils import get_accepted_media_types diff --git a/pulp_container/app/registry.py b/pulp_container/app/registry.py index 44f0796fd..d15944ea1 100644 --- a/pulp_container/app/registry.py +++ b/pulp_container/app/registry.py @@ -1,35 +1,32 @@ import json import logging import os - -from asgiref.sync import sync_to_async - from contextlib import suppress from urllib.parse import urljoin from aiohttp import web -from aiohttp.client_exceptions import ClientResponseError, ClientConnectionError +from aiohttp.client_exceptions import ClientConnectionError, ClientResponseError from aiohttp.web_exceptions import HTTPTooManyRequests -from django_guid import set_guid -from django_guid.utils import generate_guid +from asgiref.sync import sync_to_async from django.core.exceptions import ObjectDoesNotExist from django.db import IntegrityError +from django_guid import set_guid +from django_guid.utils import generate_guid from multidict import MultiDict -from pulpcore.plugin.content import Handler, PathNotResolved -from pulpcore.plugin.util import get_domain -from pulpcore.plugin.models import RemoteArtifact, Content, ContentArtifact -from pulpcore.plugin.content import ArtifactResponse -from pulpcore.plugin.tasking import dispatch +from pulpcore.plugin.content import ArtifactResponse, Handler, PathNotResolved from pulpcore.plugin.exceptions import TimeoutException +from pulpcore.plugin.models import Content, ContentArtifact, RemoteArtifact +from pulpcore.plugin.tasking import dispatch +from pulpcore.plugin.util import get_domain from pulp_container.app.cache import RegistryContentCache -from pulp_container.app.models import ContainerDistribution, Tag, Blob, Manifest, BlobManifest +from pulp_container.app.models import Blob, BlobManifest, ContainerDistribution, Manifest, Tag from pulp_container.app.tasks import download_image_data from pulp_container.app.utils import ( calculate_digest, - get_accepted_media_types, determine_media_type, + get_accepted_media_types, save_artifact, ) from pulp_container.constants import BLOB_CONTENT_TYPE, EMPTY_BLOB, MEDIA_TYPE, V2_ACCEPT_HEADERS diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index e6bccb987..c916cd880 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -7,36 +7,27 @@ import base64 import binascii +import hashlib import json import logging -import hashlib import re - -from aiohttp.client_exceptions import ClientResponseError, ClientConnectionError from itertools import chain -from urllib.parse import urljoin, urlparse, urlunparse, parse_qs, urlencode from tempfile import NamedTemporaryFile +from urllib.parse import parse_qs, urlencode, urljoin, urlparse, urlunparse +from aiohttp.client_exceptions import ClientConnectionError, ClientResponseError +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.core.files.base import ContentFile, File from django.db import IntegrityError, transaction from django.db.models import F, Value from django.forms.models import model_to_dict from django.shortcuts import get_object_or_404 - -from django.conf import settings - -from pulpcore.plugin.models import Artifact, ContentArtifact, UploadChunk -from pulpcore.plugin.files import PulpTemporaryUploadedFile -from pulpcore.plugin.tasking import add_and_remove, dispatch -from pulpcore.plugin.util import get_objects_for_user, get_url, get_domain -from pulpcore.plugin.exceptions import TimeoutException - from rest_framework.exceptions import ( AuthenticationFailed, NotAuthenticated, - PermissionDenied, ParseError, + PermissionDenied, Throttled, ) from rest_framework.generics import ListAPIView @@ -46,38 +37,44 @@ from rest_framework.response import Response from rest_framework.serializers import ModelSerializer from rest_framework.settings import api_settings -from rest_framework.viewsets import ViewSet -from rest_framework.views import APIView, exception_handler from rest_framework.status import HTTP_401_UNAUTHORIZED +from rest_framework.views import APIView, exception_handler +from rest_framework.viewsets import ViewSet + +from pulpcore.plugin.exceptions import TimeoutException +from pulpcore.plugin.files import PulpTemporaryUploadedFile +from pulpcore.plugin.models import Artifact, ContentArtifact, UploadChunk +from pulpcore.plugin.tasking import add_and_remove, dispatch +from pulpcore.plugin.util import get_domain, get_objects_for_user, get_url from pulp_container.app import models, serializers from pulp_container.app.authorization import AuthorizationService, PermissionChecker from pulp_container.app.cache import ( - find_base_path_cached, FlatpakIndexStaticCache, RegistryApiCache, + find_base_path_cached, ) from pulp_container.app.exceptions import ( BadGateway, + BlobInvalid, + BlobNotFound, GatewayTimeout, InvalidRequest, - RepositoryNotFound, - RepositoryInvalid, - BlobNotFound, - BlobInvalid, - ManifestNotFound, ManifestInvalid, + ManifestNotFound, ManifestSignatureInvalid, + RepositoryInvalid, + RepositoryNotFound, ) from pulp_container.app.redirects import ( + AzureStorageRedirects, FileStorageRedirects, S3StorageRedirects, - AzureStorageRedirects, ) from pulp_container.app.token_verification import ( RegistryAuthentication, - TokenAuthentication, RegistryPermission, + TokenAuthentication, TokenPermission, ) from pulp_container.app.utils import ( diff --git a/pulp_container/app/replica.py b/pulp_container/app/replica.py index 85fab2254..53ef268cf 100644 --- a/pulp_container/app/replica.py +++ b/pulp_container/app/replica.py @@ -1,11 +1,11 @@ -from pulpcore.plugin.replica import Replicator -from pulpcore.plugin.util import get_url - from pulp_glue.container.context import ( PulpContainerDistributionContext, PulpContainerRepositoryContext, ) +from pulpcore.plugin.replica import Replicator +from pulpcore.plugin.util import get_url + from pulp_container.app.models import ContainerDistribution, ContainerRemote, ContainerRepository from pulp_container.app.tasks import synchronize as container_synchronize diff --git a/pulp_container/app/serializers.py b/pulp_container/app/serializers.py index 82f073b69..2487462f0 100644 --- a/pulp_container/app/serializers.py +++ b/pulp_container/app/serializers.py @@ -1,7 +1,8 @@ -from gettext import gettext as _ import re +from gettext import gettext as _ from django.core.validators import URLValidator +from pulp_file.app.models import FileContent from rest_framework import serializers from pulpcore.plugin.models import ( @@ -13,8 +14,8 @@ from pulpcore.plugin.serializers import ( ContentRedirectContentGuardSerializer, DetailRelatedField, - GetOrCreateSerializerMixin, DistributionSerializer, + GetOrCreateSerializerMixin, IdentityField, ModelSerializer, NestedRelatedField, @@ -28,8 +29,7 @@ ValidateFieldsMixin, ) -from pulp_file.app.models import FileContent -from pulp_container.app import models, fields +from pulp_container.app import fields, models from pulp_container.constants import SIGNATURE_TYPE VALID_SIGNATURE_NAME_REGEX = r"^sha256:[0-9a-f]{64}@[0-9a-f]{32}$" @@ -564,10 +564,8 @@ def validate(self, data): raise serializers.ValidationError( _( "A manifest with the digest '{}' does not " - "exist in the latest repository version '{}'".format( - new_data["digest"], latest_version - ) - ) + "exist in the latest repository version '{}'" + ).format(new_data["digest"], latest_version) ) new_data["manifest"] = manifest @@ -592,10 +590,8 @@ def validate(self, data): models.Tag.objects.get(pk__in=latest_version.content.all(), name=new_data["tag"]) except models.Tag.DoesNotExist: raise serializers.ValidationError( - _( - "The tag '{}' does not exist in the latest repository version '{}'".format( - new_data["tag"], latest_version - ) + _("The tag '{}' does not exist in the latest repository version '{}'").format( + new_data["tag"], latest_version ) ) @@ -621,7 +617,7 @@ def validate(self, data): if content_units: if "*" in content_units and len(content_units) > 1: raise serializers.ValidationError( - _("'*' should be the only item present in the {}".format(content_units)) + _("'*' should be the only item present in the {}").format(content_units) ) return data @@ -664,10 +660,7 @@ def validate(self, data): new_data.update(data) return new_data raise serializers.ValidationError( - _( - "Either the 'repository' or 'repository_version' need to be specified " - "but not both." - ) + _("Either the 'repository' or 'repository_version' need to be specified but not both.") ) @@ -719,10 +712,8 @@ def validate(self, data): raise serializers.ValidationError( _( "A manifest with the digest '{}' does not " - "exist in the latest repository version '{}'".format( - new_data["digest"], latest_version - ) - ) + "exist in the latest repository version '{}'" + ).format(new_data["digest"], latest_version) ) new_data["manifest"] = manifest @@ -762,10 +753,8 @@ def validate(self, data): raise serializers.ValidationError( _( "There are no signatures in the latest repository version '{}' " - "produced with the specified key_id '{}'".format( - latest_version, new_data["signed_with_key_id"] - ) - ) + "produced with the specified key_id '{}'" + ).format(latest_version, new_data["signed_with_key_id"]) ) new_data["sigs_pks"] = sigs_pks @@ -829,7 +818,6 @@ def deferred_files_validation(self, data): unnecessary database queries when checking permissions (DRF Access Policy). """ if build_context := data.get("build_context", None): - # check if the on_demand_artifacts exist for on_demand_artifact in build_context.on_demand_artifacts.iterator(): if not on_demand_artifact.content_artifact.artifact: diff --git a/pulp_container/app/tasks/builder.py b/pulp_container/app/tasks/builder.py index 530e64480..f334a3ebe 100644 --- a/pulp_container/app/tasks/builder.py +++ b/pulp_container/app/tasks/builder.py @@ -5,6 +5,13 @@ import tempfile from uuid import uuid4 +from pulpcore.plugin.models import ( + Artifact, + Content, + ContentArtifact, + PulpTemporaryFile, +) + from pulp_container.app.models import ( Blob, BlobManifest, @@ -12,14 +19,8 @@ Manifest, Tag, ) -from pulp_container.constants import MEDIA_TYPE from pulp_container.app.utils import calculate_digest -from pulpcore.plugin.models import ( - Artifact, - ContentArtifact, - Content, - PulpTemporaryFile, -) +from pulp_container.constants import MEDIA_TYPE def get_or_create_blob(layer_json, manifest, path): diff --git a/pulp_container/app/tasks/download_image_data.py b/pulp_container/app/tasks/download_image_data.py index 4086c11a0..9dac82216 100644 --- a/pulp_container/app/tasks/download_image_data.py +++ b/pulp_container/app/tasks/download_image_data.py @@ -7,8 +7,8 @@ from pulp_container.app.utils import determine_media_type_from_json from pulp_container.constants import MEDIA_TYPE -from .synchronize import ContainerDeclarativeVersion from .sync_stages import ContainerFirstStage +from .synchronize import ContainerDeclarativeVersion log = logging.getLogger(__name__) diff --git a/pulp_container/app/tasks/recursive_add.py b/pulp_container/app/tasks/recursive_add.py index 977fb8e70..da0254bf7 100644 --- a/pulp_container/app/tasks/recursive_add.py +++ b/pulp_container/app/tasks/recursive_add.py @@ -1,8 +1,8 @@ from pulp_container.app.models import ( + MEDIA_TYPE, Blob, ContainerRepository, Manifest, - MEDIA_TYPE, Tag, ) diff --git a/pulp_container/app/tasks/recursive_remove.py b/pulp_container/app/tasks/recursive_remove.py index f20b3b71d..72835e227 100644 --- a/pulp_container/app/tasks/recursive_remove.py +++ b/pulp_container/app/tasks/recursive_remove.py @@ -1,11 +1,12 @@ from django.db.models import Q + from pulpcore.plugin.models import Content, Repository from pulp_container.app.models import ( + MEDIA_TYPE, Blob, Manifest, ManifestSignature, - MEDIA_TYPE, Tag, ) diff --git a/pulp_container/app/tasks/sign.py b/pulp_container/app/tasks/sign.py index d4028bc41..e174b17d5 100644 --- a/pulp_container/app/tasks/sign.py +++ b/pulp_container/app/tasks/sign.py @@ -15,7 +15,6 @@ ManifestSigningService, Tag, ) - from pulp_container.app.utils import extract_data_from_signature from pulp_container.constants import ( MANIFEST_MEDIA_TYPES, diff --git a/pulp_container/app/tasks/sync_stages.py b/pulp_container/app/tasks/sync_stages.py index 7556157b9..d09207ebf 100644 --- a/pulp_container/app/tasks/sync_stages.py +++ b/pulp_container/app/tasks/sync_stages.py @@ -1,25 +1,16 @@ -import aiohttp import asyncio import base64 import hashlib import json import logging - from urllib.parse import urljoin, urlparse, urlunparse +import aiohttp from asgiref.sync import sync_to_async + from pulpcore.plugin.models import Artifact, ProgressReport, Remote -from pulpcore.plugin.stages import DeclarativeArtifact, DeclarativeContent, Stage, ContentSaver +from pulpcore.plugin.stages import ContentSaver, DeclarativeArtifact, DeclarativeContent, Stage -from pulp_container.constants import ( - MANIFEST_TYPE, - MEDIA_TYPE, - SIGNATURE_API_EXTENSION_VERSION, - SIGNATURE_HEADER, - SIGNATURE_SOURCE, - SIGNATURE_TYPE, - V2_ACCEPT_HEADERS, -) from pulp_container.app.models import ( Blob, BlobManifest, @@ -29,14 +20,23 @@ Tag, ) from pulp_container.app.utils import ( - extract_data_from_signature, - urlpath_sanitize, - determine_media_type, - validate_manifest, calculate_digest, + determine_media_type, + extract_data_from_signature, filter_resources, get_content_data, integer_overflow_safe, + urlpath_sanitize, + validate_manifest, +) +from pulp_container.constants import ( + MANIFEST_TYPE, + MEDIA_TYPE, + SIGNATURE_API_EXTENSION_VERSION, + SIGNATURE_HEADER, + SIGNATURE_SOURCE, + SIGNATURE_TYPE, + V2_ACCEPT_HEADERS, ) log = logging.getLogger(__name__) @@ -556,8 +556,9 @@ async def create_signatures(self, man_dc, signature_source): break except aiohttp.client_exceptions.ClientResponseError as exc: log.info( - "{} is not accessible, can't sync an image signature. " - "Error: {} {}".format(signature_url, exc.status, exc.message) + "{} is not accessible, can't sync an image signature. Error: {} {}".format( + signature_url, exc.status, exc.message + ) ) with open(signature_download_result.path, "rb") as f: diff --git a/pulp_container/app/tasks/synchronize.py b/pulp_container/app/tasks/synchronize.py index 792c10c0d..26f2560dc 100644 --- a/pulp_container/app/tasks/synchronize.py +++ b/pulp_container/app/tasks/synchronize.py @@ -4,15 +4,16 @@ ArtifactDownloader, ArtifactSaver, DeclarativeVersion, - RemoteArtifactSaver, - ResolveContentFutures, QueryExistingArtifacts, QueryExistingContents, + RemoteArtifactSaver, + ResolveContentFutures, ) -from .sync_stages import ContainerFirstStage, ContainerContentSaver from pulp_container.app.models import ContainerRemote, ContainerRepository +from .sync_stages import ContainerContentSaver, ContainerFirstStage + log = logging.getLogger(__name__) diff --git a/pulp_container/app/tasks/tag.py b/pulp_container/app/tasks/tag.py index 924381142..d504b8d33 100644 --- a/pulp_container/app/tasks/tag.py +++ b/pulp_container/app/tasks/tag.py @@ -1,4 +1,5 @@ from pulpcore.plugin.models import CreatedResource, Repository + from pulp_container.app.models import Manifest, Tag diff --git a/pulp_container/app/tasks/untag.py b/pulp_container/app/tasks/untag.py index de75ae431..68cde6834 100644 --- a/pulp_container/app/tasks/untag.py +++ b/pulp_container/app/tasks/untag.py @@ -1,4 +1,5 @@ from pulpcore.plugin.models import Repository + from pulp_container.app.models import Tag diff --git a/pulp_container/app/token_verification.py b/pulp_container/app/token_verification.py index 344f5f16c..d314bcf12 100644 --- a/pulp_container/app/token_verification.py +++ b/pulp_container/app/token_verification.py @@ -1,19 +1,17 @@ -import jwt import logging - from collections import namedtuple +import jwt from django.conf import settings -from django.contrib.auth.models import AnonymousUser from django.contrib.auth import get_user_model - +from django.contrib.auth.models import AnonymousUser from rest_framework.authentication import ( BaseAuthentication, BasicAuthentication, RemoteUserAuthentication, ) from rest_framework.exceptions import AuthenticationFailed, NotAuthenticated -from rest_framework.permissions import BasePermission, SAFE_METHODS +from rest_framework.permissions import SAFE_METHODS, BasePermission Scope = namedtuple("Scope", "resource_type, name, action") User = get_user_model() diff --git a/pulp_container/app/urls.py b/pulp_container/app/urls.py index c1f55cb84..ff2ce00d1 100644 --- a/pulp_container/app/urls.py +++ b/pulp_container/app/urls.py @@ -1,6 +1,7 @@ from django.conf import settings from django.urls import include, path from rest_framework.routers import Route, SimpleRouter + from pulp_container.app.registry_api import ( BearerTokenView, Blobs, diff --git a/pulp_container/app/utils.py b/pulp_container/app/utils.py index 2c2a1fc57..0bb635dc5 100644 --- a/pulp_container/app/utils.py +++ b/pulp_container/app/utils.py @@ -1,34 +1,33 @@ import base64 -import hashlib import fnmatch +import hashlib import json import logging import time +from functools import partial from asgiref.sync import sync_to_async -from jsonschema import Draft7Validator, validate, ValidationError -from pulpcore.plugin.util import get_domain from django.db import IntegrityError -from functools import partial -from rest_framework.exceptions import Throttled - +from jsonschema import Draft7Validator, ValidationError, validate from pysequoia.packet import PacketPile, Tag +from rest_framework.exceptions import Throttled from pulpcore.plugin.models import Artifact, Task +from pulpcore.plugin.util import get_domain -from pulp_container.constants import ( - MANIFEST_MEDIA_TYPES, - MEDIA_TYPE, -) from pulp_container.app.exceptions import ManifestInvalid from pulp_container.app.json_schemas import ( - OCI_INDEX_SCHEMA, - OCI_MANIFEST_SCHEMA, DOCKER_MANIFEST_LIST_V2_SCHEMA, - DOCKER_MANIFEST_V2_SCHEMA, DOCKER_MANIFEST_V1_SCHEMA, + DOCKER_MANIFEST_V2_SCHEMA, + OCI_INDEX_SCHEMA, + OCI_MANIFEST_SCHEMA, SIGNATURE_SCHEMA, ) +from pulp_container.constants import ( + MANIFEST_MEDIA_TYPES, + MEDIA_TYPE, +) signature_validator = Draft7Validator(SIGNATURE_SCHEMA) @@ -142,7 +141,7 @@ def extract_data_from_signature(signature_raw, man_digest): errors = [] for error in signature_validator.iter_errors(sig_json): - errors.append(f'{".".join(error.path)}: {error.message}') + errors.append(f"{'.'.join(error.path)}: {error.message}") if errors: raise ValueError("The signature for {} is not synced due to: {}".format(man_digest, errors)) @@ -247,7 +246,7 @@ def validate_manifest(content_data, media_type, digest): except ValidationError as error: # fail on the first encountered error raise ManifestInvalid( - reason=f'{".".join(map(str, error.path))}: {error.message}', digest=digest + reason=f"{'.'.join(map(str, error.path))}: {error.message}", digest=digest ) diff --git a/pulp_container/app/viewsets.py b/pulp_container/app/viewsets.py index ca381585a..2057b26f0 100644 --- a/pulp_container/app/viewsets.py +++ b/pulp_container/app/viewsets.py @@ -8,16 +8,13 @@ import logging from django.db.models import Q - from django_filters import CharFilter, MultipleChoiceFilter from drf_spectacular.utils import extend_schema - from rest_framework import mixins from rest_framework.decorators import action -from pulpcore.plugin.models import RepositoryVersion, PulpTemporaryFile +from pulpcore.plugin.models import Content, PulpTemporaryFile, RepositoryVersion from pulpcore.plugin.serializers import AsyncOperationResponseSerializer -from pulpcore.plugin.models import Content from pulpcore.plugin.tasking import dispatch, general_multi_delete from pulpcore.plugin.util import ( extract_pk, @@ -25,21 +22,21 @@ raise_for_unknown_content_units, ) from pulpcore.plugin.viewsets import ( + NAME_FILTER_OPTIONS, AsyncUpdateMixin, - DistributionViewSet, BaseFilterSet, CharInFilter, ContentFilter, DistributionFilter, + DistributionViewSet, NamedModelViewSet, - NAME_FILTER_OPTIONS, + OperationPostponedResponse, ReadOnlyContentViewSet, ReadOnlyRepositoryViewSet, RemoteViewSet, - RepositoryViewSet, RepositoryVersionViewSet, + RepositoryViewSet, RolesMixin, - OperationPostponedResponse, ) from pulp_container.app import models, serializers, tasks diff --git a/pulp_container/tests/functional/api/test_build_images.py b/pulp_container/tests/functional/api/test_build_images.py index 243138a96..f751e9426 100644 --- a/pulp_container/tests/functional/api/test_build_images.py +++ b/pulp_container/tests/functional/api/test_build_images.py @@ -1,12 +1,12 @@ -import pytest - from tempfile import NamedTemporaryFile -from pulpcore.tests.functional.utils import PulpTaskError -from pulp_smash.pulp3.utils import gen_distribution +import pytest from pulp_smash.pulp3.bindings import monitor_task +from pulp_smash.pulp3.utils import gen_distribution from pulpcore.client.pulp_container import ApiException, ContainerContainerDistribution +from pulpcore.tests.functional.utils import PulpTaskError + from pulp_container.constants import MANIFEST_TYPE diff --git a/pulp_container/tests/functional/api/test_content_cache.py b/pulp_container/tests/functional/api/test_content_cache.py index 0c013f958..9162e3640 100644 --- a/pulp_container/tests/functional/api/test_content_cache.py +++ b/pulp_container/tests/functional/api/test_content_cache.py @@ -1,10 +1,10 @@ """Tests related to the Redis content caching.""" import os -import requests import unittest from urllib.parse import urljoin +import requests from pulp_smash import cli, config, utils from pulp_smash.pulp3.bindings import delete_orphans, monitor_task from pulp_smash.pulp3.utils import gen_distribution, gen_repo @@ -15,21 +15,19 @@ ContentManifestsApi, ContentTagsApi, DistributionsContainerApi, - RepositoriesContainerApi, - RemotesContainerApi, PatchedcontainerContainerDistribution, + RemotesContainerApi, + RepositoriesContainerApi, UnTagImage, ) +from pulp_container.constants import MEDIA_TYPE from pulp_container.tests.functional.utils import ( gen_container_client, gen_container_remote, get_auth_for_url, ) - -from pulp_container.constants import MEDIA_TYPE - cli_client = cli.Client(config.get_config()) CACHE_ENABLED = utils.get_pulp_setting(cli_client, "CACHE_ENABLED") diff --git a/pulp_container/tests/functional/api/test_crud_distributions.py b/pulp_container/tests/functional/api/test_crud_distributions.py index 87c6df952..df7b75d7c 100644 --- a/pulp_container/tests/functional/api/test_crud_distributions.py +++ b/pulp_container/tests/functional/api/test_crud_distributions.py @@ -2,18 +2,12 @@ import json import unittest - from itertools import permutations from pulp_smash import utils from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import gen_distribution -from pulp_container.tests.functional.utils import ( - skip_if, - gen_container_client, -) - from pulpcore.client.pulp_container import ( ApiException, ContainerContainerDistribution, @@ -21,6 +15,11 @@ PulpContainerNamespacesApi, ) +from pulp_container.tests.functional.utils import ( + gen_container_client, + skip_if, +) + class CRUDContainerDistributionsTestCase(unittest.TestCase): """CRUD distributions.""" diff --git a/pulp_container/tests/functional/api/test_crud_remotes.py b/pulp_container/tests/functional/api/test_crud_remotes.py index f17c5044e..c3f5b5940 100644 --- a/pulp_container/tests/functional/api/test_crud_remotes.py +++ b/pulp_container/tests/functional/api/test_crud_remotes.py @@ -1,21 +1,21 @@ """Tests that CRUD container remotes.""" -from random import choice import unittest +from random import choice from pulp_smash import utils from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.constants import ON_DEMAND_DOWNLOAD_POLICIES +from pulpcore.client.pulp_container import RemotesContainerApi +from pulpcore.client.pulp_container.exceptions import ApiException + from pulp_container.tests.functional.utils import ( gen_container_client, gen_container_remote, skip_if, ) -from pulpcore.client.pulp_container import RemotesContainerApi -from pulpcore.client.pulp_container.exceptions import ApiException - class CRUDRemotesTestCase(unittest.TestCase): """CRUD remotes.""" diff --git a/pulp_container/tests/functional/api/test_flatpak.py b/pulp_container/tests/functional/api/test_flatpak.py index 4c181a502..28bbce4b9 100644 --- a/pulp_container/tests/functional/api/test_flatpak.py +++ b/pulp_container/tests/functional/api/test_flatpak.py @@ -1,8 +1,9 @@ """Tests that verify Flatpak support""" -import pytest import subprocess +import pytest + from pulp_container.tests.functional.constants import REGISTRY_V2 diff --git a/pulp_container/tests/functional/api/test_pull_content.py b/pulp_container/tests/functional/api/test_pull_content.py index 18702a726..ebf17a953 100644 --- a/pulp_container/tests/functional/api/test_pull_content.py +++ b/pulp_container/tests/functional/api/test_pull_content.py @@ -2,44 +2,43 @@ import contextlib import hashlib -import requests import unittest import uuid - from urllib.parse import urljoin, urlparse +import requests from pulp_smash import api, cli, config, exceptions from pulp_smash.pulp3.bindings import delete_orphans, monitor_task from pulp_smash.pulp3.utils import ( - get_content, gen_distribution, gen_repo, + get_content, ) -from pulp_container.tests.functional.utils import ( - core_client, - gen_container_client, - gen_container_remote, - get_blobsums_from_remote_registry, - get_auth_for_url, -) -from pulp_container.tests.functional.constants import ( - CONTAINER_CONTENT_NAME, - REGISTRY_V2_REPO_HELLO_WORLD, - PULP_HELLO_WORLD_LINUX_TAG, -) -from pulp_container.constants import EMPTY_BLOB, EMPTY_JSON, MEDIA_TYPE - from pulpcore.client.pulp_container import ( ContainerContainerDistribution, ContainerContainerRepository, ContainerRepositorySyncURL, DistributionsContainerApi, - RepositoriesContainerApi, RemotesContainerApi, + RepositoriesContainerApi, ) from pulpcore.client.pulpcore import ArtifactsApi +from pulp_container.constants import EMPTY_BLOB, EMPTY_JSON, MEDIA_TYPE +from pulp_container.tests.functional.constants import ( + CONTAINER_CONTENT_NAME, + PULP_HELLO_WORLD_LINUX_TAG, + REGISTRY_V2_REPO_HELLO_WORLD, +) +from pulp_container.tests.functional.utils import ( + core_client, + gen_container_client, + gen_container_remote, + get_auth_for_url, + get_blobsums_from_remote_registry, +) + class PullContentTestCase(unittest.TestCase): """Verify whether images served by Pulp can be pulled.""" diff --git a/pulp_container/tests/functional/api/test_pull_through_cache.py b/pulp_container/tests/functional/api/test_pull_through_cache.py index 72a4f9b53..85b7fbd6b 100644 --- a/pulp_container/tests/functional/api/test_pull_through_cache.py +++ b/pulp_container/tests/functional/api/test_pull_through_cache.py @@ -1,19 +1,18 @@ -import time import subprocess -import pytest - -from django.conf import settings - +import time from subprocess import CalledProcessError from uuid import uuid4 +import pytest +from django.conf import settings + from pulp_container.constants import MANIFEST_TYPE from pulp_container.tests.functional.constants import ( - REGISTRY_V2, - PULP_HELLO_WORLD_REPO, - PULP_HELLO_WORLD_LINUX_AMD64_DIGEST, PULP_FIXTURE_1, PULP_FIXTURE_1_MANIFEST_A_DIGEST, + PULP_HELLO_WORLD_LINUX_AMD64_DIGEST, + PULP_HELLO_WORLD_REPO, + REGISTRY_V2, ) diff --git a/pulp_container/tests/functional/api/test_pulpimportexport.py b/pulp_container/tests/functional/api/test_pulpimportexport.py index 7e958663c..f0d66a10c 100644 --- a/pulp_container/tests/functional/api/test_pulpimportexport.py +++ b/pulp_container/tests/functional/api/test_pulpimportexport.py @@ -5,9 +5,9 @@ the case. """ -import pytest import uuid +import pytest from pulp_smash.pulp3.bindings import ( delete_orphans, monitor_task, @@ -17,8 +17,8 @@ from pulpcore.client.pulp_container import ContainerRepositorySyncURL -from pulp_container.tests.functional.utils import gen_container_remote from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP +from pulp_container.tests.functional.utils import gen_container_remote def test_import_export_standard( diff --git a/pulp_container/tests/functional/api/test_push_content.py b/pulp_container/tests/functional/api/test_push_content.py index 4033c277e..ad4b6d099 100644 --- a/pulp_container/tests/functional/api/test_push_content.py +++ b/pulp_container/tests/functional/api/test_push_content.py @@ -1,29 +1,18 @@ """Tests that verify that images can be pushed to Pulp.""" import json -import pytest -import requests import unittest - -from django.conf import settings - from subprocess import CalledProcessError -from urllib.parse import urlparse, urljoin +from urllib.parse import urljoin, urlparse +import pytest +import requests +from django.conf import settings from pulp_smash import cli, config from pulp_smash.pulp3.bindings import ( + PulpTestCase, delete_orphans, monitor_task, - PulpTestCase, -) - -from pulp_container.constants import MEDIA_TYPE, MANIFEST_TYPE - -from pulp_container.tests.functional.api import rbac_base -from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP -from pulp_container.tests.functional.utils import ( - gen_container_client, - get_auth_for_url, ) from pulpcore.client.pulp_container import ( @@ -34,6 +23,14 @@ RepositoriesContainerPushApi, ) +from pulp_container.constants import MANIFEST_TYPE, MEDIA_TYPE +from pulp_container.tests.functional.api import rbac_base +from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP +from pulp_container.tests.functional.utils import ( + gen_container_client, + get_auth_for_url, +) + def test_push_using_registry_client_admin( add_to_cleanup, diff --git a/pulp_container/tests/functional/api/test_push_signatures.py b/pulp_container/tests/functional/api/test_push_signatures.py index 5a31826f6..08c35735f 100644 --- a/pulp_container/tests/functional/api/test_push_signatures.py +++ b/pulp_container/tests/functional/api/test_push_signatures.py @@ -2,10 +2,11 @@ import base64 import json + import pytest -from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP from pulp_container.constants import SIGNATURE_TYPE +from pulp_container.tests.functional.constants import REGISTRY_V2_REPO_PULP @pytest.fixture diff --git a/pulp_container/tests/functional/api/test_rbac_push_repositories.py b/pulp_container/tests/functional/api/test_rbac_push_repositories.py index 915320a84..181e35467 100644 --- a/pulp_container/tests/functional/api/test_rbac_push_repositories.py +++ b/pulp_container/tests/functional/api/test_rbac_push_repositories.py @@ -1,9 +1,7 @@ """Tests that verify that RBAC for push repository works properly.""" import pytest - from django.conf import settings - from pulp_smash import utils from pulp_smash.pulp3.bindings import monitor_task diff --git a/pulp_container/tests/functional/api/test_rbac_remotes.py b/pulp_container/tests/functional/api/test_rbac_remotes.py index 1a5e03095..8d4dc80c2 100644 --- a/pulp_container/tests/functional/api/test_rbac_remotes.py +++ b/pulp_container/tests/functional/api/test_rbac_remotes.py @@ -1,18 +1,17 @@ """Tests that container remotes have RBAC.""" from random import choice -import pytest +import pytest from django.conf import settings - from pulp_smash import utils from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.constants import ON_DEMAND_DOWNLOAD_POLICIES -from pulp_container.tests.functional.utils import gen_container_remote - from pulpcore.client.pulp_container.exceptions import ApiException +from pulp_container.tests.functional.utils import gen_container_remote + @pytest.mark.parallel def test_rbac_remotes(gen_user, container_remote_api): diff --git a/pulp_container/tests/functional/api/test_rbac_repo_content.py b/pulp_container/tests/functional/api/test_rbac_repo_content.py index 2668ea36b..b83e53ee3 100644 --- a/pulp_container/tests/functional/api/test_rbac_repo_content.py +++ b/pulp_container/tests/functional/api/test_rbac_repo_content.py @@ -1,12 +1,14 @@ """Tests that verify that RBAC for content works properly.""" import pytest - from django.conf import settings - from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import gen_repo +from pulpcore.client.pulp_container import ( + ContainerContainerRepository, + ContainerRepositorySyncURL, +) from pulpcore.client.pulp_container.exceptions import ApiException from pulp_container.tests.functional.constants import PULP_FIXTURE_1, REGISTRY_V2_REPO_PULP @@ -14,11 +16,6 @@ gen_container_remote, ) -from pulpcore.client.pulp_container import ( - ContainerContainerRepository, - ContainerRepositorySyncURL, -) - def test_rbac_repository_content( delete_orphans_pre, diff --git a/pulp_container/tests/functional/api/test_rbac_repo_versions.py b/pulp_container/tests/functional/api/test_rbac_repo_versions.py index cbef66f98..3ad6d7acd 100644 --- a/pulp_container/tests/functional/api/test_rbac_repo_versions.py +++ b/pulp_container/tests/functional/api/test_rbac_repo_versions.py @@ -1,23 +1,20 @@ """Tests that verify that RBAC for repository versions work properly.""" import pytest - from django.conf import settings - from pulp_smash import utils from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import gen_repo -from pulpcore.client.pulp_container.exceptions import ApiException - -from pulp_container.tests.functional.constants import PULP_FIXTURE_1, REGISTRY_V2_REPO_PULP -from pulp_container.tests.functional.utils import TOKEN_AUTH_DISABLED, gen_container_remote - from pulpcore.client.pulp_container import ( ContainerContainerRepository, ContainerRepositorySyncURL, TagImage, ) +from pulpcore.client.pulp_container.exceptions import ApiException + +from pulp_container.tests.functional.constants import PULP_FIXTURE_1, REGISTRY_V2_REPO_PULP +from pulp_container.tests.functional.utils import TOKEN_AUTH_DISABLED, gen_container_remote def test_rbac_repository_version( diff --git a/pulp_container/tests/functional/api/test_rbac_sync_repositories.py b/pulp_container/tests/functional/api/test_rbac_sync_repositories.py index 6f7320867..cf97e12a6 100644 --- a/pulp_container/tests/functional/api/test_rbac_sync_repositories.py +++ b/pulp_container/tests/functional/api/test_rbac_sync_repositories.py @@ -1,9 +1,7 @@ """Tests that container sync repositories have RBAC.""" import pytest - from django.conf import settings - from pulp_smash import utils from pulp_smash.pulp3.bindings import monitor_task diff --git a/pulp_container/tests/functional/api/test_recursive_add.py b/pulp_container/tests/functional/api/test_recursive_add.py index 6c38345e2..c88d29131 100644 --- a/pulp_container/tests/functional/api/test_recursive_add.py +++ b/pulp_container/tests/functional/api/test_recursive_add.py @@ -1,20 +1,12 @@ """Tests that recursively add container content to repositories.""" from pulp_smash.pulp3.bindings import ( + PulpTestCase, delete_orphans, monitor_task, - PulpTestCase, ) from pulp_smash.pulp3.utils import gen_repo -from pulp_container.tests.functional.utils import ( - gen_container_remote, - gen_container_client, -) -from pulp_container.tests.functional.constants import PULP_FIXTURE_1 - -from pulp_container.constants import MEDIA_TYPE - from pulpcore.client.pulp_container import ( ApiException, ContainerContainerRemote, @@ -27,6 +19,13 @@ RepositoriesContainerVersionsApi, ) +from pulp_container.constants import MEDIA_TYPE +from pulp_container.tests.functional.constants import PULP_FIXTURE_1 +from pulp_container.tests.functional.utils import ( + gen_container_client, + gen_container_remote, +) + class TestManifestCopy(PulpTestCase): """ diff --git a/pulp_container/tests/functional/api/test_recursive_remove.py b/pulp_container/tests/functional/api/test_recursive_remove.py index 4c7b8c69e..4757d0d59 100644 --- a/pulp_container/tests/functional/api/test_recursive_remove.py +++ b/pulp_container/tests/functional/api/test_recursive_remove.py @@ -1,24 +1,16 @@ """Tests that recursively remove container content from repositories.""" import unittest - from urllib.parse import urlparse from pulp_smash import cli, config from pulp_smash.pulp3.bindings import ( + PulpTestCase, delete_orphans, monitor_task, - PulpTestCase, ) from pulp_smash.pulp3.utils import gen_repo -from pulp_container.tests.functional.api import rbac_base -from pulp_container.tests.functional.utils import ( - gen_container_remote, - gen_container_client, -) -from pulp_container.tests.functional.constants import PULP_FIXTURE_1, REGISTRY_V2_REPO_PULP - from pulpcore.client.pulp_container import ( ApiException, ContainerContainerRemote, @@ -28,14 +20,21 @@ ContentTagsApi, DistributionsContainerApi, PulpContainerNamespacesApi, - RemoveImage, RemotesContainerApi, + RemoveImage, RepositoriesContainerApi, RepositoriesContainerPushApi, RepositoriesContainerVersionsApi, TagImage, ) +from pulp_container.tests.functional.api import rbac_base +from pulp_container.tests.functional.constants import PULP_FIXTURE_1, REGISTRY_V2_REPO_PULP +from pulp_container.tests.functional.utils import ( + gen_container_client, + gen_container_remote, +) + class TestRecursiveRemove(PulpTestCase): """ diff --git a/pulp_container/tests/functional/api/test_remote_filter_pull_through.py b/pulp_container/tests/functional/api/test_remote_filter_pull_through.py index 7c09f8797..32642e37a 100644 --- a/pulp_container/tests/functional/api/test_remote_filter_pull_through.py +++ b/pulp_container/tests/functional/api/test_remote_filter_pull_through.py @@ -1,10 +1,11 @@ -import pytest import subprocess +import pytest + from pulp_container.tests.functional.constants import ( - REGISTRY_V2, - PULP_HELLO_WORLD_REPO, PULP_FIXTURE_1, + PULP_HELLO_WORLD_REPO, + REGISTRY_V2, ) diff --git a/pulp_container/tests/functional/api/test_repositories_list.py b/pulp_container/tests/functional/api/test_repositories_list.py index 9f092d001..1417bc6dc 100644 --- a/pulp_container/tests/functional/api/test_repositories_list.py +++ b/pulp_container/tests/functional/api/test_repositories_list.py @@ -1,34 +1,32 @@ """Tests for fetching the list of all repositories.""" import unittest - from urllib.parse import urljoin -import requests +import requests from pulp_smash import api, cli, config, utils from pulp_smash.pulp3.bindings import delete_orphans, monitor_task from pulp_smash.pulp3.utils import gen_distribution, gen_repo -from pulp_container.tests.functional.constants import PULP_FIXTURE_1 - -from pulp_container.tests.functional.utils import ( - del_user, - gen_container_remote, - gen_container_client, - gen_user, - BearerTokenAuth, - AuthenticationHeaderQueries, -) - from pulpcore.client.pulp_container import ( - ContainerContainerRepository, ContainerContainerDistribution, ContainerContainerRemote, + ContainerContainerRepository, ContainerRepositorySyncURL, DistributionsContainerApi, PulpContainerNamespacesApi, - RepositoriesContainerApi, RemotesContainerApi, + RepositoriesContainerApi, +) + +from pulp_container.tests.functional.constants import PULP_FIXTURE_1 +from pulp_container.tests.functional.utils import ( + AuthenticationHeaderQueries, + BearerTokenAuth, + del_user, + gen_container_client, + gen_container_remote, + gen_user, ) cli_client = cli.Client(config.get_config()) diff --git a/pulp_container/tests/functional/api/test_sign_manifests.py b/pulp_container/tests/functional/api/test_sign_manifests.py index 225597c80..755e17335 100644 --- a/pulp_container/tests/functional/api/test_sign_manifests.py +++ b/pulp_container/tests/functional/api/test_sign_manifests.py @@ -1,5 +1,4 @@ import pytest - from pulp_smash.pulp3.bindings import monitor_task from pulpcore.client.pulp_container import RepositorySign diff --git a/pulp_container/tests/functional/api/test_sync.py b/pulp_container/tests/functional/api/test_sync.py index f36f7fe23..32dba7c5d 100644 --- a/pulp_container/tests/functional/api/test_sync.py +++ b/pulp_container/tests/functional/api/test_sync.py @@ -1,13 +1,14 @@ """Tests that sync container plugin repositories.""" import pytest + from pulpcore.tests.functional.utils import PulpTaskError -from pulp_container.constants import MEDIA_TYPE, MANIFEST_TYPE +from pulp_container.constants import MANIFEST_TYPE, MEDIA_TYPE from pulp_container.tests.functional.constants import ( PULP_FIXTURE_1, - PULP_LABELED_FIXTURE, PULP_HELLO_WORLD_LINUX_AMD64_DIGEST, + PULP_LABELED_FIXTURE, REGISTRY_V2_FEED_URL, ) diff --git a/pulp_container/tests/functional/api/test_sync_signatures.py b/pulp_container/tests/functional/api/test_sync_signatures.py index 42dc4238c..e986ae0e7 100644 --- a/pulp_container/tests/functional/api/test_sync_signatures.py +++ b/pulp_container/tests/functional/api/test_sync_signatures.py @@ -1,5 +1,4 @@ import pytest - from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import gen_repo @@ -7,6 +6,7 @@ ContainerContainerRepository, ContainerRepositorySyncURL, ) + from pulp_container.tests.functional.utils import gen_container_remote REDHAT_REGISTRY_V2 = "https://registry.access.redhat.com" diff --git a/pulp_container/tests/functional/api/test_tagging_images.py b/pulp_container/tests/functional/api/test_tagging_images.py index a8ecc541d..774b3c273 100644 --- a/pulp_container/tests/functional/api/test_tagging_images.py +++ b/pulp_container/tests/functional/api/test_tagging_images.py @@ -1,23 +1,12 @@ """Tests for tagging and untagging images.""" import unittest - from urllib.parse import urlparse from pulp_smash import cli, config, exceptions from pulp_smash.pulp3.bindings import delete_orphans, monitor_task from pulp_smash.pulp3.utils import gen_repo -from pulp_container.tests.functional.utils import ( - gen_container_remote, - gen_container_client, -) -from pulp_container.tests.functional.constants import ( - CONTAINER_TAG_PATH, - PULP_FIXTURE_1, - REGISTRY_V2_REPO_PULP, -) - from pulpcore.client.pulp_container import ( ApiException, ContainerContainerRemote, @@ -25,14 +14,24 @@ ContainerRepositorySyncURL, ContentManifestsApi, ContentTagsApi, + RemotesContainerApi, RepositoriesContainerApi, RepositoriesContainerPushApi, RepositoriesContainerVersionsApi, - RemotesContainerApi, TagImage, UnTagImage, ) +from pulp_container.tests.functional.constants import ( + CONTAINER_TAG_PATH, + PULP_FIXTURE_1, + REGISTRY_V2_REPO_PULP, +) +from pulp_container.tests.functional.utils import ( + gen_container_client, + gen_container_remote, +) + class TaggingTestCommons: """Common utilities for tagging and untagging images.""" diff --git a/pulp_container/tests/functional/api/test_token_authentication.py b/pulp_container/tests/functional/api/test_token_authentication.py index d142b5241..cfdb5d05d 100644 --- a/pulp_container/tests/functional/api/test_token_authentication.py +++ b/pulp_container/tests/functional/api/test_token_authentication.py @@ -1,35 +1,34 @@ """Tests for token authentication.""" -import aiohttp import asyncio import unittest - from urllib.parse import urljoin, urlparse -import requests -from pulp_smash import api, config, cli +import aiohttp +import requests +from pulp_smash import api, cli, config from pulp_smash.pulp3.bindings import delete_orphans, monitor_task -from pulp_smash.pulp3.utils import gen_repo, gen_distribution - -from pulp_container.tests.functional.utils import ( - gen_container_remote, - gen_container_client, - get_auth_for_url, -) -from pulp_container.tests.functional.constants import ( - CONTAINER_TAG_PATH, - PULP_FIXTURE_1, -) -from pulp_container.constants import MEDIA_TYPE +from pulp_smash.pulp3.utils import gen_distribution, gen_repo from pulpcore.client.pulp_container import ( ContainerContainerDistribution, - ContainerContainerRepository, ContainerContainerRemote, + ContainerContainerRepository, ContainerRepositorySyncURL, DistributionsContainerApi, - RepositoriesContainerApi, RemotesContainerApi, + RepositoriesContainerApi, +) + +from pulp_container.constants import MEDIA_TYPE +from pulp_container.tests.functional.constants import ( + CONTAINER_TAG_PATH, + PULP_FIXTURE_1, +) +from pulp_container.tests.functional.utils import ( + gen_container_client, + gen_container_remote, + get_auth_for_url, ) diff --git a/pulp_container/tests/functional/conftest.py b/pulp_container/tests/functional/conftest.py index a342d3749..879188612 100644 --- a/pulp_container/tests/functional/conftest.py +++ b/pulp_container/tests/functional/conftest.py @@ -1,41 +1,40 @@ import json import os import stat -import pytest -import requests import subprocess - from contextlib import contextmanager, suppress from urllib.parse import urljoin, urlparse from uuid import uuid4 +import pytest +import requests + from pulpcore.client.pulp_container import ( ApiClient, + ContainerContainerRepository, + ContainerRepositorySyncURL, + ContentBlobsApi, + ContentManifestsApi, + ContentSignaturesApi, + ContentTagsApi, + DistributionsContainerApi, + DistributionsPullThroughApi, PulpContainerNamespacesApi, RemotesContainerApi, RemotesPullThroughApi, RepositoriesContainerApi, RepositoriesContainerPushApi, - RepositoriesContainerVersionsApi, RepositoriesContainerPushVersionsApi, - DistributionsContainerApi, - DistributionsPullThroughApi, - ContentTagsApi, - ContentManifestsApi, - ContentBlobsApi, - ContentSignaturesApi, - ContainerContainerRepository, - ContainerRepositorySyncURL, + RepositoriesContainerVersionsApi, ) +from pulp_container.tests.functional.constants import PULP_HELLO_WORLD_REPO, REGISTRY_V2_FEED_URL from pulp_container.tests.functional.utils import ( TOKEN_AUTH_DISABLED, AuthenticationHeaderQueries, BearerTokenAuth, ) -from pulp_container.tests.functional.constants import REGISTRY_V2_FEED_URL, PULP_HELLO_WORLD_REPO - def gen_container_remote(url=REGISTRY_V2_FEED_URL, **kwargs): """Return a semi-random dict for use in creating a container Remote. diff --git a/pulp_container/tests/functional/constants.py b/pulp_container/tests/functional/constants.py index 1aaeb6963..9d9aa313b 100644 --- a/pulp_container/tests/functional/constants.py +++ b/pulp_container/tests/functional/constants.py @@ -2,10 +2,10 @@ from pulp_smash.constants import PULP_FIXTURES_BASE_URL from pulp_smash.pulp3.constants import ( + BASE_CONTENT_PATH, BASE_DISTRIBUTION_PATH, BASE_REMOTE_PATH, BASE_REPO_PATH, - BASE_CONTENT_PATH, ) CONTAINER_MANIFEST_PATH = urljoin(BASE_CONTENT_PATH, "container/manifests/") diff --git a/pulp_container/tests/functional/utils.py b/pulp_container/tests/functional/utils.py index 0e2c6c720..44fd71e14 100644 --- a/pulp_container/tests/functional/utils.py +++ b/pulp_container/tests/functional/utils.py @@ -1,48 +1,51 @@ """Utilities for tests for the container plugin.""" -import pytest -import requests - -from requests.auth import AuthBase from functools import partial -from unittest import SkipTest from tempfile import NamedTemporaryFile +from unittest import SkipTest from uuid import uuid4 +import pytest +import requests from pulp_smash import cli, config, selectors, utils from pulp_smash.pulp3.bindings import monitor_task from pulp_smash.pulp3.utils import ( get_content, ) +from requests.auth import AuthBase -from pulp_container.tests.functional.constants import ( - CONTAINER_CONTENT_NAME, - CONTAINER_IMAGE_URL, - PULP_HELLO_WORLD_REPO, - REGISTRY_V2_FEED_URL, -) - -from pulpcore.client.pulpcore import ( - ApiClient as CoreApiClient, - ArtifactsApi, - GroupsApi, - GroupsUsersApi, - UsersApi, - UsersRolesApi, -) from pulpcore.client.pulp_container import ( ApiClient as ContainerApiClient, +) +from pulpcore.client.pulp_container import ( ContainerRepositorySyncURL, ContentBlobsApi, ContentManifestsApi, ContentTagsApi, - RemotesContainerApi, DistributionsContainerApi, PulpContainerNamespacesApi, + RemotesContainerApi, RepositoriesContainerApi, RepositoriesContainerPushApi, RepositoriesContainerVersionsApi, ) +from pulpcore.client.pulpcore import ( + ApiClient as CoreApiClient, +) +from pulpcore.client.pulpcore import ( + ArtifactsApi, + GroupsApi, + GroupsUsersApi, + UsersApi, + UsersRolesApi, +) + +from pulp_container.tests.functional.constants import ( + CONTAINER_CONTENT_NAME, + CONTAINER_IMAGE_URL, + PULP_HELLO_WORLD_REPO, + REGISTRY_V2_FEED_URL, +) cfg = config.get_config() cli_client = cli.Client(cfg) diff --git a/pulp_container/tests/unit/test_json_schemas.py b/pulp_container/tests/unit/test_json_schemas.py index 38dba5d54..a89c68897 100644 --- a/pulp_container/tests/unit/test_json_schemas.py +++ b/pulp_container/tests/unit/test_json_schemas.py @@ -1,13 +1,12 @@ import json -from jsonschema import Draft7Validator - from django.test import TestCase +from jsonschema import Draft7Validator -from pulp_container.constants import MEDIA_TYPE from pulp_container.app.exceptions import ManifestInvalid from pulp_container.app.json_schemas import SIGNATURE_SCHEMA from pulp_container.app.utils import validate_manifest +from pulp_container.constants import MEDIA_TYPE validator = Draft7Validator(SIGNATURE_SCHEMA) diff --git a/pulp_container/tests/unit/test_serializers.py b/pulp_container/tests/unit/test_serializers.py index 7cdbbd493..334b7e9c7 100644 --- a/pulp_container/tests/unit/test_serializers.py +++ b/pulp_container/tests/unit/test_serializers.py @@ -1,11 +1,11 @@ +from django.conf import settings from django.contrib.auth import get_user_model from django.test import TestCase -from django.conf import settings from pulpcore.plugin.util import set_current_user -from pulp_container.app.serializers import ContainerDistributionSerializer, TagOperationSerializer from pulp_container.app.models import ContainerPushRepository, ContainerRepository +from pulp_container.app.serializers import ContainerDistributionSerializer, TagOperationSerializer V3_API_ROOT = settings.V3_API_ROOT diff --git a/pyproject.toml b/pyproject.toml index e463456fd..3cfcb1ed2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,3 +98,30 @@ replace = "version = \"{new_version}\"" [[tool.bumpversion.files]] filename = "./setup.py" + + +[tool.ruff] +# This section is managed by the plugin template. Do not edit manually. +line-length = 100 +extend-exclude = [ + "docs/**", + "**/migrations/*.py", +] + +[tool.ruff.lint] +# This section is managed by the plugin template. Do not edit manually. +extend-select = [ + "I", + "INT", + "TID", + "T10", +] + +[tool.ruff.lint.flake8-tidy-imports.banned-api] +# This section is managed by the plugin template. Do not edit manually. +"pulpcore.app".msg = "The 'pulpcore' apis must only be consumed via 'pulpcore.plugin'." + +[tool.ruff.lint.isort] +# This section is managed by the plugin template. Do not edit manually. +sections = { second-party = ["pulpcore"] } +section-order = ["future", "standard-library", "third-party", "second-party", "first-party", "local-folder"] diff --git a/template_config.yml b/template_config.yml index a38fa2346..23a2ce68c 100644 --- a/template_config.yml +++ b/template_config.yml @@ -6,10 +6,7 @@ # After editing this file please always reapply the plugin template before committing any changes. --- -api_root: "/pulp/" -black: true check_commit_message: true -check_gettext: true check_manifest: true check_stray_pulpcore_imports: true ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" @@ -24,10 +21,9 @@ deploy_to_pypi: true disabled_redis_runners: [] docker_fixtures: false extra_files: [] -flake8: true -flake8_ignore: [] github_org: "pulp" latest_release_branch: "2.21" +lint_ignore: [] lint_requirements: true os_required_packages: [] parallel_test_workers: 8 @@ -53,6 +49,7 @@ pulp_settings: - "/tmp" allowed_import_paths: - "/tmp" + api_root: "/pulp/" flatpak_index: true pulp_settings_azure: MEDIA_ROOT: "" @@ -87,9 +84,9 @@ pulp_settings_s3: signature_version: "s3v4" staticfiles: BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" + api_root: "/rerouted/djnd/" flatpak_index: false token_auth_disabled: true -pydocstyle: true release_email: "pulp-infra@redhat.com" release_user: "pulpbot" stalebot: true