pyxis: allow unprivileged user namespaces so enroot works on Ubuntu 23.10+

roles/pyxis/defaults/main.yml

@@ -18,3 +18,10 @@ resize_run_partition: false
 
 # /run tmpfs size. ubuntu default is 10% of physical memory
 pyxis_run_tmpfs_size: 50%
+
+# Ubuntu 23.10 and later restrict unprivileged user namespaces through AppArmor,
+# which blocks enroot-nsenter. The role installs an AppArmor profile that grants
+# the capability to enroot-nsenter alone. Set this to true only if that profile
+# cannot be used on your installation: it clears the restriction for every
+# process on the host and lowers the security default of the OS.
+pyxis_userns_allow_globally: false

roles/pyxis/files/etc/apparmor.d/enroot-nsenter

@@ -0,0 +1,21 @@
+# enroot-nsenter creates the user namespace that a pyxis container runs in.
+# Ubuntu 23.10 and later ship kernel.apparmor_restrict_unprivileged_userns=1,
+# which makes that fail with:
+#
+#   failed to create user namespace: Permission denied
+#
+# This profile grants the userns capability to that single executable instead
+# of clearing the restriction for every process on the host. enroot-nsenter is
+# the only enroot executable that references CLONE_NEWUSER (enroot 3.2.0), so
+# the other helpers do not need it.
+#
+# The "userns" rule and the "unconfined" flag require AppArmor 4.0 or later,
+# which is present on every release that enables the restriction.
+abi <abi/4.0>,
+include <tunables/global>
+
+profile enroot-nsenter /usr/bin/enroot-nsenter flags=(unconfined) {
+  userns,
+
+  include if exists <local/enroot-nsenter>
+}

roles/pyxis/tasks/main.yml

@@ -13,6 +13,138 @@
   with_items: "{{ pyxis_el_deps }}"
   when: ansible_os_family == "RedHat"
 
+# Ubuntu 23.10 and later set kernel.apparmor_restrict_unprivileged_userns=1
+# by default, which makes enroot-nsenter fail under pyxis with
+# "failed to create user namespace: Permission denied". Detect the knob by
+# its presence in /proc rather than by distribution version, so these tasks are
+# a no-op on kernels built without it.
+- name: check for AppArmor unprivileged user namespace restriction
+  stat:
+    path: /proc/sys/kernel/apparmor_restrict_unprivileged_userns
+  register: apparmor_userns_knob
+  when: is_compute
+
+# The profile is the default path, so its userspace is a dependency of this
+# role rather than something to skip over when it is missing.
+- name: install the AppArmor userspace (Ubuntu)
+  apt:
+    name: apparmor
+    state: present
+  when:
+    - is_compute
+    - ansible_distribution == "Ubuntu"
+    - apparmor_userns_knob.stat.exists | default(false)
+    - not pyxis_userns_allow_globally
+
+- name: check for apparmor_parser
+  stat:
+    path: /sbin/apparmor_parser
+  register: apparmor_parser_bin
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+
+# Without this the scoped path is skipped silently and enroot keeps failing
+# with the very error this role exists to prevent.
+- name: fail when the AppArmor userspace is unavailable
+  fail:
+    msg: >-
+      kernel.apparmor_restrict_unprivileged_userns is active but
+      /sbin/apparmor_parser is not present, so the scoped enroot-nsenter
+      profile cannot be loaded and enroot would fail with "failed to create
+      user namespace: Permission denied". Install the AppArmor userspace, or
+      set pyxis_userns_allow_globally=true to clear the restriction for the
+      whole host instead.
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+    - not pyxis_userns_allow_globally
+    - not apparmor_parser_bin.stat.exists | default(false)
+
+# Grant the capability to the one executable that needs it rather than clearing
+# the restriction for the whole host.
+- name: install the enroot-nsenter AppArmor profile
+  copy:
+    src: etc/apparmor.d/enroot-nsenter
+    dest: /etc/apparmor.d/enroot-nsenter
+    owner: root
+    group: root
+    mode: "0644"
+  register: enroot_userns_profile
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+    - not pyxis_userns_allow_globally
+
+# Loading only on a file change leaves a node unrepaired when the first load
+# failed, or when the profile was unloaded by hand while the file stayed as it
+# was. The kernel's own list decides, so a rerun converges either way.
+- name: check whether the enroot-nsenter profile is loaded
+  command: grep -q "^enroot-nsenter " /sys/kernel/security/apparmor/profiles
+  register: enroot_userns_loaded
+  changed_when: false
+  failed_when: false
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+    - not pyxis_userns_allow_globally
+
+- name: load the enroot-nsenter AppArmor profile
+  command: apparmor_parser -r /etc/apparmor.d/enroot-nsenter
+  register: enroot_userns_load
+  failed_when: false
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+    - not pyxis_userns_allow_globally
+    - (enroot_userns_profile.changed | default(false))
+      or (enroot_userns_loaded.rc | default(1) != 0)
+
+- name: report a failure to load the enroot-nsenter AppArmor profile
+  fail:
+    msg: >-
+      Could not load /etc/apparmor.d/enroot-nsenter:
+      {{ enroot_userns_load.stderr | default('') }}.
+      The "userns" rule and the "unconfined" flag need AppArmor 4.0 or later.
+      Set pyxis_userns_allow_globally=true to clear
+      kernel.apparmor_restrict_unprivileged_userns for the whole host instead.
+  when: enroot_userns_load.rc | default(0) != 0
+
+# Opt-in fallback for installations where the profile cannot be used. This
+# lowers the security default for every process on the host, so it is off by
+# default and has to be requested explicitly.
+- name: allow unprivileged user namespaces host-wide
+  ansible.posix.sysctl:
+    name: kernel.apparmor_restrict_unprivileged_userns
+    value: "0"
+    sysctl_file: /etc/sysctl.d/60-enroot-userns.conf
+    state: present
+    reload: yes
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+    - pyxis_userns_allow_globally
+
+# Flipping the fallback back off has to take its setting with it, otherwise the
+# host keeps the lowered default that the profile was meant to replace.
+# "sysctl --system" reapplies the remaining files, so the value returns to what
+# the distribution ships rather than to one this role invents.
+- name: drop the host-wide user namespace sysctl when the fallback is off
+  ansible.posix.sysctl:
+    name: kernel.apparmor_restrict_unprivileged_userns
+    sysctl_file: /etc/sysctl.d/60-enroot-userns.conf
+    state: absent
+  register: enroot_userns_sysctl_removed
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+    - not pyxis_userns_allow_globally
+
+- name: reapply the remaining sysctl configuration
+  command: sysctl --system
+  changed_when: true
+  when: enroot_userns_sysctl_removed.changed | default(false)
+
 - name: install slurm-pmi hook
   file:
     path: /etc/enroot/hooks.d/50-slurm-pmi.sh