pyxis: allow unprivileged user namespaces so enroot works on Ubuntu 23.10+

roles/pyxis/defaults/main.yml

@@ -18,3 +18,10 @@ resize_run_partition: false
 
 # /run tmpfs size. ubuntu default is 10% of physical memory
 pyxis_run_tmpfs_size: 50%
+
+# Ubuntu 23.10 and later restrict unprivileged user namespaces through AppArmor,
+# which blocks enroot-nsenter. The role installs an AppArmor profile that grants
+# the capability to enroot-nsenter alone. Set this to true only if that profile
+# cannot be used on your installation: it clears the restriction for every
+# process on the host and lowers the security default of the OS.
+pyxis_userns_allow_globally: false

roles/pyxis/files/etc/apparmor.d/enroot-nsenter

@@ -0,0 +1,21 @@
+# enroot-nsenter creates the user namespace that a pyxis container runs in.
+# Ubuntu 23.10 and later ship kernel.apparmor_restrict_unprivileged_userns=1,
+# which makes that fail with:
+#
+#   failed to create user namespace: Permission denied
+#
+# This profile grants the userns capability to that single executable instead
+# of clearing the restriction for every process on the host. enroot-nsenter is
+# the only enroot executable that references CLONE_NEWUSER (enroot 3.2.0), so
+# the other helpers do not need it.
+#
+# The "userns" rule and the "unconfined" flag require AppArmor 4.0 or later,
+# which is present on every release that enables the restriction.
+abi <abi/4.0>,
+include <tunables/global>
+
+profile enroot-nsenter /usr/bin/enroot-nsenter flags=(unconfined) {
+  userns,
+
+  include if exists <local/enroot-nsenter>
+}

roles/pyxis/tasks/main.yml

@@ -13,6 +13,72 @@
   with_items: "{{ pyxis_el_deps }}"
   when: ansible_os_family == "RedHat"
 
+# Ubuntu 23.10 and later set kernel.apparmor_restrict_unprivileged_userns=1
+# by default, which makes enroot-nsenter fail under pyxis with
+# "failed to create user namespace: Permission denied". Detect the knob by
+# its presence in /proc rather than by distribution version, so these tasks are
+# a no-op on kernels built without it.
+- name: check for AppArmor unprivileged user namespace restriction
+  stat:
+    path: /proc/sys/kernel/apparmor_restrict_unprivileged_userns
+  register: apparmor_userns_knob
+  when: is_compute
+
+- name: check for apparmor_parser
+  stat:
+    path: /sbin/apparmor_parser
+  register: apparmor_parser_bin
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+
+# Grant the capability to the one executable that needs it rather than clearing
+# the restriction for the whole host.
+- name: install the enroot-nsenter AppArmor profile
+  copy:
+    src: etc/apparmor.d/enroot-nsenter
+    dest: /etc/apparmor.d/enroot-nsenter
+    owner: root
+    group: root
+    mode: "0644"
+  register: enroot_userns_profile
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+    - apparmor_parser_bin.stat.exists | default(false)
+    - not pyxis_userns_allow_globally
+
+- name: load the enroot-nsenter AppArmor profile
+  command: apparmor_parser -r /etc/apparmor.d/enroot-nsenter
+  register: enroot_userns_load
+  failed_when: false
+  when: enroot_userns_profile.changed | default(false)
+
+- name: report a failure to load the enroot-nsenter AppArmor profile
+  fail:
+    msg: >-
+      Could not load /etc/apparmor.d/enroot-nsenter:
+      {{ enroot_userns_load.stderr | default('') }}.
+      The "userns" rule and the "unconfined" flag need AppArmor 4.0 or later.
+      Set pyxis_userns_allow_globally=true to clear
+      kernel.apparmor_restrict_unprivileged_userns for the whole host instead.
+  when: enroot_userns_load.rc | default(0) != 0
+
+# Opt-in fallback for installations where the profile cannot be used. This
+# lowers the security default for every process on the host, so it is off by
+# default and has to be requested explicitly.
+- name: allow unprivileged user namespaces host-wide
+  ansible.posix.sysctl:
+    name: kernel.apparmor_restrict_unprivileged_userns
+    value: "0"
+    sysctl_file: /etc/sysctl.d/60-enroot-userns.conf
+    state: present
+    reload: yes
+  when:
+    - is_compute
+    - apparmor_userns_knob.stat.exists | default(false)
+    - pyxis_userns_allow_globally
+
 - name: install slurm-pmi hook
   file:
     path: /etc/enroot/hooks.d/50-slurm-pmi.sh