fix(kyber): preserve host recovery through SMART failures

config/k3s/kyber-host-health.sh

@@ -306,9 +306,37 @@ manage_orchestration_circuit_breaker() {
   fi
 }
 
+check_disk_wear_device() {
+  local disk="$1"
+  local name output remaining status=0
+
+  name="$(basename "$disk")"
+  # SMART health warnings are exit-status bits, not necessarily read failures.
+  # Capture the status separately so a worn disk cannot abort host recovery.
+  output="$(timeout --kill-after=2 10 smartctl -A "$disk" 2>/dev/null)" || status=$?
+  if [ $((status & 7)) -ne 0 ]; then
+    set_alert "disk-wear-read-$name" "unable to read SMART attributes for $name (status $status); previous wear warning retained"
+    return
+  fi
+
+  # Samsung reports remaining endurance as the normalized VALUE, not RAW_VALUE.
+  remaining="$(printf '%s\n' "$output" | awk '$2 == "Wear_Leveling_Count" { print $4; exit }')"
+  if [[ ! $remaining =~ ^[0-9]{1,3}$ ]] || [ "$((10#${remaining:-0}))" -gt 100 ]; then
+    set_alert "disk-wear-read-$name" "no valid remaining-endurance attribute for $name; previous wear warning retained"
+    return
+  fi
+  remaining=$((10#$remaining))
+  clear_alert "disk-wear-read-$name"
+  if [ "$remaining" -le "$DISK_WEAR_WARNING_PERCENT" ]; then
+    set_alert "disk-wear-$name" "$name has ${remaining}% of its rated write endurance left; schedule a replacement before moving write-heavy state onto it"
+  else
+    clear_alert "disk-wear-$name"
+  fi
+}
+
 check_disk_wear() {
   local stamp="$STATE_DIR/disk-wear.checked"
-  local now last=0 disk name remaining
+  local now last=0 disk
 
   now="$(date +%s)"
   if [ -r "$stamp" ]; then
@@ -321,23 +349,7 @@ check_disk_wear() {
 
   for disk in /dev/sd[a-z]; do
     [ -b "$disk" ] || continue
-    name="$(basename "$disk")"
-    # Samsung publishes remaining write endurance as the normalized value of
-    # Wear_Leveling_Count. smartd's -f only fires once that reaches the vendor
-    # threshold of zero, which is after the rated endurance is already spent
-    # and far too late to schedule a replacement.
-    # shellcheck disable=SC2016
-    remaining="$(smartctl -A "$disk" 2>/dev/null |
-      awk '$2 == "Wear_Leveling_Count" { print $4 + 0; exit }')"
-    if [ -z "$remaining" ]; then
-      continue
-    fi
-
-    if [ "$remaining" -le "$DISK_WEAR_WARNING_PERCENT" ]; then
-      set_alert "disk-wear-$name" "$name has ${remaining}% of its rated write endurance left; schedule a replacement before moving write-heavy state onto it"
-    else
-      clear_alert "disk-wear-$name"
-    fi
+    check_disk_wear_device "$disk"
   done
 }
 
@@ -347,10 +359,10 @@ main() {
   check_d_state
   check_node_filesystem
   check_image_filesystem
-  check_disk_wear
   check_cri
   manage_orchestration_circuit_breaker
   check_dns
+  check_disk_wear
 }
 
 if [[ ${BASH_SOURCE[0]} == "$0" ]]; then

spec/activate_kyber_spec.sh

@@ -327,4 +327,84 @@ It 'runs the wear check as part of main'
 When run bash -c "awk '/^main\(\)/,/^}/' '$SCRIPT' | grep -q check_disk_wear"
 The status should be success
 End
+
+It 'handles SMART health status bits without aborting the remaining checks'
+When run env HEALTH_CHECK="$SCRIPT" bash -c '
+  source "$HEALTH_CHECK"
+  set_alert() { printf "alert:%s:%s\n" "$1" "$2"; }
+  clear_alert() { printf "clear:%s\n" "$1"; }
+  timeout() {
+    test "$*" = "--kill-after=2 10 smartctl -A /dev/sda"
+    printf "177 Wear_Leveling_Count 0x0013 009 009 000 Pre-fail Always - 12345\n"
+    return "$smart_status"
+  }
+  for smart_status in 0 8 16 32 64 128 248; do
+    check_disk_wear_device /dev/sda
+  done
+  printf "continued\n"
+'
+The status should be success
+The output should include 'sda has 9% of its rated write endurance left'
+The output should include 'continued'
+The output should not include 'clear:disk-wear-sda'
+End
+
+It 'preserves wear warnings when reads fail, time out, or return malformed values'
+When run env HEALTH_CHECK="$SCRIPT" bash -c '
+  source "$HEALTH_CHECK"
+  set_alert() { printf "alert:%s\n" "$1"; }
+  clear_alert() { printf "unexpected-clear:%s\n" "$1"; }
+  timeout() {
+    printf "177 Wear_Leveling_Count 0x0013 %s 009 000 Pre-fail Always - 12345\n" "$smart_value"
+    return "$smart_status"
+  }
+  smart_value=090
+  for smart_status in 1 2 4 124 137; do
+    check_disk_wear_device /dev/sda
+  done
+  smart_status=0
+  for smart_value in missing 101 1.5 -1 0000; do
+    check_disk_wear_device /dev/sda
+  done
+  timeout() { return 0; }
+  check_disk_wear_device /dev/sda
+  printf "continued\n"
+'
+The status should be success
+The output should include 'alert:disk-wear-read-sda'
+The output should include 'continued'
+The output should not include 'unexpected-clear:'
+End
+
+It 'clears read and wear alerts only after a valid healthy measurement'
+When run env HEALTH_CHECK="$SCRIPT" bash -c '
+  source "$HEALTH_CHECK"
+  set_alert() { printf "unexpected-alert:%s\n" "$1"; }
+  clear_alert() { printf "clear:%s\n" "$1"; }
+  timeout() { printf "177 Wear_Leveling_Count 0x0013 090 090 000 Pre-fail Always - 12345\n"; }
+  check_disk_wear_device /dev/sda
+'
+The status should be success
+The output should include 'clear:disk-wear-read-sda'
+The output should include 'clear:disk-wear-sda'
+The output should not include 'unexpected-alert:'
+End
+
+It 'runs CRI and freezer recovery before probing disk endurance'
+When run env HEALTH_CHECK="$SCRIPT" bash -c '
+  source "$HEALTH_CHECK"
+  install() { :; }
+  check_io_pressure() { :; }
+  check_d_state() { :; }
+  check_node_filesystem() { :; }
+  check_image_filesystem() { :; }
+  check_cri() { printf "cri\n"; }
+  manage_orchestration_circuit_breaker() { printf "recovery\n"; }
+  check_dns() { printf "dns\n"; }
+  check_disk_wear() { printf "smart\n"; }
+  main
+'
+The status should be success
+The output should equal "$(printf 'cri\nrecovery\ndns\nsmart')"
+End
 End