diff --git a/config/k3s/kyber-host-health.sh b/config/k3s/kyber-host-health.sh index 7cb2b3446..9c10faede 100644 --- a/config/k3s/kyber-host-health.sh +++ b/config/k3s/kyber-host-health.sh @@ -306,9 +306,37 @@ manage_orchestration_circuit_breaker() { fi } +check_disk_wear_device() { + local disk="$1" + local name output remaining status=0 + + name="$(basename "$disk")" + # SMART health warnings are exit-status bits, not necessarily read failures. + # Capture the status separately so a worn disk cannot abort host recovery. + output="$(timeout --kill-after=2 10 smartctl -A "$disk" 2>/dev/null)" || status=$? + if [ $((status & 7)) -ne 0 ]; then + set_alert "disk-wear-read-$name" "unable to read SMART attributes for $name (status $status); previous wear warning retained" + return + fi + + # Samsung reports remaining endurance as the normalized VALUE, not RAW_VALUE. + remaining="$(printf '%s\n' "$output" | awk '$2 == "Wear_Leveling_Count" { print $4; exit }')" + if [[ ! $remaining =~ ^[0-9]{1,3}$ ]] || [ "$((10#${remaining:-0}))" -gt 100 ]; then + set_alert "disk-wear-read-$name" "no valid remaining-endurance attribute for $name; previous wear warning retained" + return + fi + remaining=$((10#$remaining)) + clear_alert "disk-wear-read-$name" + if [ "$remaining" -le "$DISK_WEAR_WARNING_PERCENT" ]; then + set_alert "disk-wear-$name" "$name has ${remaining}% of its rated write endurance left; schedule a replacement before moving write-heavy state onto it" + else + clear_alert "disk-wear-$name" + fi +} + check_disk_wear() { local stamp="$STATE_DIR/disk-wear.checked" - local now last=0 disk name remaining + local now last=0 disk now="$(date +%s)" if [ -r "$stamp" ]; then @@ -321,23 +349,7 @@ check_disk_wear() { for disk in /dev/sd[a-z]; do [ -b "$disk" ] || continue - name="$(basename "$disk")" - # Samsung publishes remaining write endurance as the normalized value of - # Wear_Leveling_Count. smartd's -f only fires once that reaches the vendor - # threshold of zero, which is after the rated endurance is already spent - # and far too late to schedule a replacement. - # shellcheck disable=SC2016 - remaining="$(smartctl -A "$disk" 2>/dev/null | - awk '$2 == "Wear_Leveling_Count" { print $4 + 0; exit }')" - if [ -z "$remaining" ]; then - continue - fi - - if [ "$remaining" -le "$DISK_WEAR_WARNING_PERCENT" ]; then - set_alert "disk-wear-$name" "$name has ${remaining}% of its rated write endurance left; schedule a replacement before moving write-heavy state onto it" - else - clear_alert "disk-wear-$name" - fi + check_disk_wear_device "$disk" done } @@ -347,10 +359,10 @@ main() { check_d_state check_node_filesystem check_image_filesystem - check_disk_wear check_cri manage_orchestration_circuit_breaker check_dns + check_disk_wear } if [[ ${BASH_SOURCE[0]} == "$0" ]]; then diff --git a/spec/activate_kyber_spec.sh b/spec/activate_kyber_spec.sh index 15849ed71..d0f87bc9d 100644 --- a/spec/activate_kyber_spec.sh +++ b/spec/activate_kyber_spec.sh @@ -327,4 +327,84 @@ It 'runs the wear check as part of main' When run bash -c "awk '/^main\(\)/,/^}/' '$SCRIPT' | grep -q check_disk_wear" The status should be success End + +It 'handles SMART health status bits without aborting the remaining checks' +When run env HEALTH_CHECK="$SCRIPT" bash -c ' + source "$HEALTH_CHECK" + set_alert() { printf "alert:%s:%s\n" "$1" "$2"; } + clear_alert() { printf "clear:%s\n" "$1"; } + timeout() { + test "$*" = "--kill-after=2 10 smartctl -A /dev/sda" + printf "177 Wear_Leveling_Count 0x0013 009 009 000 Pre-fail Always - 12345\n" + return "$smart_status" + } + for smart_status in 0 8 16 32 64 128 248; do + check_disk_wear_device /dev/sda + done + printf "continued\n" +' +The status should be success +The output should include 'sda has 9% of its rated write endurance left' +The output should include 'continued' +The output should not include 'clear:disk-wear-sda' +End + +It 'preserves wear warnings when reads fail, time out, or return malformed values' +When run env HEALTH_CHECK="$SCRIPT" bash -c ' + source "$HEALTH_CHECK" + set_alert() { printf "alert:%s\n" "$1"; } + clear_alert() { printf "unexpected-clear:%s\n" "$1"; } + timeout() { + printf "177 Wear_Leveling_Count 0x0013 %s 009 000 Pre-fail Always - 12345\n" "$smart_value" + return "$smart_status" + } + smart_value=090 + for smart_status in 1 2 4 124 137; do + check_disk_wear_device /dev/sda + done + smart_status=0 + for smart_value in missing 101 1.5 -1 0000; do + check_disk_wear_device /dev/sda + done + timeout() { return 0; } + check_disk_wear_device /dev/sda + printf "continued\n" +' +The status should be success +The output should include 'alert:disk-wear-read-sda' +The output should include 'continued' +The output should not include 'unexpected-clear:' +End + +It 'clears read and wear alerts only after a valid healthy measurement' +When run env HEALTH_CHECK="$SCRIPT" bash -c ' + source "$HEALTH_CHECK" + set_alert() { printf "unexpected-alert:%s\n" "$1"; } + clear_alert() { printf "clear:%s\n" "$1"; } + timeout() { printf "177 Wear_Leveling_Count 0x0013 090 090 000 Pre-fail Always - 12345\n"; } + check_disk_wear_device /dev/sda +' +The status should be success +The output should include 'clear:disk-wear-read-sda' +The output should include 'clear:disk-wear-sda' +The output should not include 'unexpected-alert:' +End + +It 'runs CRI and freezer recovery before probing disk endurance' +When run env HEALTH_CHECK="$SCRIPT" bash -c ' + source "$HEALTH_CHECK" + install() { :; } + check_io_pressure() { :; } + check_d_state() { :; } + check_node_filesystem() { :; } + check_image_filesystem() { :; } + check_cri() { printf "cri\n"; } + manage_orchestration_circuit_breaker() { printf "recovery\n"; } + check_dns() { printf "dns\n"; } + check_disk_wear() { printf "smart\n"; } + main +' +The status should be success +The output should equal "$(printf 'cri\nrecovery\ndns\nsmart')" +End End