diff --git a/CHANGELOG.md b/CHANGELOG.md index a2a289c..5ce585f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,33 @@ versions follow [Semantic Versioning](https://semver.org/). ## [Unreleased] +## [0.6.13] - 2026-10-24 + +### Added + +- Interactive usage sunburst (`usage-sunburst/` route): zoomable, + bookmarkable folder breakdown replacing both static doughnuts; project + scope shows only the selected project's bytes and hides volume cards. +- Browser move-apply (`move-apply/`): explicit confirm checkbox, optional + content re-hash, project scope enforces edit rights, transport + pre-flight, bounded run; result card names the verification. +- Restorable safe move: `apply_archive(retire_source=True)` stages the + verified source aside (atomic same-filesystem rename) instead of + deleting; `sweep-restorable` CLI verb deletes expired staging after the + 7-day retention; manifests record `staged_path`; `--retire-source` flag + on `archive`. +- Duplicates tab as an app: unavailable state shows the would-scan roots + and what the report holds instead of only a command. + +### Changed + +- Zero margins on the largest div: app content fills the pane edge to + edge (live dock clearance kept). +- No-project scope notice names the hub-mounted requirement where no + header picker exists. +- Archive report names staged restorable sources instead of reading as a + backup retain or a delete. + ### Fixed - Docs drift (7 fixes, all evidenced by the code): Backup named as the diff --git a/README.md b/README.md index 52ee9f6..3ef34f0 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ | 2 | **Inodes exhausted, disk looks free** — `No space left on device` with GBs free: `du` measures bytes, not the millions of tiny files starving an HPC quota | **The `FILES` column** — every child's inode count, and `--sort files` to rank by it, so an inode hog surfaces even when it's small on disk | | 3 | **Stale-build cleanup deletes a live image** — a directory of dated images (SIFs, tarballs) where the currently-live file is often the oldest-looking one still symlinked in | **`scitex-storage images prune`** — rotates to the newest N, but a file any symlink in the directory currently resolves to is never a candidate, regardless of age. Dry-run by default | | 4 | **HPC fileset hits its inode quota** — millions of small files on GPFS; deleting real data is not an option, and the fix (tar it) reads file content, barred from login nodes | **`scitex-storage sweep`** — tars an inode-hog directory in place, compute-node-only (needs `$SLURM_JOB_ID`), explicit per-directory `--confirm`, skips anything still active | -| 5 | **Copy-then-delete risks silent loss** — local disk fills with directories that should move to NAS, but "just delete after copying" risks a partial/failed copy silently losing data | **`scitex-storage archive`** — copy-verify-then-remove over ssh to `nas`/`nas2` (scitex-ssh's `sync_dir`), checksummed by default, manifest written first, `restore` reads it back | +| 5 | **Copy-then-delete risks silent loss** — local disk fills with directories that should move to NAS, but "just delete after copying" risks a partial/failed copy silently losing data | **`scitex-storage archive`** — copy-verify-then-remove over ssh to `nas`/`nas2` (scitex-ssh's `sync_dir`), checksummed by default, manifest written first, `restore` reads it back; `--retire-source` stages the original restorable instead of deleting (swept by `sweep-restorable` after retention) | | 6 | **Duplicate copies pile up** — `dataset (1).zip`, `dataset_final_v2.zip`, and other near-identical names crowding the same directory | **`scitex-storage find-duplicates`** — an explicitly opt-in, separate verb (via `fclones`) that hashes file contents to report exact-duplicate groups; kept out of `scan` on purpose (see below) | ## Quick Start diff --git a/pyproject.toml b/pyproject.toml index 8172037..dd56233 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "scitex-storage" -version = "0.6.12" +version = "0.6.13" description = "Research-data storage triage: read-only stat-only scan + referenced-file-aware rotation of versioned build artifacts (e.g. SIF images)" readme = "README.md" license = "AGPL-3.0-only" diff --git a/src/scitex_storage/__init__.py b/src/scitex_storage/__init__.py index 5352378..824b5de 100755 --- a/src/scitex_storage/__init__.py +++ b/src/scitex_storage/__init__.py @@ -53,6 +53,10 @@ "apply_sweep": "_transfer._sweep", "plan_sweep": "_transfer._sweep", "sweep_status": "_transfer._sweep", + "StagedRecord": "_transfer._retention", + "SweepReport": "_transfer._retention", + "stage_for_retention": "_transfer._retention", + "sweep_restorable": "_transfer._retention", "ArchiveManifest": "_transfer._archive", "ArchivePlan": "_transfer._archive", "RestorePlan": "_transfer._archive", @@ -142,6 +146,10 @@ def __dir__() -> list[str]: "apply_sweep", "plan_sweep", "sweep_status", + "StagedRecord", + "SweepReport", + "stage_for_retention", + "sweep_restorable", "ArchiveManifest", "ArchivePlan", "RestorePlan", diff --git a/src/scitex_storage/_cli/_archive_cmd.py b/src/scitex_storage/_cli/_archive_cmd.py index c41ed95..78efebf 100755 --- a/src/scitex_storage/_cli/_archive_cmd.py +++ b/src/scitex_storage/_cli/_archive_cmd.py @@ -122,6 +122,16 @@ def _require_rsync() -> None: is_flag=True, help="Preview only (explicit spelling of the default -- no side effects either way).", ) +@click.option( + "--retire-source", + "retire_source", + is_flag=True, + default=False, + show_default=True, + help="After a verified sync, move the local source to restorable " + "staging instead of deleting it (swept after retention; see " + "sweep-restorable).", +) @click.option( "--yes", "-y", @@ -138,6 +148,7 @@ def archive_cmd( checksum: bool, verify_content_too: bool, dry_run: bool, + retire_source: bool, confirmed: bool, as_json: bool, ) -> None: @@ -202,6 +213,7 @@ def archive_cmd( checksum=checksum, exclude=exclude_patterns, verify_content_too=verify_content_too, + retire_source=retire_source, ) if do_apply else None diff --git a/src/scitex_storage/_cli/_lazy.py b/src/scitex_storage/_cli/_lazy.py index 9fe5c84..c2bfe2b 100755 --- a/src/scitex_storage/_cli/_lazy.py +++ b/src/scitex_storage/_cli/_lazy.py @@ -59,6 +59,7 @@ "images": "._images_cmd:images_group", "sweep": "._sweep_cmd:sweep_cmd", "sweep-status": "._sweep_cmd:sweep_status_cmd", + "sweep-restorable": "._retention_cmd:sweep_restorable_cmd", "archive": "._archive_cmd:archive_cmd", "backup": "._backup_cmd:backup_cmd", "restore": "._archive_cmd:restore_cmd", diff --git a/src/scitex_storage/_cli/_retention_cmd.py b/src/scitex_storage/_cli/_retention_cmd.py new file mode 100644 index 0000000..8e6b739 --- /dev/null +++ b/src/scitex_storage/_cli/_retention_cmd.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""``scitex-storage sweep-restorable`` — delete expired restorable staging. + +Staged trees (verified moves whose source was moved aside instead of +deleted) become eligible for deletion once their retention expires +(default 7 days at stage time). This verb lists the expired ones by +default and deletes them only with ``--yes`` -- never a blanket delete, +and never anything the guards refuse (outside a ``.stx-restorable`` +directory, or a tree whose marker no longer matches). +""" + +from __future__ import annotations + +import json + +import click + +from .._transfer._retention import sweep_restorable +from ._compat import spec_command_kwargs + + +@click.command( + "sweep-restorable", + **spec_command_kwargs( + summary="Delete restorable-staged trees whose retention expired.", + description=( + "Lists staged trees (verified moves kept restorable instead of " + "deleted) whose 7-day retention has expired. Defaults to a " + "dry-run listing; --yes deletes exactly the expired, guarded " + "trees. Entries outside a `.stx-restorable` directory or whose " + "marker no longer matches are refused, never deleted. Run " + "periodically (cron) so staged originals do not accumulate." + ), + examples=( + ("{prog} sweep-restorable", "dry-run, list expired staging"), + ( + "{prog} sweep-restorable --yes", + "delete exactly the expired staged trees", + ), + ), + ), +) +@click.option( + "--dry-run", + "dry_run", + is_flag=True, + default=False, + help="List expired staging without deleting (the default behavior too).", +) +@click.option( + "--yes", + "-y", + "assume_yes", + is_flag=True, + default=False, + help="Actually delete the expired staged trees.", +) +@click.option("--json", "as_json", is_flag=True, help="Emit JSON instead of text.") +def sweep_restorable_cmd(dry_run: bool, assume_yes: bool, as_json: bool) -> None: + # --yes is the only thing that deletes: without it this is a dry-run + # listing, whether --dry-run was spelled out or not. + report = sweep_restorable(dry_run=not assume_yes) + if as_json: + click.echo( + json.dumps( + { + "removed": report.removed, + "retained": report.retained, + "pruned": report.pruned, + "refused": report.refused, + "errors": report.errors, + }, + indent=2, + ) + ) + return + if assume_yes: + for path in report.removed: + click.echo(f"removed {path}") + elif report.removed: + for path in report.removed: + click.echo(f"expired (dry-run) {path}") + else: + click.echo("no expired staging.") + for path in report.retained: + click.echo(f"retained (not yet expired) {path}") + for path in report.pruned: + click.echo(f"pruned missing record {path}") + for path in report.refused: + click.echo(f"REFUSED (not provably staged) {path}") + for path, error in report.errors.items(): + click.echo(f"error {path}: {error}") + + +# EOF diff --git a/src/scitex_storage/_django/organize.py b/src/scitex_storage/_django/organize.py index 4eb17e3..3e90790 100755 --- a/src/scitex_storage/_django/organize.py +++ b/src/scitex_storage/_django/organize.py @@ -35,9 +35,10 @@ # PS-233/PS-148 so a GUI import without the extra names the remedy instead # of a bare ModuleNotFoundError. try: - from django.http import HttpResponseForbidden + from django.http import HttpResponse, HttpResponseForbidden from django.shortcuts import render from django.utils.translation import gettext_lazy as _ + from django.views.decorators.http import require_POST except ImportError as exc: raise ImportError( "scitex-storage organize views require Django: " @@ -63,6 +64,16 @@ DUPLICATES_TIMEOUT_S = 60.0 PLAN_TIMEOUT_S = 20.0 +#: Seconds a browser-triggered archive may run before the tab reports a +#: timeout instead. Longer than the planners: an apply moves real bytes. +APPLY_TIMEOUT_S = 120.0 + +#: Transport seam for the browser move-apply. Tests inject a fake runner +#: (save/set/restore, the ``_GET_CURRENT_PROJECT`` precedent -- never a +#: mock); production leaves ``None`` so :func:`apply_archive` shells out +#: to rsync and the real pre-flight probe runs. +_APPLY_RUNNER = None + #: Donut geometry (r=70 in a 160x160 viewBox) + a fixed segment palette. DONUT_R = 70.0 DONUT_C = 2 * math.pi * DONUT_R @@ -208,6 +219,28 @@ def _scope_urls(request, tab): return f"?{base}", f"?{base}&scope=project" +def _host_can_pick_project() -> bool: + """True when the header can actually offer a project picker. + + The scope notice otherwise points at a control that does not exist: + without the host provider URL (standalone) the picker tag renders + nothing, so "pick a project in the header" is a dead-end instruction + there. Library + provider URL are both required for the picker to + appear (see ``_project_picker.html``). + """ + try: + from django.conf import settings + except ImportError: # pragma: no cover - gui extra absent + return False + + from . import views as _views + + return bool( + getattr(settings, "SCITEX_PROJECT_PROVIDER_URL", "") + and _views._project_picker_available() + ) + + def _scope_context(request, tab): """Shared scope-control context for the organize tabs.""" scope, pscope = _scope_selection(request) @@ -216,10 +249,17 @@ def _scope_context(request, tab): name = "" if scope == "project": if pscope is None: - notice = str( - _("No project in scope -- showing user space. Pick a project " - "in the header to filter these tabs to it.") - ) + if _host_can_pick_project(): + notice = str( + _("No project in scope -- showing user space. Pick a project " + "in the header to filter these tabs to it.") + ) + else: + notice = str( + _("No project in scope -- showing user space. Project " + "filtering needs the hub-mounted app with a signed-in " + "project.") + ) else: name = pscope.name or pscope.slug or str(_("this project")) return { @@ -349,9 +389,14 @@ def usage(request): _("Per-folder scan failed: %(err)s") % {"err": err} ) + from urllib.parse import urlencode + + keep = [(k, v) for k, v in request.GET.items() if k in ("volume", "scope")] + sunburst_qs = urlencode(keep) context = { **_base_context("usage"), **scope_ctx, + "sunburst_qs": sunburst_qs, "fleet_segments": donut_segments(fleet_items), "fleet_total": sum(v for _label, v in fleet_items), "statuses": statuses, @@ -366,6 +411,73 @@ def usage(request): return render(request, "scitex_storage/organize.html", context) +def usage_sunburst(request): + """Interactive folder sunburst for the Usage tab's breakdown root. + + The same root the per-folder breakdown uses (the project directory + under project scope, else the selected volume path): the breakdown's + numbers live in this zoomable UI instead of a static donut. Served + live per request with the scan bounded; failures render a plain note + page -- never a blank page and never a 500, because "gather timed + out" is a real state a first-run user will hit. + """ + import datetime + import html + + from . import volumes + + try: + from scitex_storage._render import ( + build_dir_hierarchy, + build_dir_sunburst_html, + ) + except ImportError: + return HttpResponse( + "

Usage sunburst needs the scitex-storage package.

", + content_type="text/html", + ) + user_volumes = _user_volumes(request) + statuses = volumes.measure_all(user_volumes) + scope_ctx = _scope_context(request, "usage") + if request.GET.get("volume"): + folder_volume = _selected_volume(request, user_volumes, statuses) + if folder_volume is None: + return HttpResponseForbidden(str(_("That volume is not yours."))) + else: + folder_volume = _selected_volume(request, user_volumes, statuses) + root = None + label = "" + if scope_ctx["scope"] == "project" and scope_ctx["pscope"] is not None: + root = scope_ctx["pscope"].project_dir + label = scope_ctx["project_name"] + elif folder_volume is not None: + root = folder_volume.path + label = folder_volume.label + if root is None: + return HttpResponse( + "

No reachable volumes with measurable capacity yet.

", + content_type="text/html", + ) + ok, result = _run_bounded( + lambda: build_dir_hierarchy(label or str(root), root), SCAN_TIMEOUT_S + ) + if not ok: + err = result + if type(err).__name__ == "MissingSystemDependencyError": + note = _( + "Usage sunburst is unavailable here: " + "the `fd` binary is not installed on this host." + ) + elif isinstance(err, TimeoutError): + note = _("Usage scan timed out; volume totals on the Usage tab are unaffected.") + else: + note = _("Could not scan that folder: %(err)s") % {"err": err} + return HttpResponse(f"

{html.escape(str(note))}

", content_type="text/html") + when = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + gen_line = f"Scanned {html.escape(str(root))} · {html.escape(when)}" + return HttpResponse(build_dir_sunburst_html(label or str(root), result, gen_line)) + + def duplicates(request): """Read-only duplicate report. No resolve/delete action exists by design.""" from . import volumes @@ -459,6 +571,7 @@ def duplicates(request): "reclaimable": reclaimable, "scanned": scanned, "unavailable_reason": unavailable_reason, + "roots": [str(p) for p in roots], } return render(request, "scitex_storage/organize.html", context) @@ -583,6 +696,183 @@ def move(request): return render(request, "scitex_storage/organize.html", context) +def _apply_source(request, user_volumes): + """Resolve + authorize a browser move-apply source. + + Returns ``(source, destination)`` or ``(None, HttpResponse)`` for + fail-closed refusals (unknown volume, escape, no edit rights) and + ``(None, str)`` for input errors rendered on the tab. Project scope + requires edit rights (apps-lead decision: a browser-applied move is a + real mutation, like rename/delete/mkdir); user-space volumes are the + requester's own, so no project permission applies there. + """ + from . import project_files, volumes + + form_volume = request.POST.get("volume", "") + form_dir = request.POST.get("dir", "") + form_dest = request.POST.get("destination", "") + volume = volumes.find_volume(user_volumes, form_volume) + if volume is None: + return None, HttpResponseForbidden(str(_("That volume is not yours."))) + try: + source = volumes.contained_dir(volume, (form_dir or "").strip().strip("/")) + except Exception: + return None, HttpResponseForbidden(str(_("That folder is outside the volume."))) + try: + editable = project_files.resolve_project_scope(request, require_edit=True) + except project_files.PermissionDenied as exc: + return None, HttpResponseForbidden(str(exc)) + if editable is not None: + if not _within(source, editable.project_dir): + return None, HttpResponseForbidden( + str(_("That folder is outside the current project.")) + ) + elif request.POST.get("scope") == "project": + return None, HttpResponseForbidden( + str(_("No project in scope -- applying a move needs a project you may edit.")) + ) + return (source, form_dest), None + + +@require_POST +def move_apply(request): + """Execute a planned archive from the Move tab. A real mutation. + + POST-only: ``volume`` + ``dir`` + ``destination`` identify the planned + move, ``confirm=yes`` is the explicit human confirm (no silent apply), + ``verify_content`` opts into the second content gate. Edit rights are + enforced for project-scoped sources; the transport is pre-flighted + before a byte moves; plan and apply each run bounded. + """ + from . import volumes + + user_volumes = _user_volumes(request) + destinations = _archive_destinations() + applied = None + apply_error = "" + form_volume = request.POST.get("volume", "") + form_dir = request.POST.get("dir", "") + form_dest = request.POST.get("destination", "") + scope_ctx = _scope_context(request, "move") + resolved, problem = _apply_source(request, user_volumes) + if problem is not None: + if not isinstance(problem, str): + return problem + apply_error = problem + source = None + else: + source, form_dest = resolved + if not apply_error: + if form_dest not in destinations: + apply_error = str( + _("Pick an archive tier: %(tiers)s.") + % {"tiers": ", ".join(destinations) if destinations else "—"} + ) + elif request.POST.get("confirm") != "yes": + apply_error = str( + _("Applying a move needs an explicit confirm -- check the confirm box.") + ) + plan = None + if not apply_error: + try: + from scitex_storage import plan_archive as _plan + except ImportError: + _plan = None + if _plan is None: + apply_error = str( + _("Archive planning needs the scitex-storage package.") + ) + else: + ok, result = _run_bounded( + lambda: _plan(source, form_dest), PLAN_TIMEOUT_S + ) + if ok: + plan = result + else: + err = result + if type(err).__name__ == "MissingSystemDependencyError": + apply_error = str( + _( + "Archive planning is unavailable here: " + "the `fd` binary is not installed on this host." + ) + ) + elif isinstance(err, TimeoutError): + apply_error = str(_("Archive planning timed out.")) + else: + apply_error = str( + _("Could not plan that move: %(err)s") % {"err": err} + ) + if not apply_error: + if _APPLY_RUNNER is None: + try: + from scitex_storage._transfer._archive_transport import ( + probe_transport as _probe, + ) + except ImportError: + _probe = None + if _probe is not None: + probe = _probe(form_dest) + if not probe.may_transport: + apply_error = str( + _("Transport to %(dest)s is %(verdict)s -- refusing to sync + delete. %(detail)s") + % {"dest": form_dest, "verdict": probe.verdict, "detail": probe.detail} + ) + if not apply_error: + try: + from scitex_storage import apply_archive as _apply + except ImportError: + _apply = None + if _apply is None: + apply_error = str( + _("Archive execution needs the scitex-storage package.") + ) + else: + verify_content = request.POST.get("verify_content") == "yes" + ok, result = _run_bounded( + lambda: _apply( + plan, + checksum=True, + verify_content_too=verify_content, + retire_source=True, + runner=_APPLY_RUNNER, + ), + APPLY_TIMEOUT_S, + ) + if ok: + applied = result + else: + err = result + if isinstance(err, TimeoutError): + apply_error = str( + _("Archive run timed out -- check the destination before retrying.") + ) + else: + apply_error = str( + _("Archive failed: %(err)s") % {"err": err} + ) + tiers: dict[str, list] = {} + statuses = volumes.measure_all(_user_volumes(request)) + for st in statuses: + tiers.setdefault(_tier_of(st.volume), []).append(st) + context = { + **_base_context("move"), + **scope_ctx, + "tiers": sorted(tiers.items()), + "reachable": volumes.REACHABLE, + "destinations": destinations, + "form_volume": form_volume, + "form_dir": form_dir, + "form_dest": form_dest, + "user_volumes": user_volumes, + "plan": None, + "plan_error": "", + "applied": applied, + "apply_error": apply_error, + } + return render(request, "scitex_storage/organize.html", context) + + def _backup_summaries(): """Local manifests as display dicts, newest first; never raises.""" import datetime diff --git a/src/scitex_storage/_django/static/scitex_storage/index.css b/src/scitex_storage/_django/static/scitex_storage/index.css index 09e8e38..0bf5d47 100644 --- a/src/scitex_storage/_django/static/scitex_storage/index.css +++ b/src/scitex_storage/_django/static/scitex_storage/index.css @@ -11,10 +11,10 @@ max-width here per its no-!important contract. Long-form text still caps itself (see .stx-storage-coming p). */ margin: 0; - /* Symmetric pane gutters (operator 2026-10-22, supersedes the 12px/8px - split of CCT4926): top and left air identical so the main area reads - as one consistent frame. Dock reserve + 16px air at the bottom. */ - padding: 12px 12px calc(var(--site-dock-live-height, 0px) + 16px); + /* Operator 2026-10-24: no margins on the largest div — the app content + fills the pane edge to edge. The live dock clearance stays (functional, + not a margin): without it the dock would cover the last row. */ + padding: 0 0 var(--site-dock-live-height, 0px); color: var(--text-primary, inherit); font-size: 15px; } diff --git a/src/scitex_storage/_django/templates/scitex_storage/organize.html b/src/scitex_storage/_django/templates/scitex_storage/organize.html index 5b44ebc..62febfb 100644 --- a/src/scitex_storage/_django/templates/scitex_storage/organize.html +++ b/src/scitex_storage/_django/templates/scitex_storage/organize.html @@ -11,17 +11,6 @@ -

Fleet Capacity Sunburst

+

{title}

{gen}

-
- - -
+
{modes}
- + @@ -211,6 +272,40 @@ def build_sunburst_html(snapshot: FleetSnapshot) -> str: note = html.escape(snapshot.note or "") gen_line = f"Generated {gen}" + (f" · {note}" if note else "") data = json.dumps(root).replace("Colour: Space %%\n' + ' \n' + ), + footer=( + "Angle = capacity · colour = the selected metric " + "(green→red, grey = unmeasured) · click a host to " + "zoom, the breadcrumb climbs back. Self-contained, opens offline." + ), + data=data, + ) + + +def build_dir_sunburst_html(root_name: str, hierarchy: dict, gen_line: str) -> str: + """Render a directory hierarchy as the interactive sunburst. Pure. + + Same drawing routine as :func:`build_sunburst_html`; the colour-mode + buttons are omitted (folders carry no usage metric) and the title and + footer name folders instead of hosts. ``gen_line`` is already-escaped + HTML naming the scanned root and time. + """ + data = json.dumps(hierarchy).replace(" dict: return asdict(self) @@ -285,6 +290,8 @@ def apply_archive( exclude: tuple[str, ...] = (), verify_content_too: bool = False, remove_source: bool = True, + retire_source: bool = False, + retention_days: float = 7.0, runner=None, ) -> ArchiveManifest: """Execute an archive plan: push, READ THE DESTINATION BACK, write the @@ -296,6 +303,12 @@ def apply_archive( failed read-back raises either way, because an unverified copy is not a backup either. + ``retire_source=True`` (with the default ``remove_source=True``) is the + safe-move mode: after a positive verification the source is moved aside + into restorable staging (same filesystem, atomic rename) instead of + being deleted, and a sweeper deletes it once ``retention_days`` + (default 7) expire. The manifest records ``staged_path``. + Until 2026-07-28 this docstring already said "verify" and the body did not do it — the source was removed on rsync's exit code alone. A docstring asserting a safety step the code does not perform is its own @@ -473,6 +486,13 @@ def apply_archive( # scrub baseline at zero marginal cost. content_digest = tree_digest(source_manifest) + if retire_source and not remove_source: + raise ValueError( + "retire_source=True stages the source aside instead of deleting " + "it; combine it with the default remove_source=True, or use " + "remove_source=False alone for backup mode." + ) + manifest = ArchiveManifest( source=str(plan.source), destination=plan.destination, @@ -526,7 +546,15 @@ def apply_archive( ) if remove_source: - shutil.rmtree(plan.source) + if retire_source: + from ._retention import stage_for_retention + + staged = stage_for_retention(plan.source, retention_days=retention_days) + manifest.retained_source = True + manifest.staged_path = staged.staged_path + plan.manifest_path.write_text(json.dumps(manifest.to_dict(), indent=2)) + else: + shutil.rmtree(plan.source) return manifest diff --git a/src/scitex_storage/_transfer/_retention.py b/src/scitex_storage/_transfer/_retention.py new file mode 100644 index 0000000..37edaf0 --- /dev/null +++ b/src/scitex_storage/_transfer/_retention.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# File: src/scitex_storage/_transfer/_retention.py +"""Restorable staging for verified moves (operator 2026-10-24). + +A move is only safe if the original survives verification AND stays +recoverable for a while after: rsync + checksum validate the copy, then +the source moves to a restorable staging area instead of being deleted +outright. A sweeper deletes staged trees once their retention expires +(default 7 days). + +Staging lives NEXT TO the source (``/.stx-restorable/``), so the +move is an atomic same-filesystem rename that needs no extra space. A +central registry tracks every staged tree so the sweeper can find them. + +Deletion guards (each recorded, never silent): +* only paths inside a ``.stx-restorable`` directory are ever removed -- + a tampered or hand-edited registry entry pointing elsewhere is refused; +* the staged tree must still carry its marker file with a matching + original path -- something a user repurposed the directory for is left + alone; +* symlinks are unlinked, never followed. +""" + +from __future__ import annotations + +import datetime +import json +import os +import shutil +import time +from dataclasses import asdict, dataclass +from pathlib import Path + +#: Staging directory name, always a direct child of the source's parent. +STAGING_DIRNAME = ".stx-restorable" + +#: Marker file written inside every staged tree; the sweeper only removes +#: trees whose marker matches the registry entry. +MARKER_FILENAME = ".stx-restorable-record.json" + +#: Operator 2026-10-24: a moved original stays restorable for a week. +DEFAULT_RETENTION_DAYS = 7.0 + + +@dataclass(frozen=True) +class StagedRecord: + """One staged (moved-aside, not yet deleted) source tree.""" + + staged_path: str + original_path: str + staged_at: float + delete_after: float + + +@dataclass +class SweepReport: + """What one sweeper pass saw and did.""" + + removed: list[str] + retained: list[str] + pruned: list[str] + refused: list[str] + errors: dict[str, str] + + +def _registry_path() -> Path: + """Central registry of staged trees (resolved fresh, sandbox-friendly).""" + return Path("~/.scitex/scitex-storage/runtime/restorable-registry.json").expanduser() + + +def _read_registry() -> list[dict]: + try: + data = json.loads(_registry_path().read_text(encoding="utf-8")) + except (OSError, ValueError): + return [] + return data if isinstance(data, list) else [] + + +def _write_registry(entries: list[dict]) -> None: + path = _registry_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(entries, indent=2), encoding="utf-8") + + +def stage_for_retention( + source: str | Path, *, retention_days: float = DEFAULT_RETENTION_DAYS +) -> StagedRecord: + """Move ``source`` aside into restorable staging; return the record. + + Same-filesystem atomic rename (no extra space, no copy window). + Raises ``FileNotFoundError`` for a missing source. A collision on the + staged name appends ``-N`` rather than overwriting. + """ + src = Path(source) + if not src.exists() and not src.is_symlink(): + raise FileNotFoundError(f"nothing to stage: {source}") + staging = src.parent / STAGING_DIRNAME + staging.mkdir(parents=True, exist_ok=True) + stamp = datetime.datetime.fromtimestamp( + time.time(), tz=datetime.timezone.utc + ).strftime("%Y%m%dT%H%M%SZ") + base = f"{stamp}--{src.name}" + target = staging / base + counter = 1 + while target.exists() or target.is_symlink(): + counter += 1 + target = staging / f"{base}-{counter}" + os.replace(src, target) + staged_at = time.time() + record = StagedRecord( + staged_path=str(target), + original_path=str(src), + staged_at=staged_at, + delete_after=staged_at + retention_days * 86400, + ) + marker = {"original_path": str(src), "staged_at": staged_at, + "delete_after": record.delete_after} + if target.is_dir() and not target.is_symlink(): + marker_path = target / MARKER_FILENAME + else: + # A staged file cannot hold its own marker: a sibling carries it. + marker_path = target.parent / (target.name + "." + MARKER_FILENAME) + marker_path.write_text(json.dumps(marker, indent=2), encoding="utf-8") + entries = _read_registry() + entries.append(asdict(record)) + _write_registry(entries) + return record + + +def _marker_path_for(staged: Path) -> Path: + if staged.is_dir() and not staged.is_symlink(): + return staged / MARKER_FILENAME + return staged.parent / (staged.name + "." + MARKER_FILENAME) + + +def _marker_matches(staged: Path, entry: dict) -> bool: + """True when the staged tree still carries its own staging marker.""" + try: + marker = json.loads(_marker_path_for(staged).read_text(encoding="utf-8")) + except (OSError, ValueError): + return False + return ( + isinstance(marker, dict) + and marker.get("original_path") == entry.get("original_path") + ) + + +def sweep_restorable( + *, dry_run: bool = True, now: float | None = None +) -> SweepReport: + """List staged trees whose retention expired; delete them unless dry-run. + + Dry-run is the default (nothing is removed without an explicit + ``dry_run=False``). Every sighting is reported either way. + + ``dry_run`` reports without touching the disk. Registry entries whose + tree is already gone are pruned; entries pointing outside a + ``.stx-restorable`` directory, or whose marker no longer matches, are + refused (recorded, never deleted). + """ + moment = time.time() if now is None else now + report = SweepReport(removed=[], retained=[], pruned=[], refused=[], errors={}) + entries = _read_registry() + kept: list[dict] = [] + for entry in entries: + staged = Path(entry.get("staged_path", "")) + if staged.parent.name != STAGING_DIRNAME or not staged.exists(): + if not staged.exists() and staged.parent.name == STAGING_DIRNAME: + report.pruned.append(str(staged)) + continue + report.refused.append(str(staged)) + kept.append(entry) + continue + if moment < float(entry.get("delete_after", 0)): + report.retained.append(str(staged)) + kept.append(entry) + continue + if not _marker_matches(staged, entry): + report.refused.append(str(staged)) + kept.append(entry) + continue + if dry_run: + report.removed.append(str(staged)) + kept.append(entry) + continue + try: + if staged.is_symlink() or staged.is_file(): + staged.unlink() + # The sibling marker does not live inside the removed file. + _marker_path_for(staged).unlink(missing_ok=True) + else: + shutil.rmtree(staged) + except OSError as exc: + report.errors[str(staged)] = str(exc) + kept.append(entry) + continue + report.removed.append(str(staged)) + _write_registry(kept) + return report diff --git a/tests/scitex_storage/_cli/test__archive_cmd.py b/tests/scitex_storage/_cli/test__archive_cmd.py index 733c71a..64d0c94 100755 --- a/tests/scitex_storage/_cli/test__archive_cmd.py +++ b/tests/scitex_storage/_cli/test__archive_cmd.py @@ -144,6 +144,20 @@ def test_cli_archive_yes_with_dry_run_does_not_mutate(tmp_path, sandbox_home): assert f.exists() +@pytest.mark.requires_fd +def test_cli_archive_retire_source_flag_is_accepted_on_dry_run(tmp_path, sandbox_home): + # Arrange -- the flag only changes the post-verify fate; the dry-run + # still only measures. --yes is not passed, so nothing moves. + source = tmp_path / "source" + f = _touch(source / "a.bin") + runner = CliRunner() + # Act + result = runner.invoke( + main, ["archive", str(source), "--to", "nas", "--retire-source"]) + # Assert + assert (result.exit_code, f.exists()) == (0, True) + + @pytest.mark.requires_fd def test_cli_archive_json_reports_the_destination(tmp_path, sandbox_home): # Arrange diff --git a/tests/scitex_storage/_cli/test__retention_cmd.py b/tests/scitex_storage/_cli/test__retention_cmd.py new file mode 100644 index 0000000..d413e00 --- /dev/null +++ b/tests/scitex_storage/_cli/test__retention_cmd.py @@ -0,0 +1,83 @@ +"""Unit tests for the ``scitex-storage sweep-restorable`` CLI command.""" + +import json +import os + +import pytest +from click.testing import CliRunner + +from scitex_storage._cli import main +from scitex_storage._transfer._retention import stage_for_retention + + +@pytest.fixture +def sandbox_home(tmp_path): + """Point $HOME at tmp_path; restore afterwards. No mocks involved.""" + previous = os.environ.get("HOME") + os.environ["HOME"] = str(tmp_path) + try: + yield tmp_path + finally: + if previous is None: + os.environ.pop("HOME", None) + else: + os.environ["HOME"] = previous + + +def _staged(tmp_path, name="work", retention_days=0.0): + src = tmp_path / name + (src / "sub").mkdir(parents=True) + (src / "sub" / "a.bin").write_bytes(b"x" * 64) + return stage_for_retention(src, retention_days=retention_days) + + +def test_cli_sweep_restorable_defaults_to_dry_run(sandbox_home, tmp_path): + # Arrange + record = _staged(tmp_path) + runner = CliRunner() + # Act + result = runner.invoke(main, ["sweep-restorable"]) + # Assert -- reported, not removed. + from pathlib import Path + + outcome = (result.exit_code, "expired (dry-run)" in result.output, Path(record.staged_path).is_dir()) + assert outcome == (0, True, True) + + +def test_cli_sweep_restorable_yes_deletes_expired(sandbox_home, tmp_path): + # Arrange + record = _staged(tmp_path) + runner = CliRunner() + # Act + result = runner.invoke(main, ["sweep-restorable", "--yes"]) + # Assert + from pathlib import Path + + outcome = (result.exit_code, "removed" in result.output, Path(record.staged_path).exists()) + assert outcome == (0, True, False) + + +def test_cli_sweep_restorable_retains_unexpired(sandbox_home, tmp_path): + # Arrange -- default 7-day retention: nothing is due. + record = _staged(tmp_path, retention_days=7.0) + runner = CliRunner() + # Act + result = runner.invoke(main, ["sweep-restorable", "--yes"]) + # Assert + from pathlib import Path + + outcome = (result.exit_code, "retained" in result.output, Path(record.staged_path).is_dir()) + assert outcome == (0, True, True) + + +def test_cli_sweep_restorable_json_reports_counts(sandbox_home, tmp_path): + # Arrange + _staged(tmp_path, name="old", retention_days=0.0) + _staged(tmp_path, name="new", retention_days=7.0) + runner = CliRunner() + # Act + result = runner.invoke(main, ["sweep-restorable", "--json"]) + payload = json.loads(result.output) + # Assert + outcome = (result.exit_code, len(payload["removed"]), len(payload["retained"])) + assert outcome == (0, 1, 1) diff --git a/tests/scitex_storage/_django/test_views.py b/tests/scitex_storage/_django/test_views.py index e7b5fcd..cb48bac 100755 --- a/tests/scitex_storage/_django/test_views.py +++ b/tests/scitex_storage/_django/test_views.py @@ -10,6 +10,7 @@ import importlib import os +import shutil import pytest @@ -189,8 +190,9 @@ def __init__(self, returncode, stdout, stderr): assert str(source) in body -def test_usage_tab_renders_donut(tmp_path): - # Arrange +def test_usage_tab_renders_usage_section(tmp_path): + # Arrange -- the static doughnut is gone (operator 2026-10-24: dynamic + # sunburst instead); the section heading stays as the anchor. _boot() _touch(tmp_path / "alpha" / "a.bin", 100) # Act @@ -199,6 +201,70 @@ def test_usage_tab_renders_donut(tmp_path): assert "Where your bytes live" in body +def test_usage_tab_links_interactive_sunburst(tmp_path): + # Arrange + _boot() + _touch(tmp_path / "alpha" / "a.bin", 100) + # Act + body = _get(tmp_path, {"tab": "usage"}).content.decode() + # Assert -- route name, never a hardcoded path. + assert "usage-sunburst" in body + + +def test_usage_tab_has_no_static_donut(tmp_path): + # Arrange -- operator 2026-10-24: drop the round UI; the breakdown + # lives in the interactive sunburst now. + _boot() + _touch(tmp_path / "alpha" / "a.bin", 100) + # Act + body = _get(tmp_path, {"tab": "usage"}).content.decode() + # Assert + assert ("stx-donut" in body, "Storage usage donut" in body) == (False, False) + + +def test_usage_project_scope_shows_only_project_bytes(tmp_path): + # Arrange -- operator "filtered disk": a selected project shows only + # that project's bytes, never the all-volumes cards. + _boot() + proj = tmp_path / "proj" + proj.mkdir() + (proj / "paper.pdf").write_bytes(b"x" * 64) + # Act + body = _get_scoped( + tmp_path, {"tab": "usage", "scope": "project"}, username="u", project_path=str(proj) + ).content.decode() + # Assert + result = ("Showing only" in body, "Paper Project" in body, "stx-storage-grid" in body) + assert result == (True, True, False) + + +def test_usage_tab_has_no_per_folder_breakdown_section(tmp_path): + # Arrange -- operator 2026-10-24: no per-folder breakdown; the sunburst + # resolves all of it. The vol-pick pills stay (round-pill verdict) as + # the sunburst-root picker. + _boot() + _touch(tmp_path / "alpha" / "a.bin", 100) + # Act + body = _get(tmp_path, {"tab": "usage"}).content.decode() + # Assert + result = ("Per-folder breakdown" in body, "Sunburst root:" in body) + assert result == (False, True) + + +def test_usage_scope_hides_the_root_picker(tmp_path): + # Arrange -- under project scope the project wins; no root to pick. + _boot() + proj = tmp_path / "proj" + proj.mkdir() + (proj / "paper.pdf").write_bytes(b"x" * 64) + # Act + body = _get_scoped( + tmp_path, {"tab": "usage", "scope": "project"}, username="u", project_path=str(proj) + ).content.decode() + # Assert + assert "Sunburst root:" not in body + + def test_usage_tab_links_fleet_views(scitex_dir): # Arrange -- operator 2026-10-17: no empty promo. With no snapshot # gathered the Usage tab shows no Fleet-visualizations section @@ -229,6 +295,30 @@ def test_duplicates_tab_is_read_only_report(tmp_path): assert "Read-only report." in body +_HAVE_FCLONES = bool(shutil.which("fclones")) +requires_no_fclones = pytest.mark.skipif( + _HAVE_FCLONES, reason="covers the fclones-absent unavailable app state" +) + + +@requires_no_fclones +def test_duplicates_unavailable_lists_would_scan_roots(tmp_path): + # Arrange -- operator 2026-10-24: an app, not a command dump. Without + # the binary the tab still shows its scope and what the report holds. + _boot() + _touch(tmp_path / "alpha" / "a.bin", 100) + # Act + body = _get(tmp_path, {"tab": "duplicates"}).content.decode() + result = ( + "Duplicates unavailable" in body, + "Would scan:" in body, + str(tmp_path) in body, + "find-duplicates" in body, + ) + # Assert + assert result == (True, True, True, True) + + def test_index_declares_every_shell_pane_so_none_reserves_width(tmp_path): """The pane declaration reaches the template context, for all three panes. @@ -460,6 +550,87 @@ def test_sunburst_view_serves_a_gathered_snapshot(scitex_dir): assert "FLEET-SENTINEL" in body +def _get_usage_sunburst(tmp_path, params, username=None, project_path=None, project_name="Paper Project"): + """GET through the usage-sunburst route with the test volumes provider.""" + from django.test import RequestFactory, override_settings + + from scitex_storage._django import organize, project_files + + _VOLUME_ROOT["path"] = str(tmp_path) + dotted = f"{__name__}.provide_test_volumes" + request = RequestFactory().get("/storage/usage-sunburst/", params) + prev = project_files._GET_CURRENT_PROJECT + if username is not None: + request.user = _ScopeUser(username) + + def seam(request, user=None): + return _ScopeProject(project_path, project_name) + + project_files._GET_CURRENT_PROJECT = seam + try: + with override_settings(SCITEX_STORAGE_VOLUMES_PROVIDER=dotted): + return organize.usage_sunburst(request) + finally: + project_files._GET_CURRENT_PROJECT = prev + + +@pytest.mark.requires_fd +def test_usage_sunburst_serves_interactive_page(tmp_path): + # Arrange -- real scan of a real temp tree, served self-contained. + _boot() + _touch(tmp_path / "alpha" / "a.bin", 100) + # Act + response = _get_usage_sunburst(tmp_path, {}) + body = response.content.decode() + # Assert -- dynamic UI: zoomable page, embedded data, no CDN. + result = (response.status_code, "Usage" in body, "alpha" in body, 'src="http' in body) + assert result == (200, True, True, False) + + +def test_usage_sunburst_unknown_volume_is_403(tmp_path): + # Arrange + _boot() + # Act + response = _get_usage_sunburst(tmp_path, {"volume": "nope"}) + # Assert -- fail closed, like the Usage tab itself. + assert response.status_code == 403 + + +def test_usage_sunburst_without_fd_names_the_missing_binary(tmp_path): + # Arrange -- an empty PATH hides fd and fdfind alike (sanctioned + # save/set/restore env pattern): the page still renders, naming the + # missing binary instead of 500ing. + _boot() + previous_path = os.environ.get("PATH") + os.environ["PATH"] = str(tmp_path) + try: + # Act + response = _get_usage_sunburst(tmp_path, {}) + body = response.content.decode() + finally: + if previous_path is None: + os.environ.pop("PATH", None) + else: + os.environ["PATH"] = previous_path + # Assert + assert (response.status_code, "fd" in body) == (200, True) + + +@pytest.mark.requires_fd +def test_usage_sunburst_project_scope_scans_the_project_directory(tmp_path): + # Arrange -- project dir holds the only file; the volume root is empty. + _boot() + proj = tmp_path / "proj" + proj.mkdir() + (proj / "paper.pdf").write_bytes(b"x" * 64) + # Act + body = _get_usage_sunburst( + tmp_path, {"scope": "project"}, username="u", project_path=str(proj) + ).content.decode() + # Assert -- the breakdown root is the project, not the volume. + assert ("Paper Project" in body, "paper.pdf" in body) == (True, True) + + def test_usage_tab_shows_fleet_links_once_a_snapshot_exists(scitex_dir): # Arrange -- a gathered snapshot turns the promo back on. _boot() @@ -691,6 +862,24 @@ def test_storage_header_stamps_no_provider_url_without_provider(tmp_path): assert all("data-provider-url" not in body for body in bodies) +def test_picker_availability_requires_a_configured_provider(tmp_path): + # Arrange -- the SDK tag fails loudly (ImproperlyConfigured) on a + # project-scoped render with no provider, so library presence alone + # must not count as available: the include guard has to check both. + from django.test import override_settings + + from scitex_storage._django.views import _project_picker_available + + _boot() + # Act + with override_settings(SCITEX_PROJECT_PROVIDER_URL=""): + without = _project_picker_available() + with override_settings(SCITEX_PROJECT_PROVIDER_URL="/stub-projects/"): + with_provider = _project_picker_available() + # Assert + assert (without, with_provider) == (False, True) + + # EOF @@ -803,8 +992,9 @@ def test_leaf_bottom_reserve_stays_tight(): def test_leaf_pane_padding_stays_tight(): - """Top/side pane padding stays minimal and symmetric (operator: content - fills the pane; identical-row program: top and left air match).""" + """No margins on the largest div (operator 2026-10-24: content fills the + pane edge to edge; live dock clearance kept so the dock cannot cover + the last row).""" # Arrange -- read the shipped stylesheet, not a constant. import re from pathlib import Path @@ -814,8 +1004,8 @@ def test_leaf_pane_padding_stays_tight(): match = re.search(r"^\.stx-storage \{(.*?)\}", body, re.M | re.S) # Act block = match.group(1) - # Assert -- 12px all around; top and left identical, dock reserve kept. - assert "padding: 12px 12px calc(var(--site-dock-live-height, 0px) + 16px)" in block + # Assert -- zero gutters edge to edge; live dock clearance kept. + assert "padding: 0 0 var(--site-dock-live-height, 0px)" in block class _ScopeUser: @@ -893,6 +1083,39 @@ def test_usage_project_scope_without_project_names_no_project(tmp_path): assert "No project in scope" in body +def test_usage_scope_notice_names_hub_mount_without_provider(tmp_path): + # Arrange -- standalone: no host provider URL, so the header renders no + # picker (operator "No Project Selected": the notice must not point at + # a control that does not exist). + _boot() + # Act + body = _get(tmp_path, {"tab": "usage", "scope": "project"}).content.decode() + result = ( + "No project in scope" in body, + "Pick a project in the header" not in body, + "hub-mounted" in body, + ) + # Assert + assert result == (True, True, True) + + +def test_usage_scope_notice_points_at_header_picker_with_provider(tmp_path): + # Arrange -- hub-mounted: the host provider URL is set, so the header + # can offer the picker; the notice may direct the user to it. + from django.test import override_settings + + _boot() + # Act + with override_settings(SCITEX_PROJECT_PROVIDER_URL="/stub-projects/"): + body = _get(tmp_path, {"tab": "usage", "scope": "project"}).content.decode() + result = ( + "No project in scope" in body, + "Pick a project in the header" in body, + ) + # Assert + assert result == (True, True) + + def test_usage_project_scope_scans_the_project_directory(tmp_path): # Arrange -- project dir holds the only file; the volume root is empty. _boot() @@ -937,6 +1160,244 @@ def test_move_project_scope_rejects_dir_outside_project(tmp_path): assert response.status_code == 403 +def _post_move_apply(tmp_path, data, username=None, project_path=None, project_name="Paper Project"): + """POST through the move-apply route with the test volumes provider.""" + from django.test import RequestFactory, override_settings + + from scitex_storage._django import organize, project_files + + _VOLUME_ROOT["path"] = str(tmp_path) + dotted = f"{__name__}.provide_test_volumes" + request = RequestFactory().post("/storage/move-apply/", data) + prev = project_files._GET_CURRENT_PROJECT + if username is not None: + request.user = _ScopeUser(username) + + def seam(request, user=None): + return _ScopeProject(project_path, project_name) + + project_files._GET_CURRENT_PROJECT = seam + try: + with override_settings(SCITEX_STORAGE_VOLUMES_PROVIDER=dotted): + return organize.move_apply(request) + finally: + project_files._GET_CURRENT_PROJECT = prev + + +def test_move_apply_rejects_get(tmp_path): + # Arrange -- apply is a real mutation: no GET entry point. + _boot() + from django.test import RequestFactory, override_settings + + from scitex_storage._django import organize + + _VOLUME_ROOT["path"] = str(tmp_path) + dotted = f"{__name__}.provide_test_volumes" + # Act + with override_settings(SCITEX_STORAGE_VOLUMES_PROVIDER=dotted): + response = organize.move_apply(RequestFactory().get("/storage/move-apply/")) + # Assert + assert response.status_code == 405 + + +def test_move_apply_requires_explicit_confirm(tmp_path): + # Arrange -- the confirm box unchecked: nothing moves, source intact. + _boot() + (tmp_path / "old-exp").mkdir() + (tmp_path / "old-exp" / "a.bin").write_bytes(b"x" * 64) + # Act + response = _post_move_apply( + tmp_path, {"volume": "mine", "dir": "old-exp", "destination": "scitex-nas-01"} + ) + body = response.content.decode() + # Assert + result = (response.status_code, "confirm" in body, (tmp_path / "old-exp").is_dir()) + assert result == (200, True, True) + + +def test_move_apply_unknown_volume_is_403(tmp_path): + # Arrange + _boot() + # Act + response = _post_move_apply( + tmp_path, {"volume": "nope", "dir": "", "destination": "scitex-nas-01", "confirm": "yes"} + ) + # Assert + assert response.status_code == 403 + + +def test_move_apply_unknown_destination_names_tiers(tmp_path): + # Arrange + _boot() + (tmp_path / "old-exp").mkdir() + # Act + response = _post_move_apply( + tmp_path, {"volume": "mine", "dir": "old-exp", "destination": "nope", "confirm": "yes"} + ) + body = response.content.decode() + # Assert + assert (response.status_code, "Pick an archive tier" in body) == (200, True) + + +def test_move_apply_project_scope_without_edit_is_403(tmp_path): + # Arrange -- the seam project carries no can_edit: view-only cannot apply. + _boot() + proj = tmp_path / "proj" + (proj / "old-exp").mkdir(parents=True) + # Act + response = _post_move_apply( + tmp_path, + {"volume": "mine", "dir": "", "destination": "scitex-nas-01", + "confirm": "yes", "scope": "project"}, + username="u", + project_path=str(proj), + ) + # Assert + assert response.status_code == 403 + + +class _EditableScopeProject(_ScopeProject): + """Seam project the requester may edit (can_edit is a literal True).""" + + def can_edit(self, user): + return True + + +def test_move_apply_project_scope_outside_project_is_403(tmp_path): + # Arrange -- editable project, but the source escapes it. + _boot() + import scitex_storage._django.project_files as project_files + + proj = tmp_path / "proj" + proj.mkdir() + (tmp_path / "old-exp").mkdir() + (tmp_path / "old-exp" / "a.bin").write_bytes(b"x") + request_user = _ScopeUser("u") + editable = [(proj, request_user)] + + def seam(request, user=None): + root, owner = editable[0] + project = _EditableScopeProject(str(root)) + project.name = "Paper Project" + return project + + # Act + from django.test import RequestFactory, override_settings + + from scitex_storage._django import organize + + _VOLUME_ROOT["path"] = str(tmp_path) + dotted = f"{__name__}.provide_test_volumes" + request = RequestFactory().post( + "/storage/move-apply/", + {"volume": "mine", "dir": "old-exp", "destination": "scitex-nas-01", + "confirm": "yes", "scope": "project"}, + ) + request.user = request_user + previous = project_files._GET_CURRENT_PROJECT + project_files._GET_CURRENT_PROJECT = seam + try: + with override_settings(SCITEX_STORAGE_VOLUMES_PROVIDER=dotted): + response = organize.move_apply(request) + finally: + project_files._GET_CURRENT_PROJECT = previous + # Assert + assert response.status_code == 403 + + +class _ApplyFakeResult: + """subprocess-run-shaped result for the apply fake transport.""" + + def __init__(self, returncode=0, stdout="", stderr=""): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +def _looks_like_tally(cmd) -> bool: + """True when ``cmd`` is apply_archive's destination read-back probe.""" + joined = " ".join(str(c) for c in (cmd if isinstance(cmd, (list, tuple)) else [cmd])) + return "! -type d" in joined and "wc -c" in joined + + +class _ApplyFakeRunner: + """Honest local double for the browser-apply transport. + + Records every argv; answers the destination read-back tally as a + faithful receipt tallied from the RECORDED RSYNC ARGV's source (the + mirror of test__archive's double, stated here so the view test never + imports test code). Everything else succeeds silently: mkdir, the + free-space probe (unanswerable, which apply_archive records and does + not refuse on), and the sidecar replica. + """ + + def __init__(self): + self.calls = [] + + def __call__(self, cmd, **kwargs): + self.calls.append(cmd) + if _looks_like_tally(cmd): + from scitex_storage._transfer._verify import local_tally + + for probe in reversed(self.calls): + if probe and str(probe[0]).rsplit("/", 1)[-1] == "rsync" and len(probe) >= 2: + tally = local_tally(str(probe[-2]).rstrip("/")) + if tally.entry_count is None: + return _ApplyFakeResult(0, "", "") + return _ApplyFakeResult( + 0, f"{tally.entry_count}\n{tally.size_bytes}\n", "" + ) + return _ApplyFakeResult(0, "", "") + return _ApplyFakeResult(0, "", "") + + +@pytest.mark.requires_fd +def test_move_apply_retires_source_after_verify(tmp_path): + # Arrange -- real plan over a real tree; the transport is the fake + # above, the delete and the manifest write are real. + import json + import os + + from scitex_storage._django import organize + + _boot() + (tmp_path / "old-exp").mkdir() + (tmp_path / "old-exp" / "a.bin").write_bytes(b"x" * 64) + home = tmp_path / "home" + home.mkdir() + previous_home = os.environ.get("HOME") + previous_runner = organize._APPLY_RUNNER + os.environ["HOME"] = str(home) + organize._APPLY_RUNNER = _ApplyFakeRunner() + try: + # Act + response = _post_move_apply( + tmp_path, {"volume": "mine", "dir": "old-exp", + "destination": "scitex-nas-01", "confirm": "yes"} + ) + body = response.content.decode() + finally: + if previous_home is None: + os.environ.pop("HOME", None) + else: + os.environ["HOME"] = previous_home + organize._APPLY_RUNNER = previous_runner + manifests = list((home / ".scitex" / "scitex-storage" / "runtime" / "archive-manifests").glob("*.json")) + staged = json.loads(manifests[0].read_text()).get("staged_path", "") if len(manifests) == 1 else "" + # Assert -- verified move: summary shown, source retired (not deleted), + # staged tree restorable, manifest records where. + from pathlib import Path + + result = ( + response.status_code, + "Move applied and verified" in body, + (tmp_path / "old-exp").exists(), + (staged != "" and Path(staged).is_dir() and (Path(staged) / "a.bin").is_file()), + len(manifests), + ) + assert result == (200, True, False, True, 1) + + def test_move_project_scope_plans_dir_inside_project(tmp_path): # Arrange _boot() diff --git a/tests/scitex_storage/_render/test__sunburst.py b/tests/scitex_storage/_render/test__sunburst.py index 1901c7c..e9d5b6b 100755 --- a/tests/scitex_storage/_render/test__sunburst.py +++ b/tests/scitex_storage/_render/test__sunburst.py @@ -8,6 +8,8 @@ from __future__ import annotations +import pytest + from scitex_storage._fleet_status import MEASURED, FleetSnapshot, HostStorage from scitex_storage._render import build_hierarchy, build_sunburst_html @@ -106,4 +108,144 @@ def test_a_mount_name_cannot_break_out_of_the_embedded_json_script(): # Assert -- the slash is escaped, proving the guard fired. assert "<\\/script>" in page + +def _dir_tree(tmp_path): + """Two folders: big/ holds 100B across two files, small/ holds 10B.""" + big = tmp_path / "big" + (big / "deep").mkdir(parents=True) + (big / "deep" / "f.bin").write_bytes(b"x" * 60) + (big / "top.bin").write_bytes(b"x" * 40) + small = tmp_path / "small" + small.mkdir() + (small / "s.bin").write_bytes(b"x" * 10) + return tmp_path + + +@pytest.mark.requires_fd +def test_dir_hierarchy_root_names_the_scanned_folder(tmp_path): + # Arrange + from scitex_storage._render import build_dir_hierarchy + + root = _dir_tree(tmp_path) + # Act + hierarchy = build_dir_hierarchy("Paper", root) + # Assert + assert (hierarchy["name"], hierarchy["value"]) == ("Paper", 110) + + +@pytest.mark.requires_fd +def test_dir_hierarchy_orders_largest_first_and_nests_folders(tmp_path): + # Arrange + from scitex_storage._render import build_dir_hierarchy + + root = _dir_tree(tmp_path) + # Act + hierarchy = build_dir_hierarchy("Paper", root) + # Assert + result = ( + [c["name"] for c in hierarchy["children"]], + hierarchy["children"][0]["children"][0]["name"], + ) + assert result == (["big", "small"], "deep") + + +@pytest.mark.requires_fd +def test_dir_hierarchy_rolls_up_beyond_top_n(tmp_path): + # Arrange + from scitex_storage._render import build_dir_hierarchy + + root = _dir_tree(tmp_path) + # Act + hierarchy = build_dir_hierarchy("Paper", root, top_n=1) + # Assert + result = ( + [c["name"] for c in hierarchy["children"]], + hierarchy["children"][-1]["value"], + ) + assert result == (["big", "Other"], 10) + + +def test_dir_hierarchy_needs_the_fd_binary(tmp_path): + # Arrange -- an empty PATH hides fd and fdfind alike (sanctioned + # save/set/restore env pattern: no monkeypatch anywhere in this repo). + import os + + from scitex_storage._measure._scan import MissingSystemDependencyError + from scitex_storage._render import build_dir_hierarchy + + previous_path = os.environ.get("PATH") + (tmp_path / "sub").mkdir() + os.environ["PATH"] = str(tmp_path) + try: + # Act + raised = None + try: + build_dir_hierarchy("Paper", tmp_path) + except MissingSystemDependencyError as exc: + raised = exc + finally: + if previous_path is None: + os.environ.pop("PATH", None) + else: + os.environ["PATH"] = previous_path + # Assert + assert raised is not None + + +@pytest.mark.requires_fd +def test_dir_page_has_no_colour_mode_buttons(tmp_path): + # Arrange -- folders carry no usage metric, so the fleet colour modes + # would be dead controls here. + from scitex_storage._render import build_dir_hierarchy, build_dir_sunburst_html + + hierarchy = build_dir_hierarchy("Paper", _dir_tree(tmp_path)) + # Act + page = build_dir_sunburst_html("Paper", hierarchy, "gen") + # Assert -- the JS guards reference the ids as strings; the dead + # controls are the buttons themselves. + assert ('