Coverage for scanpath_studio/persistence.py: 93%
667 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-07 21:10 +0000
1"""Local, single-user session persistence for Scanpath Studio.
3The hosted app deliberately does not persist anything: there is no user identity
4there, so a process-wide cache could expose one visitor's data to another. A
5session served on loopback only (the desktop app, a launch bound to
6``127.0.0.1``) stores uploaded datasets as Parquet plus a small JSON manifest and
7restores them on the next browser or process session. Which of the two a server
8is comes from its own bind address, never from the browser — see
9:func:`persistence_enabled`.
11Storing a researcher's tables on their disk is invisible by nature, so the cache
12is also *inspectable*: :func:`cache_status` reports what is stored, where, how
13big it is and when it was written without importing Streamlit, and it backs the
14in-app 🗂️ Data → *Saved on this computer* section
15(``app._render_saved_here_section``, UX-179), the ``scanpath-studio cache`` CLI
16subcommand and ``api.cache_status``. The stored files are deleted from outside
17the app — ``scanpath-studio cache --clear`` / ``api.clear_cache``, both
18:func:`clear_local_state`. Saving is paused for a session by BUG-71's breaker
19and when a manifest exists but cannot be read (:func:`persistence_paused`,
20:func:`cache_failure`); opting out is a launch choice
21(``run --no-persist`` / ``SCANPATH_STUDIO_PERSIST=0``).
23Each stored dataset restores on its own. One whose files are missing or
24unreadable is held back (:func:`failed_datasets`) — the others, the settings,
25the annotations and the metadata tables restore regardless — and its manifest
26entry and files are written back unchanged by every save until the user retries
27it (:func:`retry_failed_datasets`) or removes it (:func:`discard_failed_dataset`).
28"""
30from __future__ import annotations
32import hashlib
33import ipaddress
34import json
35import logging
36import os
37import tempfile
38import threading
39from collections.abc import MutableMapping
40from datetime import datetime
41from pathlib import Path
42from typing import Any
43from urllib.parse import urlparse
45import pandas as pd
47import scanpath_studio.annotations as annotations_mod
49from . import progress
50from .constants import (
51 DATASET_COUNTS_STORE_KEY,
52 DATASET_DESCRIPTIONS_KEY,
53 DATASET_SETUP_OVERRIDES_KEY,
54 DOWNLOAD_DIR_KEY,
55 RAW_GAZE_SEEDED_FOR_KEY,
56 RAW_GAZE_SNAP_RESTORE_KEY,
57 SETUP_OVERRIDE_FOR_KEY,
58 SETUP_OVERRIDE_RESTORE_KEY,
59 SETUP_OVERRIDE_SESSION_KEYS,
60)
61from .session_keys import (
62 COLUMN_MAPPING_PREFIX,
63 DESIGN_PRESETS,
64 PLOT_CONFIG_STATE_KEYS,
65 SINGLE_COMPARE_LAYOUT,
66 SINGLE_COMPARE_STIMULUS,
67 SINGLE_COMPARE_TOGGLE,
68 SINGLE_PLAYBACK_SPEED,
69 compare_state_keys,
70 keep_legacy_marker_scale,
71 rename_legacy_keys,
72)
74SCHEMA_VERSION = 1
76#: The default cache folder's name. Versioned by :data:`SCHEMA_VERSION` rather
77#: than by the app's version, and derived from it so the two cannot drift: the
78#: "v1" a user sees in the path is the *cache format's* version, so it stays put
79#: across app releases, and a format change starts a new folder beside this one
80#: instead of overwriting a session the new code could not read.
81CACHE_DIR_NAME = f"session-v{SCHEMA_VERSION}"
82PERSIST_ENV_VAR = "SCANPATH_STUDIO_PERSIST"
83STATE_DIR_ENV_VAR = "SCANPATH_STUDIO_STATE_DIR"
84_RESTORED_KEY = "_local_persistence_restored"
85_RESTORED_PAYLOAD_KEY = "_local_persistence_restored_payload"
86_PAUSED_KEY = "_local_persistence_paused"
87_LAST_FINGERPRINT_KEY = "_local_persistence_fingerprint"
88_LAST_DATASET_IDENTITY_KEY = "_local_persistence_dataset_identity"
89_LAST_DATASET_ENTRIES_KEY = "_local_persistence_dataset_entries"
90#: BUG-71 — the crash-loop breaker. Written beside the manifest just before a
91#: restore is applied, removed once a run that applied one reaches
92#: `save_local_state` (the epilogue, i.e. the run rendered). Finding it at the
93#: start of a session means the last session to restore this cache never got
94#: that far, so this one opens without it — see `restore_local_state`.
95RESTORE_MARKER_NAME = "restore-in-progress"
96_RESTORE_PENDING_KEY = "_local_persistence_restore_pending"
97_RESTORE_SKIPPED_KEY = "_local_persistence_restore_skipped"
98#: Stored datasets this session could not read back, ``{name: {"entry": the
99#: manifest entry, verbatim, "reason": why}}``. Each dataset restores on its own:
100#: one with a missing or unreadable file stays out of the session, and its entry
101#: and files stay in the cache — every save writes the entry back unchanged —
102#: until the user retries it or removes it (:func:`retry_failed_datasets`,
103#: :func:`discard_failed_dataset`).
104_FAILED_DATASETS_KEY = "_local_persistence_failed_datasets"
105#: Why the manifest itself could not be read, when it could not. A cache that
106#: exists but cannot be read pauses saving, so this session cannot replace it;
107#: an absent or cleared cache saves normally.
108_CACHE_FAILURE_KEY = "_local_persistence_cache_failure"
109#: The metadata tables' file, when the manifest points at it and it would not
110#: read (round 10): ``{"pointer": …, "reason": …}``. Held back like a dataset —
111#: every save keeps the pointer and leaves the file as it is — until
112#: :func:`retry_failed_metadata` reads it or :func:`discard_failed_metadata`
113#: removes it.
114_FAILED_METADATA_KEY = "_local_persistence_failed_metadata"
115_FRAME_KEYS = ("words", "fixations", "raw_gaze")
116#: DATA-38 — the attached metadata tables live beside the manifest, not in it,
117#: and are rewritten only when their content changes: the manifest is rewritten
118#: on every durable settings change (a layer toggle, a trial switch), and a
119#: trial table can run to tens of thousands of rows.
120METADATA_FILE = "metadata.json"
121_LAST_METADATA_SIGNATURE_KEY = "_local_persistence_metadata_signature"
122#: DATA-47 — the manifest pointer written with that file, reused while nothing
123#: changed (it lists every dataset's tables, which the signature does not).
124_LAST_METADATA_POINTER_KEY = "_local_persistence_metadata_pointer"
125_STATE_LOCK = threading.RLock()
126_LOGGER = logging.getLogger(__name__)
127_SESSION_KEYS = frozenset(PLOT_CONFIG_STATE_KEYS) | {
128 "data_source_choice",
129 "main_nav",
130 "single_select_trial_mode",
131 "single_trial_id",
132 "single_participant",
133 "single_slider",
134 "single_animate",
135 "wizard_filter_fields",
136 "_composite_trial_columns",
137 # VIZ-39 — the saved design library. It is the user's own work, not a
138 # setting derived from a dataset, so it belongs in the cache for the same
139 # reason annotations do: closing the app must not be how you lose it.
140 DESIGN_PRESETS,
141 # BUG-72 — Compare mode and the replay speed: `single_animate` was already
142 # here, so a restart kept Animate but dropped Compare, its layout, whose
143 # stimulus an overlay draws, and each scanpath's styling.
144 SINGLE_COMPARE_TOGGLE,
145 SINGLE_COMPARE_LAYOUT,
146 SINGLE_COMPARE_STIMULUS,
147 SINGLE_PLAYBACK_SPEED,
148 *compare_state_keys(0),
149 *compare_state_keys(1),
150 # UX-174 r2 — the descriptions the user wrote: their own words, like the
151 # design library. A plain session key so editing one never touches the
152 # stored frames (see `constants.DATASET_DESCRIPTIONS_KEY`).
153 DATASET_DESCRIPTIONS_KEY,
154 # VIZ-45 — which dataset the raw-gaze layer's default was last decided for,
155 # and what it overwrote. Without them a relaunch onto a raw-gaze-only
156 # dataset decides again and turns back on a layer the user switched off.
157 RAW_GAZE_SEEDED_FOR_KEY,
158 RAW_GAZE_SNAP_RESTORE_KEY,
159 # UX-184 — the folder the user chose for corpus downloads. A preference,
160 # like the design library: a restart must not send the next download back
161 # to the default folder.
162 DOWNLOAD_DIR_KEY,
163 # The recording setup the user saved for a built-in or public dataset, and
164 # which one the `global_*` keys hold now with what they held before — all
165 # three, or a relaunch would stash the override as the "before" it restores.
166 DATASET_SETUP_OVERRIDES_KEY,
167 SETUP_OVERRIDE_FOR_KEY,
168 SETUP_OVERRIDE_RESTORE_KEY,
169}
172def is_loopback_url(url: str = "") -> bool:
173 """Return whether ``url`` is addressed to this machine's loopback interface."""
174 host = (urlparse(str(url or "")).hostname or "").lower()
175 return host in {"localhost", "127.0.0.1", "::1"}
178def _is_loopback_host(host: str) -> bool:
179 name = str(host or "").strip().strip("[]").lower()
180 if name == "localhost":
181 return True
182 try:
183 return ipaddress.ip_address(name).is_loopback
184 except ValueError:
185 return False
188def server_bound_to_loopback() -> bool:
189 """Whether the running Streamlit server listens on a loopback address only.
191 This is the server's *own* configuration (``server.address``), which is what a
192 locality decision has to rest on. The page URL is not: Streamlit copies
193 ``st.context.url`` from the browser's own message, so any client can claim to
194 be at ``http://localhost/``. Unset — Streamlit's default, which listens on
195 every interface — is not loopback. Imports Streamlit lazily so the cache CLI
196 and API stay Streamlit-free.
197 """
198 try:
199 import streamlit as st
201 address = st.get_option("server.address")
202 except Exception:
203 return False
204 return _is_loopback_host(address or "")
207def persistence_enabled(url: str = "", environ: dict | None = None) -> bool:
208 """Return whether disk persistence is safe for this process.
210 ``SCANPATH_STUDIO_PERSIST=1`` explicitly enables it and ``=0`` disables it.
211 Without an override, inside a Streamlit server it is enabled only when that
212 server listens on loopback alone (:func:`server_bound_to_loopback`) — the
213 desktop app and ``scanpath-studio run`` bind ``127.0.0.1``; a bare
214 ``streamlit run``, which listens on every interface, does not persist unless
215 opted in.
217 ENG-56: this used to ask whether ``url`` was a loopback URL, and the app
218 passes ``st.context.url`` — which Streamlit copies from the browser's own
219 message. So any machine that could reach the port could claim to be at
220 ``http://localhost/`` and have the owner's cached datasets restored into its
221 session. The URL is still consulted **outside** a Streamlit server (the
222 ``cache`` CLI, the API, tests), where no browser supplies it and the caller
223 is describing where the app would be served.
224 """
225 env = os.environ if environ is None else environ
226 override = str(env.get(PERSIST_ENV_VAR, "")).strip().lower()
227 if override in {"1", "true", "yes", "on"}:
228 return True
229 if override in {"0", "false", "no", "off"}:
230 return False
231 try:
232 from streamlit import runtime
234 serving = runtime.exists()
235 except Exception: # pragma: no cover - streamlit is a hard dependency
236 serving = False
237 return server_bound_to_loopback() if serving else is_loopback_url(url)
240def state_directory(environ: dict | None = None) -> Path:
241 env = os.environ if environ is None else environ
242 configured = str(env.get(STATE_DIR_ENV_VAR, "")).strip()
243 if configured:
244 return Path(configured).expanduser()
245 return Path.home() / ".cache" / "scanpath-studio" / CACHE_DIR_NAME
248def _json_safe(value: Any) -> Any:
249 if value is None or isinstance(value, (str, int, float, bool)):
250 return value
251 if isinstance(value, dict):
252 return {str(k): _json_safe(v) for k, v in value.items()}
253 if isinstance(value, (list, tuple, set)):
254 return [_json_safe(v) for v in value]
255 if hasattr(value, "item"):
256 try:
257 return _json_safe(value.item())
258 except (TypeError, ValueError):
259 pass
260 return str(value)
263def _metadata_signature(session: MutableMapping[str, Any]) -> list:
264 """DATA-38 — the attached metadata tables' content fingerprint.
266 Imported here rather than at module level: `metadata` pulls in `data`, and
267 with it Streamlit, which :func:`cache_status` (the CLI's `cache` subcommand)
268 promises not to import.
269 """
270 from . import metadata as metadata_mod
272 return metadata_mod.store_signature(session)
275def _dataset_slug(name: str) -> str:
276 return hashlib.sha256(name.encode("utf-8")).hexdigest()[:20]
279def _dataset_identity(session: MutableMapping[str, Any]) -> list:
280 """Cheap session identity for datasets whose frames are immutable objects."""
281 datasets = []
282 for name, payload in sorted(dict(session.get("_datasets", {})).items()):
283 frames = []
284 for frame_key in _FRAME_KEYS:
285 frame = payload.get(frame_key)
286 frames.append(
287 (
288 frame_key,
289 id(frame),
290 tuple(frame.shape) if isinstance(frame, pd.DataFrame) else None,
291 )
292 )
293 metadata = {
294 k: _json_safe(v) for k, v in payload.items() if k not in _FRAME_KEYS
295 }
296 datasets.append((str(name), frames, metadata))
297 return datasets
300def _state_fingerprint(
301 session: MutableMapping[str, Any], metadata_signature: list | None = None
302) -> str:
303 """Cheap rerun fingerprint over datasets plus durable UI state."""
304 if metadata_signature is None:
305 metadata_signature = _metadata_signature(session)
306 datasets = _dataset_identity(session)
307 values = {
308 key: _json_safe(value)
309 for key, value in session.items()
310 if key in _SESSION_KEYS or str(key).startswith(COLUMN_MAPPING_PREFIX)
311 }
312 # DATA-48: the live store by content, every other dataset's by revision.
313 annotations = annotations_mod.store_signature(session)
314 encoded = json.dumps(
315 [
316 datasets,
317 values,
318 annotations,
319 metadata_signature,
320 sorted(_failed(session)),
321 # Round 10: resolving held-back metadata tables changes what the
322 # manifest says, so it is a change worth saving.
323 failed_metadata(session),
324 ],
325 ensure_ascii=False,
326 sort_keys=True,
327 )
328 return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
331def _atomic_parquet(frame: pd.DataFrame, destination: Path) -> None:
332 """Write one frame without exposing a partial Parquet file to readers."""
333 temporary: Path | None = None
334 try:
335 with tempfile.NamedTemporaryFile(
336 dir=destination.parent,
337 prefix=f".{destination.stem}.",
338 suffix=".parquet.tmp",
339 delete=False,
340 ) as handle:
341 temporary = Path(handle.name)
342 frame.to_parquet(temporary, index=False)
343 os.replace(temporary, destination)
344 temporary = None
345 finally:
346 if temporary is not None:
347 temporary.unlink(missing_ok=True)
350def _atomic_text(source: str, destination: Path) -> None:
351 """Atomically replace a UTF-8 text file using a unique sibling temporary."""
352 temporary: Path | None = None
353 try:
354 with tempfile.NamedTemporaryFile(
355 "w",
356 encoding="utf-8",
357 dir=destination.parent,
358 prefix=f".{destination.name}.",
359 suffix=".tmp",
360 delete=False,
361 ) as handle:
362 handle.write(source)
363 temporary = Path(handle.name)
364 os.replace(temporary, destination)
365 temporary = None
366 finally:
367 if temporary is not None:
368 temporary.unlink(missing_ok=True)
371def _manifest_for(
372 session: MutableMapping[str, Any], root: Path, *, reuse_datasets: bool
373) -> dict:
374 cached_entries = session.get(_LAST_DATASET_ENTRIES_KEY)
375 datasets = (
376 dict(cached_entries)
377 if reuse_datasets and isinstance(cached_entries, dict)
378 else {}
379 )
380 frames_dir = root / "datasets"
381 frames_dir.mkdir(parents=True, exist_ok=True)
382 if reuse_datasets:
383 # A manifest written before ``rows`` existed is still restorable, and
384 # restoring seeds these entries verbatim — so without this backfill an
385 # upgraded install would reuse row-less entries forever and the panel
386 # would read "1 dataset · 0 rows · 812 MB". Reuse means the live frames
387 # ARE the ones on disk (_dataset_identity matched), so counting them is
388 # accurate and free (len is O(1)); no Parquet is rewritten.
389 live = dict(session.get("_datasets", {}))
390 for name, entry in list(datasets.items()):
391 if isinstance(entry, dict) and not entry.get("rows"):
392 payload = live.get(name)
393 if isinstance(payload, dict):
394 datasets[name] = {
395 **entry,
396 "rows": {
397 frame_key: len(payload[frame_key])
398 for frame_key in _FRAME_KEYS
399 if isinstance(payload.get(frame_key), pd.DataFrame)
400 },
401 }
402 if not reuse_datasets:
403 for name, payload in dict(session.get("_datasets", {})).items():
404 slug = _dataset_slug(str(name))
405 metadata = {
406 k: _json_safe(v) for k, v in payload.items() if k not in _FRAME_KEYS
407 }
408 frame_files = {}
409 frame_rows = {}
410 for frame_key in _FRAME_KEYS:
411 frame = payload.get(frame_key)
412 if not isinstance(frame, pd.DataFrame):
413 frame = pd.DataFrame()
414 filename = f"{slug}-{frame_key}.parquet"
415 _atomic_parquet(frame, frames_dir / filename)
416 frame_files[frame_key] = f"datasets/{filename}"
417 frame_rows[frame_key] = len(frame)
418 # ``rows`` is reporting-only (cache_status / the in-app panel say how
419 # much is stored without opening the Parquet files). The restore path
420 # reads ``frames`` alone, so an older manifest without it still loads.
421 datasets[str(name)] = {
422 "metadata": metadata,
423 "frames": frame_files,
424 "rows": frame_rows,
425 }
427 # A stored dataset this session could not read goes back exactly as it was
428 # found — entry and files untouched — so a save never costs the cache a
429 # dataset it merely failed to open. A dataset of the same name added since
430 # is the user's newer work and takes the name (see `save_state`).
431 for name, record in _failed(session).items():
432 datasets.setdefault(name, record["entry"])
434 values = {key: _json_safe(session[key]) for key in _SESSION_KEYS if key in session}
435 values.update(
436 {
437 key: _json_safe(value)
438 for key, value in session.items()
439 if str(key).startswith(COLUMN_MAPPING_PREFIX)
440 }
441 )
442 return {
443 "schema": SCHEMA_VERSION,
444 "datasets": datasets,
445 "session": values,
446 # DATA-48: `{"datasets": {name: [records]}}` — each dataset's own.
447 "annotations": annotations_mod.cache_payload(session),
448 }
451def _save_metadata(
452 session: MutableMapping[str, Any], root: Path, signature: list
453) -> dict | None:
454 """DATA-38 — write the attached tables to :data:`METADATA_FILE` if they changed.
456 Returns the manifest's pointer to them, or ``None`` (and removes the file)
457 when nothing is attached. The tables are `metadata`'s JSON payloads; the
458 pointer is optional, so a manifest without it (every one written
459 before this) still restores, and the schema version does not move.
460 """
461 from . import metadata as metadata_mod
463 held = session.get(_FAILED_METADATA_KEY)
464 if isinstance(held, dict):
465 # Round 10: the stored tables did not read back. Writing this session's
466 # (often none) would replace or delete them, so the file and the
467 # manifest's pointer stay as they are until a retry or a discard.
468 pointer = held.get("pointer")
469 return dict(pointer) if isinstance(pointer, dict) else None
470 path = root / METADATA_FILE
471 if not signature:
472 path.unlink(missing_ok=True)
473 session.pop(_LAST_METADATA_SIGNATURE_KEY, None)
474 return None
475 pointer = session.get(_LAST_METADATA_POINTER_KEY)
476 if (
477 session.get(_LAST_METADATA_SIGNATURE_KEY) != signature
478 or not path.is_file()
479 or not isinstance(pointer, dict)
480 ):
481 # DATA-47: one entry per dataset, `{"datasets": {name: {grain: …}}}`.
482 datasets = metadata_mod.dataset_payloads(session)
483 if not datasets:
484 path.unlink(missing_ok=True)
485 session.pop(_LAST_METADATA_SIGNATURE_KEY, None)
486 session.pop(_LAST_METADATA_POINTER_KEY, None)
487 return None
488 # No `sort_keys`: each row keeps its columns in the table's own order.
489 encoded = json.dumps(_json_safe({"datasets": datasets}), ensure_ascii=False)
490 _atomic_text(encoded, path)
491 pointer = {
492 "file": METADATA_FILE,
493 "tables": [
494 f"{name}:{grain}"
495 for name, tables in datasets.items()
496 for grain in tables
497 ],
498 }
499 session[_LAST_METADATA_SIGNATURE_KEY] = signature
500 session[_LAST_METADATA_POINTER_KEY] = pointer
501 return pointer
504def save_state(session: MutableMapping[str, Any], root: Path) -> bool:
505 """Atomically save local datasets and durable session preferences."""
506 with _STATE_LOCK:
507 failed = _failed(session)
508 live_names = {str(name) for name in dict(session.get("_datasets", {}))}
509 if failed.keys() & live_names:
510 # Re-added under the same name: the new dataset is the one to keep.
511 _set_failed(
512 session, {k: v for k, v in failed.items() if k not in live_names}
513 )
514 metadata_signature = _metadata_signature(session)
515 fingerprint = _state_fingerprint(session, metadata_signature)
516 if session.get(_LAST_FINGERPRINT_KEY) == fingerprint:
517 return False
518 root.mkdir(parents=True, exist_ok=True)
519 dataset_identity = _dataset_identity(session)
520 reuse_datasets = session.get(
521 _LAST_DATASET_IDENTITY_KEY
522 ) == dataset_identity and isinstance(
523 session.get(_LAST_DATASET_ENTRIES_KEY), dict
524 )
525 manifest = _manifest_for(session, root, reuse_datasets=reuse_datasets)
526 # Written before the manifest, so a manifest never names a file that is
527 # not there yet.
528 metadata = _save_metadata(session, root, metadata_signature)
529 if metadata:
530 manifest["metadata"] = metadata
531 # DATA-32: the dataset table's remembered counts ride along with the
532 # datasets they describe — one small dict, and it is what stops a
533 # restored session recounting every corpus it has ever opened. Written
534 # here rather than inside `_manifest_for` because it is session state,
535 # not a frame on disk.
536 counts = session.get(DATASET_COUNTS_STORE_KEY)
537 if isinstance(counts, dict) and counts:
538 manifest["dataset_counts"] = _json_safe(counts)
539 encoded = json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2)
540 _atomic_text(encoded, root / "manifest.json")
541 session[_LAST_FINGERPRINT_KEY] = fingerprint
542 session[_LAST_DATASET_IDENTITY_KEY] = dataset_identity
543 # The live datasets' entries only: a held-back one is merged in by
544 # `_manifest_for` from its own record on every save, and must leave the
545 # manifest the moment that record is removed.
546 held_back = _failed(session)
547 session[_LAST_DATASET_ENTRIES_KEY] = {
548 name: entry
549 for name, entry in manifest["datasets"].items()
550 if name not in held_back
551 }
552 return True
555def restore_state(
556 session: MutableMapping[str, Any], root: Path, *, skip_session_keys=()
557) -> bool:
558 """Restore a manifest once, without overwriting already-seeded values.
560 Returns whether a manifest was found and applied — the *mechanism*. Whether
561 that is worth telling the user about is a different question, answered by
562 :func:`restored_summary` / :func:`restored_from_cache` and asked by
563 :func:`restore_local_state`; see UX-136 there.
564 """
565 if session.get(_RESTORED_KEY):
566 return False
567 session[_RESTORED_KEY] = True
568 with _STATE_LOCK:
569 return _restore_manifest(session, root, skip_session_keys)
572def _restore_manifest(
573 session: MutableMapping[str, Any], root: Path, skip_session_keys
574) -> bool:
575 """The body of :func:`restore_state`, under its lock."""
576 path = root / "manifest.json"
577 if not path.exists():
578 return False
579 try:
580 # BUG-71: read and check the manifest's *shape* before touching the
581 # session. It is a file on disk, so any JSON value can be in it —
582 # `[]`, `null`, a dataset entry that is a string — and each of those
583 # used to escape as an AttributeError on every launch.
584 manifest = _as_mapping(json.loads(path.read_text(encoding="utf-8")))
585 schema = int(manifest.get("schema", 0))
586 if schema != SCHEMA_VERSION:
587 raise ValueError(
588 f"it was saved by another version (format {schema}; this one "
589 f"reads {SCHEMA_VERSION})"
590 )
591 stored_datasets = _as_mapping(manifest.get("datasets", {}))
592 except (OSError, ValueError, TypeError, KeyError, AttributeError) as exc:
593 # The cache is there but cannot be read as a whole. Opening without
594 # it is right — a damaged cache must never stop the app — but
595 # overwriting it is not: the next save would replace whatever is
596 # still recoverable with this session's empty state. So saving
597 # pauses and the app says why (`cache_failure`), with a way to try
598 # again or to clear it.
599 session[_CACHE_FAILURE_KEY] = _failure_reason(exc)
600 session[_PAUSED_KEY] = True
601 _LOGGER.warning("Could not read the recovery cache at %s: %s", root, exc)
602 return False
604 # Each dataset restores on its own: one with a missing or damaged file
605 # costs that dataset — held back, entry and files kept — never the
606 # others, the settings, the annotations or the metadata tables.
607 restored_datasets = {}
608 stored_entries = {}
609 failed = {}
610 for index, (name, entry) in enumerate(stored_datasets.items(), start=1):
611 try:
612 restored_datasets[str(name)] = _read_dataset(root, entry)
613 stored_entries[str(name)] = entry
614 except Exception as exc: # any failure costs this dataset, nothing more
615 failed[str(name)] = {
616 "entry": _json_safe(entry),
617 "reason": _failure_reason(exc),
618 }
619 _LOGGER.warning(
620 "Could not restore the cached dataset %r: %s", str(name), exc
621 )
622 progress.report(index, len(stored_datasets), unit="datasets")
623 try:
624 stored_session = _restorable_session(manifest.get("session", {}))
625 except (ValueError, TypeError, AttributeError) as exc:
626 # Settings are rewritten from this session on the next save; a block
627 # that is not even an object has nothing in it worth keeping.
628 _LOGGER.warning("Ignored the recovery cache's settings: %s", exc)
629 stored_session = {}
630 existing = dict(session.get("_datasets", {}))
631 if restored_datasets:
632 session["_datasets"] = {**restored_datasets, **existing}
633 _set_failed(session, {k: v for k, v in failed.items() if k not in existing})
634 # Only the names that were not already open actually *landed* — an
635 # in-memory dataset of the same name shadows the stored one above.
636 summary = {"datasets": len(set(restored_datasets) - set(existing))}
637 skip = set(skip_session_keys)
638 # Counted before the loop below writes it: `setdefault` means a
639 # design library already in this session keeps its own, so the stored
640 # one restored nothing.
641 summary["designs"] = (
642 len(stored_session.get(DESIGN_PRESETS) or {})
643 if DESIGN_PRESETS not in skip and DESIGN_PRESETS not in session
644 else 0
645 )
646 for key, value in stored_session.items():
647 if key not in skip:
648 session.setdefault(key, value)
649 # DATA-48 — per dataset, `{"datasets": {name: [records]}}`. A
650 # manifest from before that holds one flat list naming no dataset;
651 # `restore_payload` hands it to the dataset this session opens on
652 # (the one the manifest had selected), once — see its docstring. A
653 # held-back dataset's annotations restore too: they are filed under
654 # its name and written back with it.
655 try:
656 summary["annotations"] = annotations_mod.restore_payload(
657 session, manifest.get("annotations")
658 )
659 except (ValueError, TypeError, KeyError, AttributeError) as exc:
660 _LOGGER.warning("Could not restore the cached annotations: %s", exc)
661 summary["annotations"] = 0
662 # DATA-38 — the attached metadata tables. Counted, because a user
663 # recognises their participant table coming back (UX-136's test for
664 # what is worth announcing), unlike a restored canvas width.
665 summary["metadata"] = _restore_metadata(session, root, manifest.get("metadata"))
666 # A clean restore can reuse the Parquet files on the first rendered
667 # settings change. Pre-existing in-memory datasets still need a save.
668 if not existing:
669 session[_LAST_DATASET_IDENTITY_KEY] = _dataset_identity(session)
670 session[_LAST_DATASET_ENTRIES_KEY] = stored_entries
671 counts = manifest.get("dataset_counts")
672 if isinstance(counts, dict):
673 # DATA-32 — a manifest written before this existed simply has
674 # none, and the table recounts what it can, as it always did.
675 session[DATASET_COUNTS_STORE_KEY] = dict(counts)
676 # Separate from _RESTORED_KEY, which only records that a restore was
677 # *attempted* this session. The app reads this one to tell the user
678 # their previous session came back (restored_from_cache), so it holds
679 # the summary rather than a bare flag — see the docstring.
680 session[_RESTORED_PAYLOAD_KEY] = summary
681 return True
684def _failure_reason(exc: BaseException) -> str:
685 """A short, user-facing reason for a part of the cache that would not read."""
686 if isinstance(exc, FileNotFoundError):
687 name = Path(str(exc.filename or "")).name
688 return f"a stored file is missing ({name})" if name else "a file is missing"
689 if isinstance(exc, PermissionError):
690 return "a stored file can't be opened (permission denied)"
691 if isinstance(exc, json.JSONDecodeError):
692 return "its index file (manifest.json) is damaged"
693 message = str(exc).strip()
694 if isinstance(exc, ValueError) and message.startswith("it "):
695 return message
696 # The class and message are for a bug report, not for the warning.
697 _LOGGER.warning("Saved data unreadable: %s: %s", type(exc).__name__, message)
698 return "it can't be read (damaged, or written by another version)"
701def _frame_path(root: Path, relative: Any) -> Path:
702 """Where a manifest entry's frame lives — only ever inside the cache folder."""
703 base = (root / "datasets").resolve()
704 path = (root / str(relative)).resolve()
705 if path.parent != base:
706 raise ValueError(f"it names a file outside the cache folder ({relative})")
707 return path
710def _read_dataset(root: Path, entry: Any) -> dict:
711 """One stored dataset's payload, frames read — or the exception that stopped it."""
712 entry = _as_mapping(entry)
713 payload = dict(_as_mapping(entry.get("metadata", {})))
714 for frame_key, relative in _as_mapping(entry.get("frames", {})).items():
715 payload[frame_key] = pd.read_parquet(_frame_path(root, relative))
716 for frame_key in _FRAME_KEYS:
717 payload.setdefault(frame_key, pd.DataFrame())
718 return payload
721def _failed(session: MutableMapping[str, Any]) -> dict:
722 value = session.get(_FAILED_DATASETS_KEY)
723 return dict(value) if isinstance(value, dict) else {}
726def _set_failed(session: MutableMapping[str, Any], failed: dict) -> None:
727 if failed:
728 session[_FAILED_DATASETS_KEY] = dict(failed)
729 else:
730 session.pop(_FAILED_DATASETS_KEY, None)
733def failed_datasets(session) -> dict[str, str]:
734 """The cached datasets this session could not read back: ``{name: reason}``.
736 Their entries and files stay in the cache — every save writes them back as
737 they were — until :func:`retry_failed_datasets` reads them or
738 :func:`discard_failed_dataset` removes them.
739 """
740 return {
741 name: str(record.get("reason", "")) for name, record in _failed(session).items()
742 }
745def cache_failure(session) -> str | None:
746 """Why this session could not read the cache at all, or ``None``.
748 Set only when a manifest *exists* and failed to read; saving is paused for
749 the session then, so the stored copy is not replaced. An absent cache — never
750 written, or cleared on purpose — is not a failure and saves normally.
751 """
752 reason = session.get(_CACHE_FAILURE_KEY)
753 return str(reason) if reason else None
756def retry_failed_datasets(session, root: Path | None = None) -> dict[str, str]:
757 """Try the held-back datasets again; returns those that still fail.
759 One that now reads joins the session's datasets (a dataset of the same name
760 opened since keeps its place, and the stored copy is dropped from the
761 held-back list).
762 """
763 directory = state_directory() if root is None else root
764 failed = _failed(session)
765 if not failed:
766 return {}
767 with _STATE_LOCK:
768 live = dict(session.get("_datasets", {}))
769 entries = session.get(_LAST_DATASET_ENTRIES_KEY)
770 # The reuse bookkeeping stays valid only if it described the session
771 # before these were added; extend it rather than force a full rewrite.
772 reusable = isinstance(entries, dict) and session.get(
773 _LAST_DATASET_IDENTITY_KEY
774 ) == _dataset_identity(session)
775 recovered = {}
776 for name, record in list(failed.items()):
777 if name in live:
778 failed.pop(name)
779 continue
780 try:
781 payload = _read_dataset(directory, record.get("entry"))
782 except Exception as exc: # still unreadable: it stays held back
783 failed[name] = {**record, "reason": _failure_reason(exc)}
784 continue
785 live[name] = payload
786 recovered[name] = record.get("entry")
787 failed.pop(name)
788 if recovered:
789 session["_datasets"] = live
790 if reusable:
791 session[_LAST_DATASET_ENTRIES_KEY] = {**entries, **recovered}
792 session[_LAST_DATASET_IDENTITY_KEY] = _dataset_identity(session)
793 _set_failed(session, failed)
794 return failed_datasets(session)
797def discard_failed_dataset(session, name: str, root: Path | None = None) -> bool:
798 """Remove one held-back dataset from the cache: its entry and its files.
800 The next save writes the manifest without it. Its annotations and metadata
801 tables are kept — they are filed by name, like every dataset's, and are the
802 user's own work — so a dataset added again under that name finds them.
803 """
804 directory = state_directory() if root is None else root
805 failed = _failed(session)
806 record = failed.pop(str(name), None)
807 if record is None:
808 return False
809 with _STATE_LOCK:
810 entry = record.get("entry")
811 frames = entry.get("frames") if isinstance(entry, dict) else None
812 paths = {
813 directory / "datasets" / f"{_dataset_slug(str(name))}-{key}.parquet"
814 for key in _FRAME_KEYS
815 }
816 for relative in (frames or {}).values() if isinstance(frames, dict) else ():
817 try:
818 paths.add(_frame_path(directory, relative))
819 except ValueError:
820 continue # never delete outside the cache folder
821 for path in paths:
822 _unlink_quietly(path)
823 _set_failed(session, failed)
824 return True
827def retry_cache_restore(session, url: str) -> bool:
828 """Try a cache that could not be read again, from scratch, this session.
830 Clears the failure and the pause it set, and restores as at launch — what a
831 reload would do, without losing what the session already holds.
832 """
833 session.pop(_CACHE_FAILURE_KEY, None)
834 session.pop(_PAUSED_KEY, None)
835 session.pop(_RESTORED_KEY, None)
836 return restore_local_state(session, url)
839def _as_mapping(value: Any) -> dict:
840 """``value`` if it is a JSON object, else ``ValueError`` (BUG-71)."""
841 if not isinstance(value, dict):
842 raise ValueError(
843 f"expected an object in the manifest, got {type(value).__name__}"
844 )
845 return value
848def _restorable_session(stored: Any) -> dict:
849 """The manifest's ``session`` block, cut down to what is safe to seed (BUG-71).
851 Only keys this module writes are restored — the durable settings and the
852 column mapping — so a hand-edited or foreign manifest cannot seed arbitrary
853 session state. Each value is held to its widget's rules by
854 ``url_state.sanitize_session_value`` (clamped into range, or dropped): a
855 restored value is seeded before its widget renders, exactly like a deep link,
856 so an opacity of 7 or a colour of ``"zzz"`` stopped the app on every launch.
857 Dropping it costs the user that one setting. Imported here, not at the top,
858 because ``url_state`` pulls in Streamlit and the cache CLI must not.
859 """
860 from .url_state import sanitize_session_value
862 clean = {}
863 # Renamed keys move to their new names before the allow-list sees them.
864 for key, value in rename_legacy_keys(_as_mapping(stored)).items():
865 if not isinstance(key, str) or not (
866 key in _SESSION_KEYS or key.startswith(COLUMN_MAPPING_PREFIX)
867 ):
868 continue
869 if key == DATASET_DESCRIPTIONS_KEY:
870 # UX-174 r2 — ``{dataset: sentence}``; anything else is dropped.
871 if isinstance(value, dict):
872 clean[key] = {
873 str(name): str(text)
874 for name, text in value.items()
875 if isinstance(text, str)
876 }
877 continue
878 if key == DOWNLOAD_DIR_KEY:
879 # UX-184 — a path seeds a text box: only a string may.
880 if isinstance(value, str):
881 clean[key] = value
882 continue
883 if key == DATASET_SETUP_OVERRIDES_KEY:
884 # ``{dataset: setup}``, each read back through `SetupSnapshot`,
885 # which degrades a bad field rather than the whole setup.
886 if isinstance(value, dict):
887 from .experimental_setup import SetupSnapshot
889 clean[key] = {
890 str(name): SetupSnapshot.from_dict(setup).to_dict()
891 for name, setup in value.items()
892 if isinstance(setup, dict)
893 }
894 continue
895 if key == SETUP_OVERRIDE_FOR_KEY:
896 if isinstance(value, str):
897 clean[key] = value
898 continue
899 if key == SETUP_OVERRIDE_RESTORE_KEY:
900 # ``{global_* key: value or None}`` — each value held to its own
901 # control's rules, and a key that is not a setup key dropped.
902 if isinstance(value, dict):
903 restore = {}
904 for name, saved in value.items():
905 if name not in SETUP_OVERRIDE_SESSION_KEYS:
906 continue
907 try:
908 restore[name] = (
909 None
910 if saved is None
911 else sanitize_session_value(name, saved)
912 )
913 except (TypeError, ValueError, OverflowError):
914 continue
915 clean[key] = restore
916 continue
917 if key == DESIGN_PRESETS:
918 # The design library is the user's own work: keep every well-formed
919 # design rather than all-or-nothing.
920 # One saved before the fixed duration scale keeps the relative one.
921 if isinstance(value, dict):
922 from .controls import sanitize_design
924 clean[key] = {
925 str(name): sanitize_design(
926 keep_legacy_marker_scale(rename_legacy_keys(design))
927 )[0]
928 for name, design in value.items()
929 if isinstance(design, dict)
930 }
931 continue
932 try:
933 clean[key] = sanitize_session_value(key, value)
934 except (TypeError, ValueError, OverflowError):
935 _LOGGER.warning(
936 "Dropped %s from the recovery cache: %.80r is not a value its "
937 "control accepts.",
938 key,
939 value,
940 )
941 # A session saved before the fixed duration scale reopens on the relative
942 # one it was drawn with, as an old design or Share link does.
943 return keep_legacy_marker_scale(clean)
946def _restore_metadata(session: MutableMapping[str, Any], root: Path, pointer) -> int:
947 """DATA-38 — re-attach the tables :func:`_save_metadata` wrote; how many.
949 DATA-47: they come back into the per-dataset store, each dataset's own, and
950 reach the session keys when that dataset is selected. Its own error
951 boundary: a missing or unreadable sidecar costs the tables, never the
952 datasets and settings the rest of the manifest restores — and it is held
953 back (:func:`failed_metadata`), so no save replaces it (round 10).
954 """
955 if not isinstance(pointer, dict) or not pointer.get("file"):
956 return 0
957 from . import metadata as metadata_mod
959 try:
960 payloads = _read_metadata_file(root, pointer)
961 return metadata_mod.restore_dataset_payloads(session, payloads)
962 except Exception as exc: # any failure costs the tables, nothing more
963 session[_FAILED_METADATA_KEY] = {
964 "pointer": _json_safe(pointer),
965 "reason": _metadata_failure_reason(exc),
966 }
967 _LOGGER.warning("Could not restore the cached metadata tables: %s", exc)
968 return 0
971def _read_metadata_file(root: Path, pointer: Any) -> Any:
972 """The metadata sidecar's JSON, from inside the cache folder only."""
973 name = str(_as_mapping(pointer).get("file") or "")
974 path = (root / name).resolve()
975 if path.parent != root.resolve():
976 raise ValueError(f"it names a file outside the cache folder ({name})")
977 return json.loads(path.read_text("utf-8"))
980def _metadata_failure_reason(exc: BaseException) -> str:
981 if isinstance(exc, json.JSONDecodeError):
982 return "its file is not valid JSON"
983 if isinstance(exc, FileNotFoundError):
984 return f"its file is missing ({METADATA_FILE})"
985 return _failure_reason(exc)
988def failed_metadata(session) -> str | None:
989 """Why the cached metadata tables did not read back, or ``None``."""
990 held = session.get(_FAILED_METADATA_KEY)
991 return str(held.get("reason", "")) if isinstance(held, dict) else None
994def retry_failed_metadata(session, root: Path | None = None) -> str | None:
995 """Read the held-back metadata tables again; the reason if they still fail.
997 Tables that read join the store (a dataset holding tables of its own keeps
998 them), and saving them resumes.
999 """
1000 held = session.get(_FAILED_METADATA_KEY)
1001 if not isinstance(held, dict):
1002 return None
1003 directory = state_directory() if root is None else root
1004 from . import metadata as metadata_mod
1006 with _STATE_LOCK:
1007 try:
1008 payloads = _read_metadata_file(directory, held.get("pointer"))
1009 metadata_mod.restore_dataset_payloads(session, payloads)
1010 except Exception as exc: # still unreadable: it stays held back
1011 reason = _metadata_failure_reason(exc)
1012 session[_FAILED_METADATA_KEY] = {**held, "reason": reason}
1013 return reason
1014 session.pop(_FAILED_METADATA_KEY, None)
1015 # Written afresh on the next save, from the store as it now stands.
1016 session.pop(_LAST_METADATA_SIGNATURE_KEY, None)
1017 return None
1020def discard_failed_metadata(session, root: Path | None = None) -> bool:
1021 """Delete the held-back metadata tables' file; saving them resumes.
1023 Only that copy: the datasets, their annotations and the tables attached
1024 this session are left alone, and the next save writes the latter.
1025 """
1026 held = session.pop(_FAILED_METADATA_KEY, None)
1027 if not isinstance(held, dict):
1028 return False
1029 directory = state_directory() if root is None else root
1030 with _STATE_LOCK:
1031 _unlink_quietly(directory / METADATA_FILE)
1032 session.pop(_LAST_METADATA_SIGNATURE_KEY, None)
1033 session.pop(_LAST_METADATA_POINTER_KEY, None)
1034 return True
1037def forget_state(root: Path) -> None:
1038 """Remove the known persistence files without recursively deleting ``root``."""
1039 with _STATE_LOCK:
1040 manifest = root / "manifest.json"
1041 if manifest.exists():
1042 manifest.unlink()
1043 (root / RESTORE_MARKER_NAME).unlink(missing_ok=True)
1044 (root / METADATA_FILE).unlink(missing_ok=True)
1045 frames_dir = root / "datasets"
1046 if frames_dir.is_dir():
1047 for path in frames_dir.glob("*.parquet"):
1048 path.unlink()
1049 try:
1050 frames_dir.rmdir()
1051 except OSError:
1052 pass
1055def _reslug_entry(entry: Any, slug: str) -> dict:
1056 """One manifest dataset entry with its frame paths moved onto ``slug``."""
1057 frames = {
1058 frame_key: f"datasets/{slug}-{frame_key}.parquet"
1059 for frame_key in dict(entry.get("frames", {}))
1060 }
1061 return {**dict(entry), "frames": frames}
1064def rename_cached_dataset(
1065 session: MutableMapping[str, Any],
1066 old: str,
1067 new: str,
1068 root: Path | None = None,
1069) -> bool:
1070 """Follow a dataset rename (DATA-23) through the cache instead of rewriting it.
1072 A dataset's Parquet files are named after ``_dataset_slug(name)``, so a rename
1073 would otherwise leave the old slug's files behind as orphans nothing deletes,
1074 and make the next save re-encode every frame under the new slug. Renaming the
1075 files, re-keying ``manifest.json`` and re-keying this session's reuse
1076 bookkeeping keeps the next :func:`save_state` on the cheap ``reuse_datasets``
1077 path — it rewrites the manifest only.
1079 Call it **after** the store itself has been re-keyed: the new reuse identity is
1080 read from the live ``session["_datasets"]``. Best-effort like the rest of this
1081 module — on any failure the session's bookkeeping is dropped so the next save
1082 rebuilds the cache in full rather than trusting a half-moved one.
1083 """
1084 if old == new:
1085 return False
1086 directory = state_directory() if root is None else root
1087 with _STATE_LOCK:
1088 try:
1089 old_slug, new_slug = _dataset_slug(old), _dataset_slug(new)
1090 frames_dir = directory / "datasets"
1091 for frame_key in _FRAME_KEYS:
1092 source = frames_dir / f"{old_slug}-{frame_key}.parquet"
1093 if source.is_file():
1094 os.replace(source, frames_dir / f"{new_slug}-{frame_key}.parquet")
1095 manifest_path = directory / "manifest.json"
1096 if manifest_path.is_file():
1097 manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
1098 datasets = dict(manifest.get("datasets", {}))
1099 if old in datasets:
1100 datasets[new] = _reslug_entry(datasets.pop(old), new_slug)
1101 manifest["datasets"] = datasets
1102 values = dict(manifest.get("session", {}))
1103 if values.get("data_source_choice") == old:
1104 values["data_source_choice"] = new
1105 manifest["session"] = values
1106 _atomic_text(
1107 json.dumps(
1108 manifest, ensure_ascii=False, sort_keys=True, indent=2
1109 ),
1110 manifest_path,
1111 )
1112 entries = session.get(_LAST_DATASET_ENTRIES_KEY)
1113 if isinstance(entries, dict) and old in entries:
1114 entries = dict(entries)
1115 entries[new] = _reslug_entry(entries.pop(old), new_slug)
1116 session[_LAST_DATASET_ENTRIES_KEY] = entries
1117 session[_LAST_DATASET_IDENTITY_KEY] = _dataset_identity(session)
1118 return True
1119 except (OSError, ValueError, TypeError, KeyError):
1120 for key in (
1121 _LAST_FINGERPRINT_KEY,
1122 _LAST_DATASET_IDENTITY_KEY,
1123 _LAST_DATASET_ENTRIES_KEY,
1124 ):
1125 session.pop(key, None)
1126 _LOGGER.warning(
1127 "Could not follow a dataset rename through the local cache.",
1128 exc_info=True,
1129 )
1130 return False
1133def restore_local_state(
1134 session, url: str, *, protect_data_source: bool = False
1135) -> bool:
1136 """Restore this machine's cached session, once, and say whether to announce it.
1138 UX-136: the return value is **not** "a manifest was applied" — that is
1139 :func:`restore_state`'s answer, and the app used to toast "Recovered your
1140 last session from this computer" on it. Every rerun writes the cache, so a
1141 session that has only ever changed view settings still leaves a manifest
1142 behind, and restoring one announced a recovery that the user could not see
1143 anywhere — worst of all immediately after they had cleared the cache by
1144 hand, where it reads as "clearing it did nothing". So the announcement is
1145 gated on something a user would *recognise* coming back: a dataset they
1146 added, an annotation they wrote, a design they saved. Settings still restore;
1147 they just do it silently, because nobody can tell a restored canvas width
1148 from the default one.
1149 """
1150 if not persistence_enabled(url):
1151 session[_RESTORED_KEY] = True
1152 return False
1153 if session.get(_RESTORED_KEY):
1154 return False
1155 root = state_directory()
1156 marker = root / RESTORE_MARKER_NAME
1157 if marker.is_file():
1158 # BUG-71: the last session that applied this cache never finished a run,
1159 # and a restore that breaks the app breaks it before the Data page can
1160 # offer a reset — so every launch would break again. Open
1161 # without it, once. The files stay, and saving is paused so this
1162 # session's (empty) state cannot overwrite them; the marker goes, so a
1163 # reload tries again — one strike, because a tab closed mid-way through a
1164 # slow first run leaves the marker too, and that must not cost the cache.
1165 session[_RESTORED_KEY] = True
1166 session[_RESTORE_SKIPPED_KEY] = True
1167 session[_PAUSED_KEY] = True
1168 _unlink_quietly(marker)
1169 _LOGGER.warning(
1170 "The previous session never finished opening with the recovery cache "
1171 "at %s; opened without restoring it (the files are kept).",
1172 root,
1173 )
1174 return False
1175 if (root / "manifest.json").is_file():
1176 try:
1177 marker.write_text(datetime.now().isoformat(timespec="seconds"), "utf-8")
1178 session[_RESTORE_PENDING_KEY] = str(marker)
1179 except OSError:
1180 pass # a cache we cannot write to simply goes without the breaker
1181 protected = {"data_source_choice"} if protect_data_source else set()
1182 if not restore_state(session, root, skip_session_keys=protected):
1183 _finish_restore(session)
1184 return False
1185 return restored_from_cache(session)
1188def local_state_restored(session) -> bool:
1189 """Whether this session has had its one restore attempt already.
1191 Set by :func:`restore_local_state` on its first call whatever the outcome —
1192 restored, nothing cached, persistence off, or skipped (BUG-71) — and every
1193 later call returns at once. The app opens the restore card only before it
1194 (UX-166): a card around a call that does nothing cost each run a timer
1195 thread and a task for nothing.
1196 """
1197 return bool(session.get(_RESTORED_KEY))
1200def _unlink_quietly(path: Path) -> None:
1201 try:
1202 path.unlink(missing_ok=True)
1203 except OSError:
1204 _LOGGER.warning("Could not remove %s.", path)
1207def _finish_restore(session) -> None:
1208 """This session's restore held (or never happened): drop its marker."""
1209 marker = session.pop(_RESTORE_PENDING_KEY, None)
1210 if marker:
1211 _unlink_quietly(Path(marker))
1214def consume_restore_skipped(session) -> bool:
1215 """Whether this session opened without its cache because the last one broke.
1217 One-shot, so the app says it once: the notice belongs to the launch, not to
1218 every rerun after it. See :func:`restore_local_state` (BUG-71).
1219 """
1220 return bool(session.pop(_RESTORE_SKIPPED_KEY, False))
1223def restored_summary(session) -> dict:
1224 """How much of *what* this session got back from the cache, by kind.
1226 ``{"datasets": n, "annotations": n, "designs": n, "metadata": n}`` — what
1227 the *Saved on this computer* section counts, and what a user would recognise as
1228 their last session (``metadata`` is the number of attached participant /
1229 trial / text tables, DATA-38). View settings are deliberately absent: they restore, but
1230 silently (UX-136 — see :func:`restore_state`). Empty when no restore
1231 happened, and after :func:`clear_local_state`.
1232 """
1233 summary = session.get(_RESTORED_PAYLOAD_KEY)
1234 return dict(summary) if isinstance(summary, dict) else {}
1237def session_was_restored(session) -> bool:
1238 """Whether this session applied a saved manifest at all (#374 F32).
1240 Wider than :func:`restored_from_cache`: settings alone count, since they
1241 still say the app was used here before. The welcome tour reads it."""
1242 return isinstance(session.get(_RESTORED_PAYLOAD_KEY), dict)
1245def restored_from_cache(session) -> bool:
1246 """Whether this session got back something the user would recognise.
1248 Not "was a manifest read" — see :func:`restore_state` for why the two came
1249 apart.
1250 """
1251 return any(restored_summary(session).values())
1254def persistence_paused(session) -> bool:
1255 """Whether saving is paused for this session.
1257 Set by :func:`restore_local_state`'s BUG-71 breaker, after a launch that
1258 never finished opening with the cache, and by a manifest that exists but
1259 cannot be read (:func:`cache_failure`): this session then leaves the stored
1260 copy untouched, and a reload — or :func:`retry_cache_restore` — tries it
1261 again. :func:`clear_local_state` lifts it.
1262 """
1263 return bool(session.get(_PAUSED_KEY))
1266def clear_local_state(session=None, root: Path | None = None) -> bool:
1267 """Delete the stored cache and forget what this session had written.
1269 The in-memory datasets are deliberately left alone — this removes the copy
1270 on disk, it does not close the user's work.
1272 Deleting the files is best-effort: a locked or read-only cache directory
1273 must not wedge ``scanpath-studio cache --clear``, which a user reaches for
1274 precisely when the session is already broken. An :class:`OSError` is logged and reported as ``False``; the
1275 in-session bookkeeping is cleared either way.
1276 """
1277 removed = True
1278 try:
1279 forget_state(state_directory() if root is None else root)
1280 except OSError as exc:
1281 removed = False
1282 _LOGGER.warning("Could not delete the recovery cache: %s", exc)
1283 if session is not None:
1284 for key in (
1285 _LAST_FINGERPRINT_KEY,
1286 _LAST_DATASET_IDENTITY_KEY,
1287 _LAST_DATASET_ENTRIES_KEY,
1288 _RESTORED_PAYLOAD_KEY,
1289 _LAST_METADATA_SIGNATURE_KEY,
1290 _LAST_METADATA_POINTER_KEY,
1291 # DATA-32: the remembered counts are part of what "forget this
1292 # session" means — the ask named clearing the cache explicitly.
1293 DATASET_COUNTS_STORE_KEY,
1294 # Nothing is left to protect or to retry: a cleared cache saves
1295 # normally, like one that never existed.
1296 _FAILED_DATASETS_KEY,
1297 _FAILED_METADATA_KEY,
1298 _CACHE_FAILURE_KEY,
1299 _PAUSED_KEY,
1300 ):
1301 session.pop(key, None)
1302 return removed
1305def clear_saved_work(session) -> bool:
1306 """Delete what is saved on this computer and start over.
1308 The Data page's *Clear what is saved…* (#374 F33). Deleting the files alone
1309 would be undone within a click, because every run ends in
1310 ``save_local_state`` and this tab still holds the datasets, annotations and
1311 settings. So the session is emptied too: the next run is a first visit (the
1312 default dataset and settings), and saving carries on from there as usual.
1313 Returns whether the files were removed.
1314 """
1315 removed = clear_local_state(session)
1316 session.clear()
1317 return removed
1320def _cache_files(root: Path) -> list:
1321 """The files this module owns under ``root`` (mirrors forget_state)."""
1322 files = [root / "manifest.json", root / RESTORE_MARKER_NAME, root / METADATA_FILE]
1323 frames_dir = root / "datasets"
1324 if frames_dir.is_dir():
1325 files.extend(sorted(frames_dir.glob("*.parquet")))
1326 return [path for path in files if path.is_file()]
1329def human_size(num_bytes: int) -> str:
1330 """Format a byte count for the cache panel / CLI (1 decimal, binary units)."""
1331 size = float(max(int(num_bytes), 0))
1332 for unit in ("B", "KB", "MB", "GB"):
1333 if size < 1024 or unit == "GB":
1334 return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
1335 size /= 1024
1336 return f"{size:.1f} GB" # pragma: no cover - unreachable, loop returns first
1339def cache_status(
1340 root: Path | None = None,
1341 *,
1342 url: str = "",
1343 environ: dict | None = None,
1344) -> dict:
1345 """Describe the on-device recovery cache — the whole read-only surface.
1347 Streamlit-free on purpose: the same dict backs the in-app panel, the
1348 ``scanpath-studio cache`` subcommand and ``api.cache_status``. Reading is
1349 best-effort — a partial or corrupt manifest reports ``readable=False``
1350 rather than raising, exactly as ``restore_state`` refuses to break the app
1351 over one. ``rows`` is ``None`` (not ``0``) when a stored dataset predates
1352 the manifest's ``rows`` field: "0 rows · 812 MB on disk" would be a lie,
1353 "size only" is the truth. The next save backfills it.
1354 """
1355 env = os.environ if environ is None else environ
1356 override = str(env.get(PERSIST_ENV_VAR, "")).strip().lower()
1357 directory = state_directory(env) if root is None else Path(root)
1358 manifest_path = directory / "manifest.json"
1359 status = {
1360 "enabled": persistence_enabled(url, env),
1361 "override": (
1362 "on"
1363 if override in {"1", "true", "yes", "on"}
1364 else "off"
1365 if override in {"0", "false", "no", "off"}
1366 else ""
1367 ),
1368 "directory": str(directory),
1369 "exists": manifest_path.is_file(),
1370 "readable": False,
1371 "schema": None,
1372 "datasets": [],
1373 "rows": 0,
1374 "annotations": 0,
1375 "designs": 0,
1376 "metadata": 0,
1377 "settings": 0,
1378 "bytes": 0,
1379 "saved_at": None,
1380 # Stored datasets the app cannot restore: ``[{"name", "reason"}]``.
1381 "damaged": [],
1382 # Why the stored metadata tables cannot restore, or "" (round 10).
1383 "damaged_metadata": "",
1384 }
1385 if not directory.is_dir():
1386 # The common hosted case: one stat, then out — no glob over a folder
1387 # this deployment never creates.
1388 return status
1389 status["bytes"] = sum(path.stat().st_size for path in _cache_files(directory))
1390 if not status["exists"]:
1391 return status
1392 try:
1393 status["saved_at"] = datetime.fromtimestamp(
1394 manifest_path.stat().st_mtime
1395 ).isoformat(timespec="seconds")
1396 manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
1397 schema = int(manifest.get("schema", 0))
1398 status["schema"] = schema
1399 datasets = dict(manifest.get("datasets", {}))
1400 # Each stored dataset is described on its own, and one that cannot be
1401 # restored — an entry of the wrong shape, a file that is gone — is
1402 # listed under `damaged` with the reason, as the app's restore holds it
1403 # back. A stat per file, never a Parquet read: this runs on every
1404 # render of the Data page. (A file that is present but corrupt shows
1405 # only when the app tries to read it.)
1406 good, damaged = [], []
1407 for name, entry in sorted(datasets.items()):
1408 reason = _entry_problem(directory, entry)
1409 if reason:
1410 damaged.append({"name": str(name), "reason": reason})
1411 continue
1412 good.append(
1413 {
1414 "name": str(name),
1415 "rows": {
1416 key: int(value)
1417 for key, value in dict(entry.get("rows") or {}).items()
1418 },
1419 }
1420 )
1421 status["datasets"] = good
1422 status["damaged"] = damaged
1423 status["rows"] = (
1424 sum(sum(entry["rows"].values()) for entry in status["datasets"])
1425 if all(entry["rows"] for entry in status["datasets"])
1426 else None
1427 )
1428 status["annotations"] = annotations_mod.payload_count(
1429 manifest.get("annotations")
1430 )
1431 stored_session = dict(manifest.get("session", {}))
1432 status["designs"] = len(dict(stored_session.get(DESIGN_PRESETS, {})))
1433 pointer = dict(manifest.get("metadata") or {})
1434 status["metadata"] = len(list(pointer.get("tables") or []))
1435 if pointer:
1436 status["damaged_metadata"] = _metadata_file_problem(directory, pointer)
1437 status["settings"] = len(stored_session)
1438 # A newer/unknown schema is present but will not restore — say so here
1439 # rather than let the panel claim the work is safely stored.
1440 status["readable"] = schema == SCHEMA_VERSION
1441 except (OSError, ValueError, TypeError, KeyError, AttributeError):
1442 status["readable"] = False
1443 return status
1446def _metadata_file_problem(root: Path, pointer: dict) -> str:
1447 """Why the manifest's metadata tables cannot restore, by a ``stat``; ``""``
1448 when nothing is wrong that one can see (a corrupt file shows only when the
1449 app reads it, as for datasets)."""
1450 name = str(pointer.get("file") or "")
1451 path = (root / name).resolve()
1452 if not name or path.parent != root.resolve():
1453 return "its entry in the manifest is damaged"
1454 if not path.is_file():
1455 return f"its file is missing ({name})"
1456 return ""
1459def _entry_problem(root: Path, entry: Any) -> str:
1460 """Why a manifest dataset entry cannot restore, by its shape and files.
1462 ``""`` when nothing is wrong that a ``stat`` can see.
1463 """
1464 if not isinstance(entry, dict) or not isinstance(entry.get("frames", {}), dict):
1465 return "its entry in the manifest is damaged"
1466 for relative in entry.get("frames", {}).values():
1467 try:
1468 path = _frame_path(root, relative)
1469 except ValueError as exc:
1470 return str(exc)
1471 if not path.is_file():
1472 return f"a stored file is missing ({path.name})"
1473 return ""
1476def save_local_state(session, url: str) -> bool:
1477 # BUG-71: reaching the epilogue means this run rendered, so a restore it
1478 # applied is not the kind that breaks the app — before any early return, since
1479 # a paused save still ran to here.
1480 _finish_restore(session)
1481 if not persistence_enabled(url) or persistence_paused(session):
1482 return False
1483 try:
1484 return save_state(session, state_directory())
1485 except Exception:
1486 # Persistence is a recovery convenience, never a reason for the app to
1487 # stop rendering (read-only homes, full disks, or unsupported Parquet
1488 # values are all recoverable by continuing without this save).
1489 _LOGGER.warning(
1490 "Could not persist the local Scanpath Studio session.", exc_info=True
1491 )
1492 return False