Coverage for scanpath_studio/data_health.py: 100%
82 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-07 21:10 +0000
1"""Values that parsed as numbers but cannot be what they claim to be.
3The load already reports cells that are not numbers at all
4(`data.numeric_parse_issues`) and trial ids that merge several readings
5(`data.diagnose_trial_identity`). A value can pass both and still be unusable:
6a fixation lasting −40 ms, a gaze position at infinity, a word box with no
7width. Nothing here changes or drops a row — each check counts the rows, names
8the columns they came from, shows a few, and says what the app does with them,
9so the researcher can decide.
11Pure (no Streamlit): the 🗂️ Data page draws :func:`check_data_health` through a
12cached wrapper, and `api.check_data_health` returns the same findings as a table.
13"""
15from __future__ import annotations
17from collections.abc import Callable
18from dataclasses import dataclass, field
20import numpy as np
21import pandas as pd
23from .multipart import SCREEN_ID
25#: How many offending rows a finding quotes.
26EXAMPLE_ROWS = 3
28#: The identity a quoted example row is shown with, when the table has it.
29_EXAMPLE_KEYS = ("participant_id", "trial_id", SCREEN_ID)
32@dataclass(frozen=True)
33class HealthFinding:
34 """One check that found rows: how many, where, and what happens to them."""
36 check: str
37 table: str
38 title: str
39 columns: tuple[str, ...]
40 rows: int
41 total_rows: int
42 trials: int
43 breakdown: dict[str, int] = field(default_factory=dict)
44 examples: tuple[dict, ...] = ()
45 consequence: str = ""
46 #: ``"warning"``, or ``"note"`` for what is ordinary in such data (raw-gaze
47 #: samples whose only gap is a missing position: blinks, track loss).
48 severity: str = "warning"
50 def to_record(self) -> dict:
51 """A flat row for a table (`api.check_data_health`)."""
52 return {
53 "table": self.table,
54 "check": self.check,
55 "problem": self.title,
56 "columns": ", ".join(self.columns),
57 "rows": self.rows,
58 "of_rows": self.total_rows,
59 "trials": self.trials,
60 "severity": self.severity,
61 "breakdown": ", ".join(f"{n:,} {k}" for k, n in self.breakdown.items()),
62 "what_happens": self.consequence,
63 "examples": list(self.examples),
64 }
67@dataclass(frozen=True)
68class _Check:
69 key: str
70 table: str
71 title: str
72 columns: tuple[str, ...]
73 #: ``frame → {label: mask}``; the finding's rows are their union.
74 masks: Callable[[pd.DataFrame], dict[str, pd.Series]]
75 consequence: str
78def _number(frame: pd.DataFrame, column: str) -> pd.Series:
79 return pd.to_numeric(frame[column], errors="coerce").astype(float)
82def _duration_masks(frame: pd.DataFrame) -> dict[str, pd.Series]:
83 duration = _number(frame, "duration_ms")
84 return {"negative": duration < 0, "zero": duration == 0}
87def _timing_masks(frame: pd.DataFrame) -> dict[str, pd.Series]:
88 duration, onset = _number(frame, "duration_ms"), _number(frame, "timestamp_ms")
89 return {"infinite duration": np.isinf(duration), "infinite onset": np.isinf(onset)}
92def _canvas_masks(frame: pd.DataFrame) -> dict[str, pd.Series]:
93 width, height = _number(frame, "canvas_width"), _number(frame, "canvas_height")
94 infinite = np.isinf(width) | np.isinf(height)
95 return {
96 "infinite": infinite,
97 "0 or less": ((width <= 0) | (height <= 0)) & ~infinite,
98 }
101def _position_masks(frame: pd.DataFrame) -> dict[str, pd.Series]:
102 xs, ys = _number(frame, "x"), _number(frame, "y")
103 infinite = np.isinf(xs) | np.isinf(ys)
104 return {"infinite": infinite, "missing": (xs.isna() | ys.isna()) & ~infinite}
107def _box_masks(frame: pd.DataFrame) -> dict[str, pd.Series]:
108 width, height = _number(frame, "width"), _number(frame, "height")
109 xs, ys = _number(frame, "x"), _number(frame, "y")
110 return {
111 "width ≤ 0": width <= 0,
112 "height ≤ 0": height <= 0,
113 "infinite size": np.isinf(width) | np.isinf(height),
114 "infinite position": np.isinf(xs) | np.isinf(ys),
115 }
118#: What a per-screen screen size the figure cannot use does, on either table.
119_CANVAS_CONSEQUENCE = (
120 "They stay in every table and export. The figure ignores a screen size that "
121 "is not one finite, positive number per screen, and draws that screen at the "
122 "size it uses when there is none: the Recording setup in the app, the size "
123 "given or one fitted to the data in the API and on the command line."
124)
127#: The checks, in the order they are reported. Each consequence is what the
128#: app does with those rows today: none of them is removed from a table.
129CHECKS: tuple[_Check, ...] = (
130 _Check(
131 "fixation_duration",
132 "fixations",
133 "Fixations lasting 0 ms or less",
134 ("duration_ms",),
135 _duration_masks,
136 "They stay in every table and export, and are counted at their value "
137 "wherever durations are summed (reading time, dwell). The plot draws them "
138 "at the smallest marker size. A blank or unreadable duration cell also "
139 "loads as 0 ms.",
140 ),
141 _Check(
142 "fixation_timing",
143 "fixations",
144 "Fixations with an infinite duration or onset",
145 ("duration_ms", "timestamp_ms"),
146 _timing_masks,
147 "They stay in every table and export, and an infinite duration makes "
148 "every sum it enters infinite (reading time, dwell). The figure draws it "
149 "at the smallest marker size and the replay counts it as 0 ms; a trial "
150 "with an infinite onset draws that fixation last, and its replay is "
151 "timed by the fixation durations instead of the onsets.",
152 ),
153 _Check(
154 "fixation_position",
155 "fixations",
156 "Fixations with no usable position",
157 ("x", "y"),
158 _position_masks,
159 "They stay in every table and export, but the plot cannot place them: "
160 "their marker, and the saccades to and from them, are left out of the "
161 "figure. A missing position is one neither the data nor the fixated "
162 "word's box could supply.",
163 ),
164 _Check(
165 "raw_gaze_position",
166 "raw_gaze",
167 "Raw-gaze samples with no usable position",
168 ("x", "y"),
169 _position_masks,
170 "They stay in the table and its export; the raw-gaze layer leaves them "
171 "out. Missing positions are usual in sample data (blinks, track loss).",
172 ),
173 _Check(
174 "word_box_size",
175 "words",
176 "Word boxes with no area or no usable position",
177 ("x", "y", "width", "height"),
178 _box_masks,
179 "They stay in every table and export. A box with no area holds no "
180 "fixation, so fixations reach such a word only through the data's own "
181 "word ids, and the box draws as a line or not at all; a box with an "
182 "infinite size or position is left out of the figure.",
183 ),
184 _Check(
185 "word_canvas",
186 "words",
187 "Word rows with an unusable screen size",
188 ("canvas_width", "canvas_height"),
189 _canvas_masks,
190 _CANVAS_CONSEQUENCE,
191 ),
192 _Check(
193 "fixation_canvas",
194 "fixations",
195 "Fixations with an unusable screen size",
196 ("canvas_width", "canvas_height"),
197 _canvas_masks,
198 _CANVAS_CONSEQUENCE,
199 ),
200)
203def _examples(frame: pd.DataFrame, mask: pd.Series, columns, n: int) -> tuple:
204 keys = [c for c in _EXAMPLE_KEYS if c in frame.columns]
205 shown = frame.loc[mask, [*keys, *columns]].head(n)
206 return tuple(shown.to_dict("records"))
209def check_data_health(
210 words: pd.DataFrame | None,
211 fixations: pd.DataFrame | None,
212 raw_gaze: pd.DataFrame | None = None,
213 *,
214 examples: int = EXAMPLE_ROWS,
215) -> list[HealthFinding]:
216 """Run every check on the normalized tables; return the ones that found rows.
218 Reads the canonical columns (``duration_ms``, ``x``/``y``, ``width``/
219 ``height``) and changes nothing. A table without a check's columns is not
220 checked. Each finding names its rows, the trials they fall in (by
221 participant and trial), a breakdown by kind, ``examples`` rows with their
222 identity, and what the app does with them.
223 """
224 frames = {"words": words, "fixations": fixations, "raw_gaze": raw_gaze}
225 findings: list[HealthFinding] = []
226 for check in CHECKS:
227 frame = frames[check.table]
228 if frame is None or frame.empty:
229 continue
230 if any(c not in frame.columns for c in check.columns):
231 continue
232 masks = {k: m.fillna(False) for k, m in check.masks(frame).items()}
233 union = pd.Series(False, index=frame.index)
234 for mask in masks.values():
235 union |= mask
236 rows = int(union.sum())
237 if not rows:
238 continue
239 trial_keys = [c for c in ("participant_id", "trial_id") if c in frame.columns]
240 trials = (
241 len(frame.loc[union, trial_keys].drop_duplicates()) if trial_keys else 0
242 )
243 breakdown = {k: int(m.sum()) for k, m in masks.items() if m.any()}
244 ordinary = check.table == "raw_gaze" and set(breakdown) == {"missing"}
245 findings.append(
246 HealthFinding(
247 check=check.key,
248 table=check.table,
249 title=check.title,
250 columns=check.columns,
251 rows=rows,
252 total_rows=len(frame),
253 trials=trials,
254 breakdown=breakdown,
255 examples=_examples(frame, union, check.columns, examples),
256 consequence=check.consequence,
257 severity="note" if ordinary else "warning",
258 )
259 )
260 return findings
263def findings_frame(findings: list[HealthFinding]) -> pd.DataFrame:
264 """The findings as one row each (empty, with the columns, when none)."""
265 columns = list(HealthFinding("", "", "", (), 0, 0, 0).to_record())
266 return pd.DataFrame([f.to_record() for f in findings], columns=columns)