{ "schema_version": "1.0.0", "package_id": "W07", "title": "Metrics, Uncertainty, and Repeated Observation", "language": "en", "phase": "Explain", "essential_question": "When is a visibility change repeatable?", "research_cutoff": "2026-08-24", "status": "structurally_complete_text_first_authored_draft_human_review_and_timed_pilot_pending", "required_artifacts": [ "README.md", "LECTURE_NOTES.md", "SLIDE_SCRIPT.md", "WORKED_CASE.md", "TRANSCRIPT.md", "manifest.json", "validate_package.py" ], "artifact_hash_scope": "five_narrative_markdown_artifacts; manifest and validator are reported at validation time to avoid circular self-hashing", "artifact_hashes_sha256": { "README.md": "42734287dc1a7fc791747559a49b7da90c65271e9cc4cd1ee6bd0ebf95f60907", "LECTURE_NOTES.md": "3ce7269f5eea1ff6fcb24151136a0b1c808df6a51349e850f6479347b6806e11", "SLIDE_SCRIPT.md": "c3fbb2ae17d5eb447f64eaa08a9305327b699cb593a4c746ee65b1ae6d4dd81c", "WORKED_CASE.md": "ee29b9c5c162c79723a0b291fcb604b044ffc61c4a0cdce528fcf3f05d3cc95f", "TRANSCRIPT.md": "00db40805227a3036ceacdd725f3c85db2abbab82f68415388c42e10a3e66a77" }, "namespace_policy": { "research_papers": "PAPER-xx", "platform_and_protocol_records": "PLAT-xx", "labs": "LAB-Lxx", "practitioner_synthesis": "Rxx", "human_facing_required_ids": [ "PAPER-29", "PAPER-33", "PAPER-38", "PLAT-04", "LAB-L06", "R07" ], "boundary": "Internal catalog IDs such as P04 must never appear in human-facing W07 narratives; PLAT-04 is the public identifier." }, "learning_outcomes": [ "Specify event, numerator, denominator, unit, query frame, weighting, time window, repetitions, missingness, aggregation, uncertainty, and claim ceiling in a metric card.", "Keep mention, citation, entailment, absorption, referral, and user action as distinct event namespaces.", "Validate a complete query-by-surface-by-time-by-repetition panel and preserve noncomplete designed cells.", "Separate within-block repeated-run variation from between-block descriptive drift.", "Explain why repeated response rows remain dependent inside query clusters.", "Interpret Wilson score intervals as descriptive under the L06 denominator and not dependence-adjusted.", "Reproduce a seeded query-cluster bootstrap with declared statistic, generator, seed, replicates, and endpoint convention.", "Evaluate predefined query-weight, aggregation, and missing-state sensitivities without post-result selection.", "Mark metrics non-comparable when events, units, populations, times, missingness, or aggregation do not align.", "Write a bounded frozen-panel repeatability conclusion without live, causal, cross-engine, or population generalization." ], "time_design": { "status": "authoring_budgets_not_observed_delivery_times", "synchronous_seminar_minutes": 90, "synchronous_core_studio_minutes": 75, "asynchronous_core_minutes_min": 195, "asynchronous_core_minutes_max": 205, "transcript_equivalent_minutes_min": 35, "transcript_equivalent_minutes_max": 45, "transcript_declared_minutes": 42, "full_lab_effort_hours": 8, "pilot_state": "not_taught_not_rehearsed_not_timed", "media_state": "no_video_or_audio_recording_exists" }, "local_dependencies": [ { "id": "COURSE-W07", "path": "course-site/app/lib/course-data.ts#W07", "role": "Authoritative current week title, essential question, outcomes, reading route, LAB-L06 link, stop condition, and evidence boundary.", "boundary": "A curriculum record does not establish delivery, learning effectiveness, or scientific validity." }, { "id": "NOTES-MEASUREMENT", "path": "notes-latex/chapters/04-measurement.tex", "role": "Conceptual alignment for evaluation cells, query weighting, stage metrics, repeated measures, denominator flow, and decision boundaries.", "boundary": "The notes are a research protocol scaffold, not a dashboard specification or production-platform result." }, { "id": "NOTES-BRIDGE-SAMPLING", "path": "notes-latex/appendices/f-prerequisite-bridges.tex#sec:bridge-sampling-bootstrap", "role": "Prerequisite route for strata, target weights, query-cluster bootstrap, and the distinction between response rows and sampled query intents.", "boundary": "Worked arithmetic and method guidance do not establish a GEO effect." }, { "id": "NOTES-BRIDGE-REPEATED", "path": "notes-latex/appendices/f-prerequisite-bridges.tex#sec:bridge-repeated-measures", "role": "Prerequisite route for nested and crossed dependence, intraclass correlation intuition, and within-cell versus between-state variation.", "boundary": "The descriptive decomposition is not automatically the final fitted model." }, { "id": "LAB-L06", "path": "course-labs/L06_repeated_measurement", "role": "Frozen 360-event synthetic panel, standard-library validation and analysis, event-specific metrics, Wilson intervals, drift table, query-stratum sensitivity, missingness, and validity boundary.", "boundary": "No named product, live engine, causal intervention, business outcome, or population demand is estimated." } ], "catalog_readings": [ { "id": "PAPER-29", "title": "Don't Measure Once: Measuring Visibility in AI Search (GEO)", "url": "https://arxiv.org/abs/2604.07585", "status": "Preprint / venue not confirmed here", "role": "Core route for repeated observation, temporal variation, and study-specific precision questions.", "boundary": "Its systems, languages, verticals, dates, repeat counts, and metrics do not define a universal measurement schedule." }, { "id": "PAPER-33", "title": "From Citation Selection to Citation Absorption: A Measurement Framework for Generative Engine Optimization Across AI Search Platforms", "url": "https://arxiv.org/abs/2604.25707", "status": "Preprint / venue not confirmed here", "role": "Core route for keeping citation selection, source support, and absorption measurement distinct.", "boundary": "Its data and conditioning sets cannot be silently converted into dashboard citation incidence or a cross-platform standard." }, { "id": "PAPER-38", "title": "Generative Engine Optimization at Scale: Measuring Brand Visibility Across AI Search Engines", "url": "https://arxiv.org/abs/2606.20065", "status": "Preprint / venue not confirmed here", "role": "Extension route for measurement-at-scale sampling, aggregation, and operational questions.", "boundary": "At-scale reporting retains its query, engine, date, label, and missingness boundaries and does not validate L06 fixture patterns." }, { "id": "PLAT-04", "title": "Introducing AI Performance in Bing Webmaster Tools Public Preview", "url": "https://blogs.bing.com/webmaster/February-2026/Introducing-AI-Performance-in-Bing-Webmaster-Tools-Public-Preview", "status": "Official Microsoft/Bing product guidance; 2026-02-10 local catalog record", "role": "Candidate interface case for auditing citation, cited-page, sampled-grounding-query, and trend labels on specified Microsoft surfaces.", "boundary": "Early-preview, aggregated, Microsoft-surface-specific data; counts do not indicate rank, authority, placement, cross-engine share, or business outcome." }, { "id": "R07", "title": "Visibility Has Two Axes: Repeated-Sampling Variance and Drift", "url": "https://thegeocommunity.com/blogs/generative-engine-optimization/visibility-two-axes-repeated-sampling-variance-drift/", "status": "Candidate practitioner research synthesis in the local reference catalog", "role": "Motivation for separating within-time repeated-sampling variation from between-time drift.", "boundary": "Derived synthesis only; the decomposition must be formalized and empirically validated with primary evidence." }, { "id": "NIST-METHOD", "title": "NIST/SEMATECH e-Handbook of Statistical Methods", "url": "https://www.itl.nist.gov/div898/handbook/", "status": "Official statistical-method guidance", "role": "Bounded method route for uncertainty, dependence diagnostics, time-ordered observations, and bootstrap assumptions.", "boundary": "Method guidance only; it reports no GEO visibility effect and does not select the query/time clustering structure for this study." } ], "course_record_signature": { "title": "Metrics, uncertainty, and repeated observation", "essential_question": "When is a visibility change repeatable?", "core_papers": [29, 33], "extend_papers": [38], "platform_display_ids": ["PLAT-04"], "lab_ids": ["L06"], "low_compute_phrase": "Run the supplied analysis on a small CSV; all charts have precomputed table equivalents." }, "metric_namespace": { "designed_event_unit": "one query-surface-time-repetition event", "events": { "mention": "binary resolved-entity event among complete responses under the fixture label", "citation": "binary visible-source-attribution event among complete responses under the fixture label", "entailment": "binary fixture support event; not interchangeable with citation", "absorption": "binary fixture source-information-use event; not interchangeable with entailment", "referral": "binary fixture referral event; not interchangeable with absorption", "user_action": "not present in L06 and not evaluated" }, "forbidden_collapses": [ "mention=citation", "citation=entailment", "entailment=absorption", "absorption=referral", "referral=user_action" ], "comparability_fields": [ "event", "unit", "eligible_denominator", "query_frame_and_weights", "surface_and_locale", "time_window", "missingness", "aggregation", "uncertainty" ] }, "l06_fixture_contract": { "status": "frozen_synthetic_offline_deterministic", "event_count": 360, "complete_event_count": 354, "noncomplete_event_count": 6, "missing_event_count": 6, "refused_event_count": 0, "query_count": 20, "surface_count": 2, "time_block_count": 3, "repetition_count": 3, "intent_strata": ["compare", "learn", "select", "verify"], "outcomes": ["mention", "citation", "entailment", "absorption", "referral"], "interval": "95% Wilson score interval; descriptive and not dependence-adjusted", "claim_ceiling": "deterministic synthetic fixture; no named product, live engine, causal intervention, business outcome, or population demand is estimated", "input_hashes_sha256": { "README.md": "b297f0c960a3edcafe685fb8f4b976c7209773e655438d1fb9d5b9a534553c13", "panel.csv": "c013f9aee92085ed91ef32261ad2e1a98956f1cc3667cd0c658985fb4a7ab5d7", "analyze_panel.py": "7a3fae3588176972ac5a7493b8079764b98497df10d85ec4cba018e5ba050f02" } }, "worked_case_contract": { "primary_metric": "mention", "primary_estimator": "equal mean of query-specific complete-case SYN-B minus SYN-A mention differences across all blocks and repetitions", "equal_query_contrast": 0.08402777777777778, "complete_row_pooled_contrast": 0.0790960451977401, "designed_event_compound_contrast": 0.07777777777777778, "syn_a_first_block_mention": { "numerator": 28, "denominator": 58, "rate": 0.4827586206896552, "wilson_low": 0.359282, "wilson_high": 0.608377 }, "syn_a_last_block_mention": { "numerator": 24, "denominator": 60, "rate": 0.4 }, "syn_a_last_minus_first": -0.082759, "syn_b_last_minus_first": -0.042958, "query_weight_sensitivity": { "weights": { "compare": 0.2, "learn": 0.5, "select": 0.2, "verify": 0.1 }, "syn_a_citation_equal_strata": 0.3569444444444444, "syn_a_citation_target_weighted": 0.33055555555555555, "syn_b_citation_equal_strata": 0.4548611111111111, "syn_b_citation_target_weighted": 0.46055555555555555 } }, "bootstrap_contract": { "statistic": "equal mean of 20 query-specific complete-case SYN-B minus SYN-A mention differences", "cluster": "query_id", "retain_attached_rows": true, "generator": "Python random.Random", "seed": 707, "resamples": 5000, "draws_per_resample": 20, "interval": "empirical percentile using sorted values at int(0.025*(B-1)) and int(0.975*(B-1))", "low": 0.010416666666666668, "high": 0.1590277777777778, "boundary": "Fixture query resampling only; not a causal interval, a live-engine interval, or proof of target-population coverage." }, "visual_policy": { "external_figure_reuse": false, "external_screenshot_reuse": false, "external_logo_reuse": false, "visuals": "original_instructional_diagram_specifications_only", "required_encodings": [ "label", "shape_or_position", "line_style", "numeric_endpoint", "alt_text", "text_or_table_equivalent" ], "license_review_required_before_any_later_external_asset_reuse": true }, "claim_boundary": { "frozen_fixture_observable": [ "panel identities, timestamps, statuses, dimensions, and hashes", "binary event rows and outcome-specific complete-response denominators", "surface-by-block numerators, rates, and Wilson endpoints", "first-to-last authored block differences", "query-stratum estimates", "missingness counts and response rates", "query-cluster statistic, seed, resamples, and empirical endpoints", "predefined weighting and denominator sensitivities" ], "not_identified": [ "cause of between-block drift", "independence of repeated rows", "real query-demand distribution", "causal effect of surface identity", "live-platform repeatability", "current product internals", "cross-engine metric equivalence", "missing-at-random mechanism", "external population coverage" ], "not_evaluated": [ "live collection", "named engine performance", "causal intervention", "organic discovery", "retrieval rank", "source authority", "citation placement", "business referral beyond the synthetic fixture label", "user action", "commercial outcome", "population demand", "cross-locale generalization", "cross-engine generalization" ] }, "quality_status": { "structural_validator": "passed_local_standard_library_validator_2026-08-24", "scientific_human_review": "pending", "accessibility_human_review": "pending", "timed_rehearsal": "pending", "classroom_pilot": "pending", "external_peer_review": "pending" } }