{ "schema_version": "1.0.0", "package_id": "W06", "title": "Queries as a Measurement Instrument", "language": "en", "phase": "Explain", "essential_question": "What distribution do the prompts represent?", "research_cutoff": "2026-08-24", "status": "structurally_complete_text_first_authored_draft_human_review_recording_and_pilot_pending", "required_artifacts": [ "README.md", "LECTURE_NOTES.md", "SLIDE_SCRIPT.md", "WORKED_CASE.md", "TRANSCRIPT.md", "manifest.json", "validate_package.py" ], "learning_outcomes": [ "Separate a declared query universe, operational sampling frame, eligible frame, selected sample, and realized evaluation cells.", "Predeclare strata, allocation, weight provenance, inclusion and exclusion rules, locale, time, surface, account, conversation, and tool state.", "Assign stable query-intent clusters, provenance, development and held-out roles, and version identities.", "Detect exact and candidate near duplicates without treating lexical similarity as semantic truth.", "Prevent desired-answer, treatment, split, test-reuse, label, temporal, and provenance leakage.", "Govern qrels and other constructed labels as protocol-bounded judgments rather than universal truth or demand frequency.", "Write an external-validity statement that identifies represented, thin, excluded, unreachable, and unknown populations." ], "time_design": { "status": "authoring_budgets_not_observed_delivery_times", "synchronous_seminar_minutes": 90, "synchronous_core_studio_minutes": 75, "asynchronous_core_minutes_min": 210, "asynchronous_core_minutes_max": 220, "transcript_equivalent_minutes_min": 35, "transcript_equivalent_minutes_max": 45, "transcript_planned_total_minutes": 42, "full_lab_effort_hours": 5, "pilot_state": "not_taught_not_rehearsed_not_timed", "media_state": "no_video_or_audio_recording_exists" }, "local_dependencies": [ { "id": "COURSE-W06", "path": "course-site/app/lib/course-data.ts#W06", "role": "Authoritative current week title, essential question, outcomes, preparation, L02 route, and boundary wording.", "boundary": "A curriculum record does not establish that the lesson has been delivered, reviewed, or effective." }, { "id": "NOTES-CH04-MEASUREMENT", "path": "notes-latex/chapters/04-measurement.tex", "role": "Conceptual alignment for query distributions, strata, design and target weights, evaluation cells, split leakage, and repeated measures.", "boundary": "The chapter supplies a general measurement framework; it does not validate a particular query frame or external platform." }, { "id": "NOTES-APP-F-SAMPLING", "path": "notes-latex/appendices/f-prerequisite-bridges.tex", "role": "Prerequisite bridge for stratified target means, query-cluster resampling, and the difference between repeats and independent sampled intents.", "boundary": "Worked arithmetic is pedagogical and does not determine target weights or interval coverage for an undeclared population." }, { "id": "NOTES-APP-B-REPRODUCIBILITY", "path": "notes-latex/appendices/b-reproducibility.tex", "role": "Reproducibility fields for query panels, splits, weights, system state, raw cells, missingness, labels, and hashes.", "boundary": "A complete manifest improves auditability but does not create representativeness or scientific validity." }, { "id": "LAB-L02", "path": "course-labs/L02_query_set_engineering", "role": "Deterministic offline query-bank fixture, sampling frame, lexical audit, exclusions, coverage report, and reproducibility route.", "boundary": "The 20-row instructor-authored synthetic fixture supports method rehearsal only; it does not estimate organic demand, platform behavior, or external validity." } ], "controlled_source_routes": [ { "id": "PAPER-29", "type": "preprint_case", "title": "Don't Measure Once: Measuring Visibility in AI Search (GEO)", "url": "https://arxiv.org/abs/2604.07585", "role": "Bounded case for prompt-panel repetition, within-window variability, and time-aware measurement design.", "boundary": "The small Swiss-German commercial panel and finite observed runs do not establish a universal query distribution, repeat count, or monitoring window." }, { "id": "PAPER-38", "type": "vendor_preprint_case", "title": "Generative Engine Optimization at Scale: Measuring Brand Visibility Across AI Search Engines", "url": "https://arxiv.org/abs/2606.20065", "role": "Case for auditing generated-prompt provenance, branded wording, platform-specific collection, and changing analytic universes.", "boundary": "The vendor cohort and generated prompts do not estimate organic demand or causal effects, and denominators from distinct analytic universes must not be combined." }, { "id": "PAPER-26", "type": "preprint_case", "title": "Role-Augmented Intent-Driven Generative Search Engine Optimization", "url": "https://arxiv.org/abs/2508.11158", "role": "Bounded case for generated query variants and intent-conditioned evaluation in a supplied fixed-context simulation.", "boundary": "The case does not observe live query planning, retrieval, natural user demand, or a commercial engine's internal intent model." }, { "id": "PAPER-01", "type": "quarantined_preprint_audit_only", "title": "Cultural Encoding in Large Language Models: The Existence Gap in AI-Mediated Brand Discovery", "url": "https://arxiv.org/abs/2601.00869", "role": "Time-provenance failure case only.", "boundary": "Reported collection dates in the local full text cross the 2026-08-24 course freeze; no substantive finding is used until the dates and source version receive independent resolution." }, { "id": "PLAT-04", "type": "official_platform_preview_documentation", "title": "Introducing AI Performance in Bing Webmaster Tools Public Preview", "url": "https://blogs.bing.com/webmaster/February-2026/Introducing-AI-Performance-in-Bing-Webmaster-Tools-Public-Preview", "role": "Named measurement-interface case for citations, cited pages, sampled grounding queries, and product-specific reporting fields.", "boundary": "The preview documentation is Microsoft-surface-specific and does not establish rank, authority, placement, a cross-platform sampling frame, or this course instrument's validity." } ], "fixture_contract": { "lab_id": "L02", "frame_id": "FRAME-DEMO-001", "frame_status": "demonstration_only", "locale": "en-HK", "provenance": "instructor_authored_synthetic", "input_count": 20, "accepted_count": 18, "excluded_count": 2, "duplicate_candidate_count": 1, "development_count": 13, "held_out_count": 5, "similarity_method": "English-oriented token-set Jaccard over unique lowercase tokens", "similarity_threshold": 0.8, "duplicate_pair": { "query_id_a": "Q-004", "query_id_b": "Q-019", "score": 0.818182, "decision": "exclude_later_record" }, "leakage_exclusion": { "query_id": "Q-020", "reason": "desired_answer_leakage" }, "intent_counts": { "learn": 3, "compare": 3, "select": 3, "verify": 4, "counterfactual": 3, "negative_control": 2 }, "warning": "accepted bank has fewer than the 60 queries required for the full assignment", "representativeness_boundary": "Authored synthetic fixture; it does not estimate population demand or real-user frequency.", "input_hashes": { "data/sampling_frame.json": "f856683408cc322866e7dfd81c624328c45eacb3c7f2a2d249625cdddab0668f", "data/query_bank.csv": "e1f83706adf78cd452fe1594625f4900f01ce48e17e49697d62c034a339489ea" } }, "visual_policy": { "external_figure_reuse": false, "external_screenshot_reuse": false, "attached_tex_figure_reuse": false, "visuals": "original_instructional_diagram_specifications_only", "required_encodings": [ "label", "shape_or_position", "line_or_fill_pattern", "alt_text", "text_table_when_data_are_encoded" ], "license_review_required_before_any_later_external_asset_reuse": true }, "claim_boundary": { "observable_in_fixture": [ "declared frame and its limitations", "twenty input records and schemas", "stable query identifiers", "declared intent, entity, locale, provenance, synthetic, and split labels", "tokenization and Jaccard rule", "duplicate and leakage exclusions", "accepted coverage counts and warning", "input and output hashes" ], "constructed_protocol_objects": [ "query universe statement", "sampling frame", "intent strata", "intent clusters", "inclusion and exclusion rules", "qrels and relevance grades", "development and held-out roles", "design or target weights" ], "not_evaluated": [ "organic query frequency", "real-user demand", "cross-locale equivalence", "semantic duplicate accuracy", "commercial platform query planning", "retrieval effectiveness", "ranking quality", "answer generation", "citation correctness", "source absorption", "referral or user action", "conversion or commercial outcome" ] }, "content_hashes": { "README.md": "09b7272b58c9c55764e2f62ebbeb6432724175de8751456f9527dc5a617e05e4", "LECTURE_NOTES.md": "f343a439f35410d2eb73fda27d342308d49a03c39493c0b55fa0cc55ab6fb84a", "SLIDE_SCRIPT.md": "3350edd4e2f8b5fa5050227c59f4096032547cd9f06285e7c1d96195d0e99668", "WORKED_CASE.md": "dd473a5c6dcbb092b5dd61a08f4a24fd676eea525befd06abfb122099cfddb03", "TRANSCRIPT.md": "4beabb298766b66a45f61b68c4c2860c03c85041ff09170b1c3fa37b0be65a04" }, "quality_status": { "structural_validator": "passed_2026-08-24", "deterministic_fixture_reproduction": "passed_2026-08-24", "narrative_hash_lock": "verified_2026-08-24", "scientific_human_review": "pending", "qrel_protocol_human_review": "pending", "accessibility_human_review": "pending", "screen_reader_review": "pending", "timed_rehearsal": "pending", "classroom_pilot": "pending", "grading_calibration": "pending", "recording": "not_created", "external_peer_review": "pending" } }