{ "schema_version": "1.0.0", "package_id": "W04", "title": "Ranking and Context Selection", "language": "en", "phase": "Observe", "essential_question": "Why does an eligible source enter the context?", "research_cutoff": "2026-08-24", "status": "structurally_complete_text_first_authored_draft_human_review_and_timed_pilot_pending", "required_artifacts": [ "README.md", "LECTURE_NOTES.md", "SLIDE_SCRIPT.md", "WORKED_CASE.md", "TRANSCRIPT.md", "manifest.json", "validate_package.py" ], "artifact_hash_scope": "five_narrative_markdown_artifacts; manifest and validator are reported at validation time to avoid circular self-hashing", "artifact_hashes_sha256": { "README.md": "416c8f188ea4f500af2a481f90e72218c251210f049fa35c29d58e0561e5ab55", "LECTURE_NOTES.md": "e74e8118e180c7fc7a57b526b9f64e274918f74da1fc0d6c4f2ea8189673cb77", "SLIDE_SCRIPT.md": "65d73f9c871e60a0d6cd6a9dae9d1b7c45adb3ef3495704576d620d498695a34", "WORKED_CASE.md": "9b150a175b7a2a10d0d07ae8166bb8f75d7d6649085cc79c273911d7837ea5af", "TRANSCRIPT.md": "e5832c6d310ab23578bbb25131b5bd30e8bcd862f6d2e2b042e022923f34ea77" }, "learning_outcomes": [ "Separate candidate construction, fixed-set reranking, context packing, answer use, citation presentation, and production outcomes.", "Prove candidate-set and representation identity before interpreting a reranking delta.", "Calculate reciprocal rank, DCG, and NDCG under a declared gain scale, cutoff, tie rule, aggregation, and judgment policy.", "Explain why pure reranking preserves full-candidate-depth membership recall while shallower Recall and NDCG may change.", "Run a fixed-order, fixed-budget context-packing ablation that changes exactly one allocation rule.", "Report relevance, material-claim coverage, dependency-origin diversity, and redundancy separately.", "Locate candidate, ordering, packing, generation-use, citation, and outcome failures without crossing an evidence boundary.", "Write separate bounded conclusions for an inspectable fixture and an opaque answer surface." ], "time_design": { "status": "authoring_budgets_not_observed_delivery_times", "synchronous_seminar_minutes": 90, "synchronous_core_studio_minutes": 75, "asynchronous_core_minutes_min": 190, "asynchronous_core_minutes_max": 200, "transcript_equivalent_minutes_min": 35, "transcript_equivalent_minutes_max": 45, "transcript_declared_minutes": 42, "full_lab_effort_hours": 7, "pilot_state": "not_taught_not_rehearsed_not_timed", "media_state": "no_video_or_audio_recording_exists" }, "local_dependencies": [ { "id": "COURSE-W04", "path": "course-site/app/lib/course-data.ts#W04", "role": "Authoritative current week title, essential question, outcomes, reading route, L03 link, and evidence-boundary language.", "boundary": "A curriculum record does not establish delivery, learner achievement, or scientific validity." }, { "id": "NOTES-CH04", "path": "notes-latex/chapters/04-retrieval-context.tex", "role": "Notation and conceptual alignment for fixed-candidate evidence, reranking, NDCG, context allocation, redundancy, and contribution limits.", "boundary": "The notes provide an analytical research scaffold, not a verified blueprint of any closed platform." }, { "id": "NOTES-APP-F-RANKING", "path": "notes-latex/appendices/f-prerequisite-bridges.tex#ranking-metrics", "role": "Prerequisite route for gain, cutoff, DCG, NDCG, Recall, and judgment-policy interpretation.", "boundary": "Synthetic arithmetic teaches metric interpretation and contains no production effect." }, { "id": "LAB-L03", "path": "course-labs/L03_retrieval_reranking_sandbox", "role": "Deterministic offline candidate, fusion, reranking, metric, and stage-boundary fixture shared with W03.", "boundary": "The shared script does not evaluate context packing, answer generation, citation, or production behavior." } ], "catalog_readings": [ { "id": "PAPER-17", "title": "MaxShapley: Towards Incentive-compatible Generative Search with Fair Context Attribution", "url": "https://arxiv.org/abs/2512.05958", "status": "Preprint / venue not confirmed here", "role": "Bounded route for conditional contribution, utility, player definition, and attribution assumptions.", "boundary": "Does not establish retrieval membership, universal authorship, displayed citation, or a commercial context rule." }, { "id": "PAPER-22", "title": "When Attention Becomes Exposure in Generative Search", "url": "https://arxiv.org/abs/2601.01750", "status": "Preprint / venue not confirmed here", "role": "Bounded route for examining how an internal quantity and exposure are operationalized in one research setting.", "boundary": "Does not make attention a universal explanation of context admission, answer use, or citation." }, { "id": "PAPER-18", "title": "MillStone: How Open-Minded Are LLMs?", "url": "https://arxiv.org/abs/2509.11967", "status": "Preprint / venue not confirmed here", "role": "Extension route for evidence composition and response behavior under a bounded experimental design.", "boundary": "A fixed-evidence result cannot establish organic acquisition, first-stage retrieval, or production ranking." }, { "id": "PLAT-01", "title": "Google’s Guide to Optimizing for Generative AI Features on Google Search", "url": "https://developers.google.com/search/docs/fundamentals/ai-optimization-guide", "status": "Official named-product documentation; local catalog snapshot accessed 2026-08-24", "role": "Product-specific route for documented guidance and explicit serving or indexing boundaries.", "boundary": "Authoritative only for the named documentation; no cross-engine or outcome claim and no hidden reranker or packing telemetry." }, { "id": "NDCG-FOUNDATION", "title": "Cumulated Gain-Based Evaluation of IR Techniques", "url": "https://doi.org/10.1145/582415.582418", "status": "Peer-reviewed primary metric source", "role": "Foundational route for graded, rank-sensitive evaluation.", "boundary": "NDCG evaluates judged order under a declared convention; it is not answer fidelity or user utility." }, { "id": "G04", "title": "RankLLM", "url": "https://github.com/castorini/rank_llm", "status": "Candidate academic implementation route in the local reference catalog", "role": "Optional extension route for pinned reranking workflows under a separate environment and model/API record.", "boundary": "Repository capabilities do not independently validate results or describe a closed production system." } ], "course_record_signature": { "title": "Ranking and context selection", "essential_question": "Why does an eligible source enter the context?", "core_papers": [17, 22], "extend_papers": [18], "lab_ids": ["L03"], "low_compute_phrase": "Use precomputed scores and the deterministic packing simulator." }, "l03_fixture_contract": { "corpus_records": 10, "topics": 3, "positive_graded_qrels": 7, "offline": true, "deterministic": true, "candidate_depth": 5, "evaluation_k": 3, "hybrid_alpha": 0.55, "rerank_weight": 0.25, "opaque_run_identity": "none; scores are authored fixture values", "context_packing_evaluated_by_shared_script": false, "input_hashes_sha256": { "corpus.jsonl": "af6a958d9c5e340f893399f5e9945b8ea58a59a55f14b95be9a139cdbaed5052", "topics.jsonl": "ef048bf04a0cc0b239acfa4fb36da0a653f9c40b65bd17642f9271f1c1da664e", "qrels.tsv": "16031ded7d5b275637a288050ee38757714fd79b3be8d3f5be7195d98feb037c", "frozen_dense_run.tsv": "ede718078948e4d8f69a706af4383df90c09b16c15faf82d655945070c5d5d7f", "frozen_dense_run_metadata.json": "7f3786e0d70358d311ee87ec5d4d7c32505f8c4c3a3e6de3d8114d3caec0bd9e" } }, "worked_case_contract": { "status": "authored_synthetic_hand_auditable_fixture", "candidate_count": 5, "positive_gain_candidates": 4, "evaluation_k": 3, "candidate_depth": 5, "baseline_ndcg_at_3": 0.5766666455144387, "reranked_ndcg_at_3": 1.0, "baseline_recall_at_3": 0.5, "reranked_recall_at_3": 0.75, "both_recall_at_5": 1.0, "packing_budget_authored_units": 180, "baseline_context": ["C2", "C4"], "treatment_context": ["C2", "C1"], "baseline_material_claim_coverage": "2/4", "treatment_material_claim_coverage": "3/4", "manipulated_packing_factor": "skip a same-cluster near duplicate when it adds no new preregistered material claim" }, "visual_policy": { "external_figure_reuse": false, "external_screenshot_reuse": false, "external_logo_reuse": false, "visuals": "original_instructional_diagram_specifications_only", "required_encodings": [ "label", "shape_or_position", "line_style", "alt_text", "text_or_table_equivalent" ], "license_review_required_before_any_later_external_asset_reuse": true }, "claim_boundary": { "open_fixture_observable": [ "candidate IDs, counts, versions, hashes, and gains", "baseline and reranked scores and orders", "fixed-pool membership equality", "Recall and NDCG under the declared policy", "authored passage costs, claim labels, origin clusters, and redundancy relations", "packing budget, selected spans, order, exclusions, and reason codes", "input and narrative hashes" ], "closed_platform_unknown_without_separate_evidence": [ "candidate pool", "candidate representations and truncation", "reranker identity, features, scores, and tie policy", "context budget, selected spans, and packed order", "generator use of a source", "citation resolution mechanism" ], "not_evaluated": [ "public crawling", "production indexing", "organic candidate recovery", "named learned reranker performance", "commercial context packing", "answer generation", "answer correctness", "source absorption", "citation correctness", "citation completeness", "referral", "user action", "commercial outcome" ] }, "quality_status": { "structural_validator": "passed_2026-08-24", "deterministic_fixture_reproduction": "passed_2026-08-24", "narrative_hash_lock": "verified_2026-08-24", "scientific_human_review": "pending", "accessibility_human_review": "pending", "timed_rehearsal": "pending", "classroom_pilot": "pending", "external_peer_review": "pending" } }