diff --git a/app/services/simulations/__tests__/llws-simulator.test.ts b/app/services/simulations/__tests__/llws-simulator.test.ts index 32fef37..6684652 100644 --- a/app/services/simulations/__tests__/llws-simulator.test.ts +++ b/app/services/simulations/__tests__/llws-simulator.test.ts @@ -157,7 +157,7 @@ describe("LLWSSimulator", () => { let mockDb: { select: MockInstance; query: { - scoringEvents: { findFirst: MockInstance }; + scoringEvents: { findMany: MockInstance }; playoffMatches: { findMany: MockInstance }; }; }; @@ -169,7 +169,7 @@ describe("LLWSSimulator", () => { mockDb = { select: vi.fn(), query: { - scoringEvents: { findFirst: vi.fn().mockResolvedValue(undefined) }, + scoringEvents: { findMany: vi.fn().mockResolvedValue([]) }, playoffMatches: { findMany: vi.fn().mockResolvedValue([]) }, }, }; @@ -189,7 +189,9 @@ describe("LLWSSimulator", () => { }); if (bracketMatches) { - mockDb.query.scoringEvents.findFirst.mockResolvedValue({ id: "event-1" }); + mockDb.query.scoringEvents.findMany.mockResolvedValue([ + { id: "event-1", createdAt: new Date("2026-08-01") }, + ]); mockDb.query.playoffMatches.findMany.mockResolvedValue( bracketMatches.map((m) => ({ ...EMPTY_MATCH, ...m })) ); @@ -501,6 +503,63 @@ describe("LLWSSimulator", () => { expect(rmse).toBeLessThan(0.02); }); + // Regression: the previous mapping rescaled every field onto a fixed 1250–1750 + // Elo span, which discarded how spread out the board actually was and pulled a + // nearly flat field apart into contenders and no-hopers the market never implied. + const TIGHT_BOARD = Array.from({ length: 20 }, (_, i) => 1500 + i * 35); + + it("does not inflate the favorite on a tightly priced board", async () => { + setupMockDb( + defaultParticipants(), + ALL_IDS.map((participantId, i) => ({ participantId, sourceOdds: TIGHT_BOARD[i] })) + ); + const results = await new LLWSSimulator(20_000).simulate("season-1"); + + const market = marketProbabilities(TIGHT_BOARD); + const simulated = probsFor(results, "us-1").probFirst; + + // The favorite prices near 6%. A fixed-span mapping simulated it around 13%, + // so the band is wide enough for Monte Carlo noise but nowhere near that. + expect(Math.abs(simulated - market[0])).toBeLessThan(0.015); + }); + + it("keeps a tightly priced field tight", async () => { + setupMockDb( + defaultParticipants(), + ALL_IDS.map((participantId, i) => ({ participantId, sourceOdds: TIGHT_BOARD[i] })) + ); + const results = await new LLWSSimulator(20_000).simulate("season-1"); + const probs = ALL_IDS.map((id) => probsFor(results, id).probFirst); + + // Every team prices between roughly 4% and 6%, so nobody should run away with + // it and nobody should be written off. + expect(Math.max(...probs)).toBeLessThan(0.09); + expect(Math.min(...probs)).toBeGreaterThan(0.02); + }); + + it("rates a team with no odds entered around the middle of the field", async () => { + // us-5 is priced mid-board; blanking its odds should not move it far. The old + // 1500 fallback was the centre of the Elo scale rather than of the field, which + // promoted an unpriced team to roughly 6th of 20. + const priced = ALL_IDS.map((participantId, i) => ({ participantId, sourceOdds: BOARD[i] })); + setupMockDb(defaultParticipants(), priced); + const withOdds = probsFor( + await new LLWSSimulator(20_000).simulate("season-1"), "us-5" + ).probFirst; + + const blanked = priced.map((row) => + row.participantId === "us-5" ? { ...row, sourceOdds: null } : row + ); + setupMockDb(defaultParticipants(), blanked); + const withoutOdds = probsFor( + await new LLWSSimulator(20_000).simulate("season-1"), "us-5" + ).probFirst; + + // Priced 5th of 20, so the median rating should land it in the same territory. + expect(withoutOdds).toBeGreaterThan(withOdds / 2); + expect(withoutOdds).toBeLessThan(withOdds * 2); + }); + it("does not starve longshots of championship probability", async () => { setupMockDb(defaultParticipants(), boardEvRows()); const results = await new LLWSSimulator(20_000).simulate("season-1"); @@ -549,6 +608,38 @@ describe("LLWSSimulator", () => { expect(results.reduce((s, r) => s + r.probabilities.probFirst, 0)).toBeCloseTo(1.0, 1); }); + it("uses the most recent bracket event when several exist", async () => { + // A stale event's matches would carry no draw, silently reverting to a + // randomized one and discarding every recorded result. + setupMockDb(defaultParticipants(), makeEvRows(ALL_IDS), seededBracket()); + mockDb.query.scoringEvents.findMany.mockResolvedValue([ + { id: "stale-event", createdAt: new Date("2026-07-01") }, + { id: "event-1", createdAt: new Date("2026-08-01") }, + ]); + + const results = await new LLWSSimulator(20_000).simulate("season-1"); + + // Bracket mode is in force, so the fixed bye slots still show their advantage. + expect(probsFor(results, "us-9").probFirst).toBeGreaterThan( + probsFor(results, "us-1").probFirst + ); + }); + + it("throws when the bracket is only partially seeded", async () => { + // participant1Id/participant2Id are ON DELETE SET NULL, so removing and + // re-adding one participant mid-tournament empties a single slot. Falling back + // to a randomized draw there would put eliminated teams back in contention. + const holed = seededBracket().map((m) => + m.round === "Opening Round" && m.matchNumber === 3 + ? { ...m, participant2Id: null } + : m + ); + setupMockDb(defaultParticipants(), makeEvRows(ALL_IDS), holed); + await expect(new LLWSSimulator(100).simulate("season-1")).rejects.toThrow( + /partially seeded \(19 of 20/ + ); + }); + it("throws when the bracket seeds the same team into two slots", async () => { const duplicated = seededBracket().map((m) => m.round === "Opening Round" && m.matchNumber === 2 diff --git a/app/services/simulations/llws-simulator.ts b/app/services/simulations/llws-simulator.ts index 2d43c15..83594fe 100644 --- a/app/services/simulations/llws-simulator.ts +++ b/app/services/simulations/llws-simulator.ts @@ -39,11 +39,12 @@ * A championship future already bakes in the ~6 wins needed to lift the trophy, so * using it directly as a single-game strength (p1 / (p1 + p2)) makes every * individual game as lopsided as the whole tournament and compounds the favorite's - * edge over and over. convertFuturesToElo undoes that compression first (the - * empirically calibrated cube-root step in decompressProbability) before mapping to - * an Elo scale, which is how the other bracket simulators on the platform consume - * futures. LLWS_PARITY_FACTOR then widens the Elo curve to reflect how much - * single-game variance there is in six-inning Little League baseball. + * edge over and over. buildLLWSElos undoes that compression first (the empirically + * calibrated cube-root step in decompressProbability) before mapping to an Elo + * scale, and LLWS_PARITY_FACTOR then widens the Elo curve to reflect how much + * single-game variance there is in six-inning Little League baseball. Unlike the + * shared convertFuturesToElo helper, the mapping preserves how spread out the board + * actually is — see buildLLWSElos for why that matters. * * Side assignment (externalId): "US" or "Intl". The legacy pool suffixes * ("US:A", "US:B", "Intl:A", "Intl:B") are still accepted and read as the side @@ -74,7 +75,8 @@ import { database } from "~/database/context"; import { and, eq } from "drizzle-orm"; import * as schema from "~/database/schema"; import { - convertFuturesToElo, + convertAmericanOddsToProbability, + decompressProbability, eloWinProbabilityWithParity, } from "~/services/probability-engine"; import { llwsMatchNumber } from "~/lib/bracket-templates"; @@ -92,20 +94,36 @@ const LLWS_TEMPLATE_ID = "llws_20"; /** * Elo scaling for a single LLWS game. * - * Much higher than the 400-point standard because a six-inning Little League game - * between 12-year-olds is far closer to a coin flip than a pro game: one pitcher, one - * big inning, and the mercy rule all compress the gap. + * Higher than the 400-point standard because a six-inning Little League game between + * 12-year-olds is far closer to a coin flip than a pro game: one pitcher, one big + * inning, and the mercy rule all compress the gap. * * Calibrated by sweeping this value until a randomized-draw simulation reproduces the - * championship futures it was fed. Against a representative 20-team futures board, - * simulated championship probability vs. the market it came from: - * parity 400 → favorite 21.8% priced, 51.5% simulated (RMSE 0.075) - * parity 700 → favorite 21.8% priced, 31.5% simulated (RMSE 0.026) - * parity 1000 → favorite 21.8% priced, 21.8% simulated (RMSE 0.003) ← chosen - * parity 1200 → favorite 21.8% priced, 18.3% simulated (RMSE 0.010) + * championship futures it was fed, across boards of different shape (see + * LLWS_ELO_SPREAD for why the shape matters). Total RMSE over a wide board, a + * top-heavy board, and a nearly flat one: + * parity 450 → 0.028 + * parity 550 → 0.016 ← chosen + * parity 750 → 0.040 + * parity 1000 → 0.061 * Overridable per season via the `parityFactor` simulator config. */ -const LLWS_PARITY_FACTOR = 1_000; +const LLWS_PARITY_FACTOR = 550; + +/** + * Elo points per natural-log unit of relative team strength. + * + * Only the ratio LLWS_ELO_SPREAD / parityFactor affects the simulation, so this fixes + * the readable scale of the ratings and LLWS_PARITY_FACTOR does the calibrating. 300 + * puts a typical 20-team board in the familiar ~1350–1700 range. + */ +const LLWS_ELO_SPREAD = 300; + +/** + * Power transform undoing the compounding baked into a championship future. + * Matches DEFAULT_CALIBRATION.exponent in the probability engine. + */ +const LLWS_DECOMPRESSION_EXPONENT = 0.33; // ─── Types ──────────────────────────────────────────────────────────────────── @@ -362,21 +380,58 @@ function inferExternalIdFromName(name: string): string { /** * Map participants to single-game Elo ratings from their championship futures. * - * Teams with no odds entered sit at DEFAULT_ELO, which is also where every team lands - * when the season has no odds at all — so an unconfigured season still simulates as a - * field of coin flips rather than throwing. + * Deliberately NOT convertFuturesToElo. That helper finishes by rescaling the field + * onto a fixed 1250–1750 span (mapToElo), which throws away how spread out the board + * actually is: a board whose favorite is priced at 22% and one whose favorite is + * priced at 6% both come out 500 Elo wide, so the tight board's field gets pulled + * apart into contenders and no-hopers that the market never implied. On such a board + * that inflated the favorite from 6% to 13%. + * + * Instead the decompressed strengths are mapped by their log-ratio to the field's + * geometric mean, which preserves dispersion: a tight board yields a narrow Elo span + * and a top-heavy one a wide span, both centred on DEFAULT_ELO. + * + * Returns the ratings alongside the rating to use for a team with no odds entered — + * the median of the priced field, so leaving odds blank neither promotes nor buries a + * team. (DEFAULT_ELO is the centre of the scale, but futures fields are skewed, so on + * a typical board it would rank a team around 6th of 20.) */ export function buildLLWSElos( evRows: Array<{ participantId: string; sourceOdds: number | null }> -): Map { - const oddsInput = evRows - .filter((row) => row.sourceOdds !== null) - .map((row) => ({ participantId: row.participantId, odds: row.sourceOdds ?? 0 })); +): { elos: Map; unpricedElo: number } { + const priced = evRows.filter((row) => row.sourceOdds !== null); - // convertFuturesToElo needs a spread to normalise against; a single priced team - // carries no information about the rest of the field. - if (oddsInput.length < 2) return new Map(); - return convertFuturesToElo(oddsInput); + // A single priced team carries no information about the rest of the field, so + // there is nothing to normalise against — treat the season as unpriced. + if (priced.length < 2) return { elos: new Map(), unpricedElo: DEFAULT_ELO }; + + const rawProbs = priced.map((row) => convertAmericanOddsToProbability(row.sourceOdds ?? 0)); + const rawSum = rawProbs.reduce((a, b) => a + b, 0); + if (rawSum <= 0) return { elos: new Map(), unpricedElo: DEFAULT_ELO }; + + // Vig-removed championship probability → single-game strength. + const logStrengths = rawProbs.map((prob) => + Math.log( + Math.max(decompressProbability(prob / rawSum, LLWS_DECOMPRESSION_EXPONENT), Number.MIN_VALUE) + ) + ); + const meanLog = logStrengths.reduce((a, b) => a + b, 0) / logStrengths.length; + + const elos = new Map( + priced.map((row, i) => [ + row.participantId, + DEFAULT_ELO + LLWS_ELO_SPREAD * (logStrengths[i] - meanLog), + ]) + ); + + return { elos, unpricedElo: median([...elos.values()]) }; +} + +function median(values: number[]): number { + if (values.length === 0) return DEFAULT_ELO; + const sorted = values.toSorted((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid]; } // ─── Bracket loading ────────────────────────────────────────────────────────── @@ -384,10 +439,16 @@ export function buildLLWSElos( /** * Read the seeded llws_20 bracket for this season, if there is one. * - * Returns null when no bracket exists yet or its opening slots have not been filled - * in — in that case the caller falls back to a randomized draw. Throws when the - * bracket is seeded with participants that don't belong to the season, which is a - * misconfiguration worth surfacing rather than silently ignoring. + * Returns null only when the bracket carries no draw at all — no matches, or a + * freshly generated bracket with every slot still empty — in which case the caller + * falls back to a randomized draw. + * + * A *partially* seeded bracket is an error rather than a fallback. Silently falling + * back there would throw away the real draw and every recorded result along with it, + * putting eliminated teams back in contention; and it is reachable in practice, + * because playoff_matches.participant1Id/participant2Id are ON DELETE SET NULL, so + * removing and re-adding a single participant mid-tournament empties a slot. + * Likewise, a bracket seeded with unknown or duplicated participants fails loudly. */ export function readBracketSlots( matches: BracketMatch[], @@ -396,39 +457,59 @@ export function readBracketSlots( if (matches.length === 0) return null; const byKey = new Map(matches.map((m) => [matchKey(m.round, m.matchNumber), m])); - const slots: Record = { US: [], Intl: [] }; - const seen = new Set(); + + // Collect both sides' draws before deciding, so "nothing seeded" is judged over the + // whole bracket rather than one side at a time. + const draw: Record = { US: [], Intl: [] }; for (const side of ["US", "Intl"] as const) { const sideIndex = SIDE_INDEX[side]; - const ids: (string | null)[] = []; for (let local = 1; local <= 4; local++) { const match = byKey.get( matchKey("Opening Round", llwsMatchNumber("Opening Round", sideIndex, local)) ); - ids.push(match?.participant1Id ?? null, match?.participant2Id ?? null); + draw[side].push(match?.participant1Id ?? null, match?.participant2Id ?? null); } for (let local = 1; local <= 2; local++) { const match = byKey.get( matchKey("Winners Round 2", llwsMatchNumber("Winners Round 2", sideIndex, local)) ); - ids.push(match?.participant1Id ?? null); + draw[side].push(match?.participant1Id ?? null); } + } - // An unseeded (or partially seeded) bracket carries no draw information. - if (ids.some((id) => id === null)) return null; + const allSlots = [...draw.US, ...draw.Intl]; + const seededCount = allSlots.filter((id) => id !== null).length; - for (const id of ids) { - if (seen.has(id as string)) { - throw new Error(`LLWS bracket seeds participant ${id} into more than one slot.`); + // Generated but not yet filled in — no draw to honor. + if (seededCount === 0) return null; + + if (seededCount < allSlots.length) { + throw new Error( + `LLWS bracket is only partially seeded (${seededCount} of ${allSlots.length} slots ` + + `filled). Re-seed the bracket in Admin → Bracket before simulating; simulating ` + + `around the gap would discard the draw and every recorded result.` + ); + } + + const slots: Record = { US: [], Intl: [] }; + const seen = new Set(); + + for (const side of ["US", "Intl"] as const) { + for (const id of draw[side]) { + const participantId = id as string; + if (seen.has(participantId)) { + throw new Error( + `LLWS bracket seeds participant ${participantId} into more than one slot.` + ); } - seen.add(id as string); + seen.add(participantId); - const team = teamsById.get(id as string); + const team = teamsById.get(participantId); if (!team) { throw new Error( - `LLWS bracket references participant ${id}, which is not in this sports season.` + `LLWS bracket references participant ${participantId}, which is not in this sports season.` ); } // The bracket is authoritative about which side a team is on. @@ -472,7 +553,7 @@ export class LLWSSimulator implements Simulator { .where(eq(schema.seasonParticipantExpectedValues.sportsSeasonId, sportsSeasonId)); // 3. Decompress the futures into single-game Elo ratings. - const eloMap = buildLLWSElos(evRows); + const { elos, unpricedElo } = buildLLWSElos(evRows); // 4. Parse externalId for each participant to determine which side they're on. // A seeded bracket overrides this below, but the field still has to be a legal @@ -487,20 +568,26 @@ export class LLWSSimulator implements Simulator { // Provisional: a seeded bracket overwrites this below. participantId: p.id, side: parsed?.side ?? "Intl", - elo: eloMap.get(p.id) ?? DEFAULT_ELO, + elo: elos.get(p.id) ?? unpricedElo, }); } const teamsById = new Map(teams.map((t) => [t.participantId, t])); // 5. Load the real bracket (draw + results so far), if one has been generated. - const bracketEvent = await db.query.scoringEvents.findFirst({ + // If several llws_20 playoff events exist, take the most recent so a re-created + // event wins over a stale one — landing on the stale row would silently discard + // the real draw and every recorded result. + const playoffEvents = await db.query.scoringEvents.findMany({ where: and( eq(schema.scoringEvents.sportsSeasonId, sportsSeasonId), eq(schema.scoringEvents.eventType, "playoff_game"), eq(schema.scoringEvents.bracketTemplateId, LLWS_TEMPLATE_ID) ), }); + const bracketEvent = playoffEvents.toSorted( + (a, b) => (b.createdAt?.getTime() ?? 0) - (a.createdAt?.getTime() ?? 0) + )[0]; const bracketMatches = bracketEvent ? await db.query.playoffMatches.findMany({ diff --git a/app/services/simulations/manifest.ts b/app/services/simulations/manifest.ts index 41160fb..0a6f801 100644 --- a/app/services/simulations/manifest.ts +++ b/app/services/simulations/manifest.ts @@ -183,7 +183,7 @@ const PROFILES: Record