claude/llws-simulator-logic-0aqbjb #142

Merged
chrisp merged 2 commits from claude/llws-simulator-logic-0aqbjb into main 2026-08-23 01:57:38 +00:00
3 changed files with 229 additions and 51 deletions
Showing only changes of commit 3fae78c521 - Show all commits

View file

@ -157,7 +157,7 @@ describe("LLWSSimulator", () => {
let mockDb: { let mockDb: {
select: MockInstance; select: MockInstance;
query: { query: {
scoringEvents: { findFirst: MockInstance }; scoringEvents: { findMany: MockInstance };
playoffMatches: { findMany: MockInstance }; playoffMatches: { findMany: MockInstance };
}; };
}; };
@ -169,7 +169,7 @@ describe("LLWSSimulator", () => {
mockDb = { mockDb = {
select: vi.fn(), select: vi.fn(),
query: { query: {
scoringEvents: { findFirst: vi.fn().mockResolvedValue(undefined) }, scoringEvents: { findMany: vi.fn().mockResolvedValue([]) },
playoffMatches: { findMany: vi.fn().mockResolvedValue([]) }, playoffMatches: { findMany: vi.fn().mockResolvedValue([]) },
}, },
}; };
@ -189,7 +189,9 @@ describe("LLWSSimulator", () => {
}); });
if (bracketMatches) { if (bracketMatches) {
mockDb.query.scoringEvents.findFirst.mockResolvedValue({ id: "event-1" }); mockDb.query.scoringEvents.findMany.mockResolvedValue([
{ id: "event-1", createdAt: new Date("2026-08-01") },
]);
mockDb.query.playoffMatches.findMany.mockResolvedValue( mockDb.query.playoffMatches.findMany.mockResolvedValue(
bracketMatches.map((m) => ({ ...EMPTY_MATCH, ...m })) bracketMatches.map((m) => ({ ...EMPTY_MATCH, ...m }))
); );
@ -501,6 +503,63 @@ describe("LLWSSimulator", () => {
expect(rmse).toBeLessThan(0.02); expect(rmse).toBeLessThan(0.02);
}); });
// Regression: the previous mapping rescaled every field onto a fixed 12501750
// Elo span, which discarded how spread out the board actually was and pulled a
// nearly flat field apart into contenders and no-hopers the market never implied.
const TIGHT_BOARD = Array.from({ length: 20 }, (_, i) => 1500 + i * 35);
it("does not inflate the favorite on a tightly priced board", async () => {
setupMockDb(
defaultParticipants(),
ALL_IDS.map((participantId, i) => ({ participantId, sourceOdds: TIGHT_BOARD[i] }))
);
const results = await new LLWSSimulator(20_000).simulate("season-1");
const market = marketProbabilities(TIGHT_BOARD);
const simulated = probsFor(results, "us-1").probFirst;
// The favorite prices near 6%. A fixed-span mapping simulated it around 13%,
// so the band is wide enough for Monte Carlo noise but nowhere near that.
expect(Math.abs(simulated - market[0])).toBeLessThan(0.015);
});
it("keeps a tightly priced field tight", async () => {
setupMockDb(
defaultParticipants(),
ALL_IDS.map((participantId, i) => ({ participantId, sourceOdds: TIGHT_BOARD[i] }))
);
const results = await new LLWSSimulator(20_000).simulate("season-1");
const probs = ALL_IDS.map((id) => probsFor(results, id).probFirst);
// Every team prices between roughly 4% and 6%, so nobody should run away with
// it and nobody should be written off.
expect(Math.max(...probs)).toBeLessThan(0.09);
expect(Math.min(...probs)).toBeGreaterThan(0.02);
});
it("rates a team with no odds entered around the middle of the field", async () => {
// us-5 is priced mid-board; blanking its odds should not move it far. The old
// 1500 fallback was the centre of the Elo scale rather than of the field, which
// promoted an unpriced team to roughly 6th of 20.
const priced = ALL_IDS.map((participantId, i) => ({ participantId, sourceOdds: BOARD[i] }));
setupMockDb(defaultParticipants(), priced);
const withOdds = probsFor(
await new LLWSSimulator(20_000).simulate("season-1"), "us-5"
).probFirst;
const blanked = priced.map((row) =>
row.participantId === "us-5" ? { ...row, sourceOdds: null } : row
);
setupMockDb(defaultParticipants(), blanked);
const withoutOdds = probsFor(
await new LLWSSimulator(20_000).simulate("season-1"), "us-5"
).probFirst;
// Priced 5th of 20, so the median rating should land it in the same territory.
expect(withoutOdds).toBeGreaterThan(withOdds / 2);
expect(withoutOdds).toBeLessThan(withOdds * 2);
});
it("does not starve longshots of championship probability", async () => { it("does not starve longshots of championship probability", async () => {
setupMockDb(defaultParticipants(), boardEvRows()); setupMockDb(defaultParticipants(), boardEvRows());
const results = await new LLWSSimulator(20_000).simulate("season-1"); const results = await new LLWSSimulator(20_000).simulate("season-1");
@ -549,6 +608,38 @@ describe("LLWSSimulator", () => {
expect(results.reduce((s, r) => s + r.probabilities.probFirst, 0)).toBeCloseTo(1.0, 1); expect(results.reduce((s, r) => s + r.probabilities.probFirst, 0)).toBeCloseTo(1.0, 1);
}); });
it("uses the most recent bracket event when several exist", async () => {
// A stale event's matches would carry no draw, silently reverting to a
// randomized one and discarding every recorded result.
setupMockDb(defaultParticipants(), makeEvRows(ALL_IDS), seededBracket());
mockDb.query.scoringEvents.findMany.mockResolvedValue([
{ id: "stale-event", createdAt: new Date("2026-07-01") },
{ id: "event-1", createdAt: new Date("2026-08-01") },
]);
const results = await new LLWSSimulator(20_000).simulate("season-1");
// Bracket mode is in force, so the fixed bye slots still show their advantage.
expect(probsFor(results, "us-9").probFirst).toBeGreaterThan(
probsFor(results, "us-1").probFirst
);
});
it("throws when the bracket is only partially seeded", async () => {
// participant1Id/participant2Id are ON DELETE SET NULL, so removing and
// re-adding one participant mid-tournament empties a single slot. Falling back
// to a randomized draw there would put eliminated teams back in contention.
const holed = seededBracket().map((m) =>
m.round === "Opening Round" && m.matchNumber === 3
? { ...m, participant2Id: null }
: m
);
setupMockDb(defaultParticipants(), makeEvRows(ALL_IDS), holed);
await expect(new LLWSSimulator(100).simulate("season-1")).rejects.toThrow(
/partially seeded \(19 of 20/
);
});
it("throws when the bracket seeds the same team into two slots", async () => { it("throws when the bracket seeds the same team into two slots", async () => {
const duplicated = seededBracket().map((m) => const duplicated = seededBracket().map((m) =>
m.round === "Opening Round" && m.matchNumber === 2 m.round === "Opening Round" && m.matchNumber === 2

View file

@ -39,11 +39,12 @@
* A championship future already bakes in the ~6 wins needed to lift the trophy, so * A championship future already bakes in the ~6 wins needed to lift the trophy, so
* using it directly as a single-game strength (p1 / (p1 + p2)) makes every * using it directly as a single-game strength (p1 / (p1 + p2)) makes every
* individual game as lopsided as the whole tournament and compounds the favorite's * individual game as lopsided as the whole tournament and compounds the favorite's
* edge over and over. convertFuturesToElo undoes that compression first (the * edge over and over. buildLLWSElos undoes that compression first (the empirically
* empirically calibrated cube-root step in decompressProbability) before mapping to * calibrated cube-root step in decompressProbability) before mapping to an Elo
* an Elo scale, which is how the other bracket simulators on the platform consume * scale, and LLWS_PARITY_FACTOR then widens the Elo curve to reflect how much
* futures. LLWS_PARITY_FACTOR then widens the Elo curve to reflect how much * single-game variance there is in six-inning Little League baseball. Unlike the
* single-game variance there is in six-inning Little League baseball. * shared convertFuturesToElo helper, the mapping preserves how spread out the board
* actually is see buildLLWSElos for why that matters.
* *
* Side assignment (externalId): "US" or "Intl". The legacy pool suffixes * Side assignment (externalId): "US" or "Intl". The legacy pool suffixes
* ("US:A", "US:B", "Intl:A", "Intl:B") are still accepted and read as the side * ("US:A", "US:B", "Intl:A", "Intl:B") are still accepted and read as the side
@ -74,7 +75,8 @@ import { database } from "~/database/context";
import { and, eq } from "drizzle-orm"; import { and, eq } from "drizzle-orm";
import * as schema from "~/database/schema"; import * as schema from "~/database/schema";
import { import {
convertFuturesToElo, convertAmericanOddsToProbability,
decompressProbability,
eloWinProbabilityWithParity, eloWinProbabilityWithParity,
} from "~/services/probability-engine"; } from "~/services/probability-engine";
import { llwsMatchNumber } from "~/lib/bracket-templates"; import { llwsMatchNumber } from "~/lib/bracket-templates";
@ -92,20 +94,36 @@ const LLWS_TEMPLATE_ID = "llws_20";
/** /**
* Elo scaling for a single LLWS game. * Elo scaling for a single LLWS game.
* *
* Much higher than the 400-point standard because a six-inning Little League game * Higher than the 400-point standard because a six-inning Little League game between
* between 12-year-olds is far closer to a coin flip than a pro game: one pitcher, one * 12-year-olds is far closer to a coin flip than a pro game: one pitcher, one big
* big inning, and the mercy rule all compress the gap. * inning, and the mercy rule all compress the gap.
* *
* Calibrated by sweeping this value until a randomized-draw simulation reproduces the * Calibrated by sweeping this value until a randomized-draw simulation reproduces the
* championship futures it was fed. Against a representative 20-team futures board, * championship futures it was fed, across boards of different shape (see
* simulated championship probability vs. the market it came from: * LLWS_ELO_SPREAD for why the shape matters). Total RMSE over a wide board, a
* parity 400 favorite 21.8% priced, 51.5% simulated (RMSE 0.075) * top-heavy board, and a nearly flat one:
* parity 700 favorite 21.8% priced, 31.5% simulated (RMSE 0.026) * parity 450 0.028
* parity 1000 favorite 21.8% priced, 21.8% simulated (RMSE 0.003) chosen * parity 550 0.016 chosen
* parity 1200 favorite 21.8% priced, 18.3% simulated (RMSE 0.010) * parity 750 0.040
* parity 1000 0.061
* Overridable per season via the `parityFactor` simulator config. * Overridable per season via the `parityFactor` simulator config.
*/ */
const LLWS_PARITY_FACTOR = 1_000; const LLWS_PARITY_FACTOR = 550;
/**
* Elo points per natural-log unit of relative team strength.
*
* Only the ratio LLWS_ELO_SPREAD / parityFactor affects the simulation, so this fixes
* the readable scale of the ratings and LLWS_PARITY_FACTOR does the calibrating. 300
* puts a typical 20-team board in the familiar ~13501700 range.
*/
const LLWS_ELO_SPREAD = 300;
/**
* Power transform undoing the compounding baked into a championship future.
* Matches DEFAULT_CALIBRATION.exponent in the probability engine.
*/
const LLWS_DECOMPRESSION_EXPONENT = 0.33;
// ─── Types ──────────────────────────────────────────────────────────────────── // ─── Types ────────────────────────────────────────────────────────────────────
@ -362,21 +380,58 @@ function inferExternalIdFromName(name: string): string {
/** /**
* Map participants to single-game Elo ratings from their championship futures. * Map participants to single-game Elo ratings from their championship futures.
* *
* Teams with no odds entered sit at DEFAULT_ELO, which is also where every team lands * Deliberately NOT convertFuturesToElo. That helper finishes by rescaling the field
* when the season has no odds at all so an unconfigured season still simulates as a * onto a fixed 12501750 span (mapToElo), which throws away how spread out the board
* field of coin flips rather than throwing. * actually is: a board whose favorite is priced at 22% and one whose favorite is
* priced at 6% both come out 500 Elo wide, so the tight board's field gets pulled
* apart into contenders and no-hopers that the market never implied. On such a board
* that inflated the favorite from 6% to 13%.
*
* Instead the decompressed strengths are mapped by their log-ratio to the field's
* geometric mean, which preserves dispersion: a tight board yields a narrow Elo span
* and a top-heavy one a wide span, both centred on DEFAULT_ELO.
*
* Returns the ratings alongside the rating to use for a team with no odds entered
* the median of the priced field, so leaving odds blank neither promotes nor buries a
* team. (DEFAULT_ELO is the centre of the scale, but futures fields are skewed, so on
* a typical board it would rank a team around 6th of 20.)
*/ */
export function buildLLWSElos( export function buildLLWSElos(
evRows: Array<{ participantId: string; sourceOdds: number | null }> evRows: Array<{ participantId: string; sourceOdds: number | null }>
): Map<string, number> { ): { elos: Map<string, number>; unpricedElo: number } {
const oddsInput = evRows const priced = evRows.filter((row) => row.sourceOdds !== null);
.filter((row) => row.sourceOdds !== null)
.map((row) => ({ participantId: row.participantId, odds: row.sourceOdds ?? 0 }));
// convertFuturesToElo needs a spread to normalise against; a single priced team // A single priced team carries no information about the rest of the field, so
// carries no information about the rest of the field. // there is nothing to normalise against — treat the season as unpriced.
if (oddsInput.length < 2) return new Map(); if (priced.length < 2) return { elos: new Map(), unpricedElo: DEFAULT_ELO };
return convertFuturesToElo(oddsInput);
const rawProbs = priced.map((row) => convertAmericanOddsToProbability(row.sourceOdds ?? 0));
const rawSum = rawProbs.reduce((a, b) => a + b, 0);
if (rawSum <= 0) return { elos: new Map(), unpricedElo: DEFAULT_ELO };
// Vig-removed championship probability → single-game strength.
const logStrengths = rawProbs.map((prob) =>
Math.log(
Math.max(decompressProbability(prob / rawSum, LLWS_DECOMPRESSION_EXPONENT), Number.MIN_VALUE)
)
);
const meanLog = logStrengths.reduce((a, b) => a + b, 0) / logStrengths.length;
const elos = new Map<string, number>(
priced.map((row, i) => [
row.participantId,
DEFAULT_ELO + LLWS_ELO_SPREAD * (logStrengths[i] - meanLog),
])
);
return { elos, unpricedElo: median([...elos.values()]) };
}
function median(values: number[]): number {
if (values.length === 0) return DEFAULT_ELO;
const sorted = values.toSorted((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
} }
// ─── Bracket loading ────────────────────────────────────────────────────────── // ─── Bracket loading ──────────────────────────────────────────────────────────
@ -384,10 +439,16 @@ export function buildLLWSElos(
/** /**
* Read the seeded llws_20 bracket for this season, if there is one. * Read the seeded llws_20 bracket for this season, if there is one.
* *
* Returns null when no bracket exists yet or its opening slots have not been filled * Returns null only when the bracket carries no draw at all no matches, or a
* in in that case the caller falls back to a randomized draw. Throws when the * freshly generated bracket with every slot still empty in which case the caller
* bracket is seeded with participants that don't belong to the season, which is a * falls back to a randomized draw.
* misconfiguration worth surfacing rather than silently ignoring. *
* A *partially* seeded bracket is an error rather than a fallback. Silently falling
* back there would throw away the real draw and every recorded result along with it,
* putting eliminated teams back in contention; and it is reachable in practice,
* because playoff_matches.participant1Id/participant2Id are ON DELETE SET NULL, so
* removing and re-adding a single participant mid-tournament empties a slot.
* Likewise, a bracket seeded with unknown or duplicated participants fails loudly.
*/ */
export function readBracketSlots( export function readBracketSlots(
matches: BracketMatch[], matches: BracketMatch[],
@ -396,39 +457,59 @@ export function readBracketSlots(
if (matches.length === 0) return null; if (matches.length === 0) return null;
const byKey = new Map(matches.map((m) => [matchKey(m.round, m.matchNumber), m])); const byKey = new Map(matches.map((m) => [matchKey(m.round, m.matchNumber), m]));
const slots: Record<Side, Team[]> = { US: [], Intl: [] };
const seen = new Set<string>(); // Collect both sides' draws before deciding, so "nothing seeded" is judged over the
// whole bracket rather than one side at a time.
const draw: Record<Side, (string | null)[]> = { US: [], Intl: [] };
for (const side of ["US", "Intl"] as const) { for (const side of ["US", "Intl"] as const) {
const sideIndex = SIDE_INDEX[side]; const sideIndex = SIDE_INDEX[side];
const ids: (string | null)[] = [];
for (let local = 1; local <= 4; local++) { for (let local = 1; local <= 4; local++) {
const match = byKey.get( const match = byKey.get(
matchKey("Opening Round", llwsMatchNumber("Opening Round", sideIndex, local)) matchKey("Opening Round", llwsMatchNumber("Opening Round", sideIndex, local))
); );
ids.push(match?.participant1Id ?? null, match?.participant2Id ?? null); draw[side].push(match?.participant1Id ?? null, match?.participant2Id ?? null);
} }
for (let local = 1; local <= 2; local++) { for (let local = 1; local <= 2; local++) {
const match = byKey.get( const match = byKey.get(
matchKey("Winners Round 2", llwsMatchNumber("Winners Round 2", sideIndex, local)) matchKey("Winners Round 2", llwsMatchNumber("Winners Round 2", sideIndex, local))
); );
ids.push(match?.participant1Id ?? null); draw[side].push(match?.participant1Id ?? null);
}
} }
// An unseeded (or partially seeded) bracket carries no draw information. const allSlots = [...draw.US, ...draw.Intl];
if (ids.some((id) => id === null)) return null; const seededCount = allSlots.filter((id) => id !== null).length;
for (const id of ids) { // Generated but not yet filled in — no draw to honor.
if (seen.has(id as string)) { if (seededCount === 0) return null;
throw new Error(`LLWS bracket seeds participant ${id} into more than one slot.`);
if (seededCount < allSlots.length) {
throw new Error(
`LLWS bracket is only partially seeded (${seededCount} of ${allSlots.length} slots ` +
`filled). Re-seed the bracket in Admin → Bracket before simulating; simulating ` +
`around the gap would discard the draw and every recorded result.`
);
} }
seen.add(id as string);
const team = teamsById.get(id as string); const slots: Record<Side, Team[]> = { US: [], Intl: [] };
const seen = new Set<string>();
for (const side of ["US", "Intl"] as const) {
for (const id of draw[side]) {
const participantId = id as string;
if (seen.has(participantId)) {
throw new Error(
`LLWS bracket seeds participant ${participantId} into more than one slot.`
);
}
seen.add(participantId);
const team = teamsById.get(participantId);
if (!team) { if (!team) {
throw new Error( throw new Error(
`LLWS bracket references participant ${id}, which is not in this sports season.` `LLWS bracket references participant ${participantId}, which is not in this sports season.`
); );
} }
// The bracket is authoritative about which side a team is on. // The bracket is authoritative about which side a team is on.
@ -472,7 +553,7 @@ export class LLWSSimulator implements Simulator {
.where(eq(schema.seasonParticipantExpectedValues.sportsSeasonId, sportsSeasonId)); .where(eq(schema.seasonParticipantExpectedValues.sportsSeasonId, sportsSeasonId));
// 3. Decompress the futures into single-game Elo ratings. // 3. Decompress the futures into single-game Elo ratings.
const eloMap = buildLLWSElos(evRows); const { elos, unpricedElo } = buildLLWSElos(evRows);
// 4. Parse externalId for each participant to determine which side they're on. // 4. Parse externalId for each participant to determine which side they're on.
// A seeded bracket overrides this below, but the field still has to be a legal // A seeded bracket overrides this below, but the field still has to be a legal
@ -487,20 +568,26 @@ export class LLWSSimulator implements Simulator {
// Provisional: a seeded bracket overwrites this below. // Provisional: a seeded bracket overwrites this below.
participantId: p.id, participantId: p.id,
side: parsed?.side ?? "Intl", side: parsed?.side ?? "Intl",
elo: eloMap.get(p.id) ?? DEFAULT_ELO, elo: elos.get(p.id) ?? unpricedElo,
}); });
} }
const teamsById = new Map(teams.map((t) => [t.participantId, t])); const teamsById = new Map(teams.map((t) => [t.participantId, t]));
// 5. Load the real bracket (draw + results so far), if one has been generated. // 5. Load the real bracket (draw + results so far), if one has been generated.
const bracketEvent = await db.query.scoringEvents.findFirst({ // If several llws_20 playoff events exist, take the most recent so a re-created
// event wins over a stale one — landing on the stale row would silently discard
// the real draw and every recorded result.
const playoffEvents = await db.query.scoringEvents.findMany({
where: and( where: and(
eq(schema.scoringEvents.sportsSeasonId, sportsSeasonId), eq(schema.scoringEvents.sportsSeasonId, sportsSeasonId),
eq(schema.scoringEvents.eventType, "playoff_game"), eq(schema.scoringEvents.eventType, "playoff_game"),
eq(schema.scoringEvents.bracketTemplateId, LLWS_TEMPLATE_ID) eq(schema.scoringEvents.bracketTemplateId, LLWS_TEMPLATE_ID)
), ),
}); });
const bracketEvent = playoffEvents.toSorted(
(a, b) => (b.createdAt?.getTime() ?? 0) - (a.createdAt?.getTime() ?? 0)
)[0];
const bracketMatches = bracketEvent const bracketMatches = bracketEvent
? await db.query.playoffMatches.findMany({ ? await db.query.playoffMatches.findMany({

View file

@ -183,7 +183,7 @@ const PROFILES: Record<SimulatorType, Omit<SimulatorManifestProfile, "simulatorT
setupSections: ["participants", "eloRatings", "futuresOdds", "bracket"], setupSections: ["participants", "eloRatings", "futuresOdds", "bracket"],
}, },
llws_bracket: { llws_bracket: {
defaultConfig: { ...BASE_CONFIG, parityFactor: 1_000, usTeamCount: 10, internationalTeamCount: 10 }, defaultConfig: { ...BASE_CONFIG, parityFactor: 550, usTeamCount: 10, internationalTeamCount: 10 },
requiredInputs: ["sourceOdds"], requiredInputs: ["sourceOdds"],
optionalInputs: ["metadata"], optionalInputs: ["metadata"],
// The bracket is optional — without one the draw is randomized — but once it // The bracket is optional — without one the draw is randomized — but once it