I flip a fair coin. Compute the probability that it lands heads. End your program with the numeric answer.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var model = function() { return flip() ? "H" : "T" };2var ANSWER = (Math.exp(Infer({method:'enumerate'}, model).score('H')));3
1var ANSWER = 0.5;
0.5000
0.5000
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ✓val+ | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ✓val+ | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ✓val+ | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ✓val+ | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ✓val+ | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ✓val+ | — | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ✓val+ | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ✓val+ | — | — | · |
I have a biased coin with P(heads) = 0.9, and a fair coin. Someone hands me one of the two coins, chosen uniformly at random, without telling me which. I flip it three times. Given that the first two flips landed heads, what is the posterior distribution over the third flip (as a boolean: true=heads, false=tails)? End your program with the Infer(...) call returning that distribution.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var flipCoin = function(coinType) {2 return coinType == "fair" ? flip() : flip(0.9);3}4var model = function() {5 var coinType = flip() ? "fair" : "biased";6 var flip1 = flipCoin(coinType);7 var flip2 = flipCoin(coinType);8 var flip3 = flipCoin(coinType);9 condition(flip1 && flip2);10 return flip3;11};12var ANSWER = (Infer({method:'enumerate'}, model));13
1var model = function() {2 // Choose coin uniformly at random3 var biased = flip(0.5);4 var p = biased ? 0.9 : 0.5;56 // Condition on first two flips being heads7 observe(Bernoulli({p: p}), true);8 observe(Bernoulli({p: p}), true);910 // Return the third flip11 return flip(p);12};1314var ANSWER = Infer({method: 'enumerate'}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
I have a biased coin with P(heads) = 0.9, and a fair coin. Someone hands me one of the two coins, chosen uniformly at random, without telling me which. I flip it three times. Given that all three flips landed heads, what is the posterior distribution over the coin type (return the string 'fair' or 'biased')? End your program with the Infer(...) call.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var flipCoin = function(coinType) {2 return coinType == "fair" ? flip() : flip(0.9);3}4var model = function() {5 var coinType = flip() ? "fair" : "biased";6 var flip1 = flipCoin(coinType);7 var flip2 = flipCoin(coinType);8 var flip3 = flipCoin(coinType);9 condition(flip1 && flip2 && flip3);10 return coinType;11};12var ANSWER = (Infer({method:'enumerate'}, model));13
1var model = function() {2 var coin = flip(0.5) ? 'biased' : 'fair';3 var p = coin === 'biased' ? 0.9 : 0.5;4 observe(Binomial({p: p, n: 3}), 3);5 return coin;6};78var ANSWER = Infer({method: 'enumerate'}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
I have a biased coin with P(heads) = 0.9, and a fair coin. Someone hands me one of the two coins, chosen uniformly at random, without telling me which. I flip it three times. Given that the first two flips were different, what is the posterior distribution over the third flip (as a boolean: true=heads, false=tails)? End your program with the Infer(...) call.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var flipCoin = function(coinType) {2 return coinType == "fair" ? flip() : flip(0.9);3}4var model = function() {5 var coinType = flip() ? "fair" : "biased";6 var flip1 = flipCoin(coinType);7 var flip2 = flipCoin(coinType);8 var flip3 = flipCoin(coinType);9 condition(flip1 != flip2);10 return flip3;11};12var ANSWER = (Infer({method:'enumerate'}, model));13
1var model = function() {2 // Choose coin uniformly at random3 var biased = flip(0.5);4 var p = biased ? 0.9 : 0.5;56 // First two flips7 var flip1 = flip(p);8 var flip2 = flip(p);910 // Condition on first two flips being different11 condition(flip1 !== flip2);1213 // Third flip14 var flip3 = flip(p);1516 return flip3;17};1819var ANSWER = Infer({method: 'enumerate'}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
Consider this generative model of a cough symptom:
1var lungCancer = flip(0.01);2var cold = flip(0.2);3var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));
Show that *intervening* on lungCancer (setting it to true directly) produces the same marginal of cough as *conditioning* on lungCancer. Return an object literal with three keys, each a distribution over cough:
- original: the unconditional marginal
- intervention: the marginal under lungCancer = true
- conditioning: the marginal under condition(lungCancer)
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (({2 original: Infer({method: "enumerate"}, function() {3 var lungCancer = flip(0.01);4 var cold = flip(0.2);5 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));6 return cough;7 }),8 intervention: Infer({method: "enumerate"}, function() {9 var lungCancer = true;10 var cold = flip(0.2);11 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));12 return cough;13 }),14 conditioning: Infer({method: "enumerate"}, function() {15 var lungCancer = flip(0.01);16 condition(lungCancer);17 var cold = flip(0.2);18 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));19 return cough;20 })21}));22
1var model = function() {2 var lungCancer = flip(0.01);3 var cold = flip(0.2);4 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));5 return cough;6};78var interventionModel = function() {9 var lungCancer = true;10 var cold = flip(0.2);11 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));12 return cough;13};1415var conditioningModel = function() {16 var lungCancer = flip(0.01);17 var cold = flip(0.2);18 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));19 condition(lungCancer);20 return cough;21};2223var ANSWER = {24 original: Infer({method: 'enumerate'}, model),25 intervention: Infer({method: 'enumerate'}, interventionModel),26 conditioning: Infer({method: 'enumerate'}, conditioningModel)27};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Using the same lungCancer/cold/cough generative model:
1var lungCancer = flip(0.01);2var cold = flip(0.2);3var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));
Show that *intervening* on cough (setting cough = true directly) produces a different marginal of lungCancer than *conditioning* on cough does. Return an object literal with three distributions over lungCancer:
- original: the unconditional marginal
- intervention: replace cough = (...) with cough = true
- conditioning: keep the original assignment and add condition(cough)
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (({2 original: Infer({method: "enumerate"}, function() {3 var lungCancer = flip(0.01);4 var cold = flip(0.2);5 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));6 return lungCancer;7 }),8 intervention: Infer({method: "enumerate"}, function() {9 var lungCancer = flip(0.01);10 var cold = flip(0.2);11 var cough = true;12 return lungCancer;13 }),14 conditioning: Infer({method: "enumerate"}, function() {15 var lungCancer = flip(0.01);16 var cold = flip(0.2);17 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));18 condition(cough);19 return lungCancer;20 })21}));22
1var model_original = function() {2 var lungCancer = flip(0.01);3 var cold = flip(0.2);4 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));5 return lungCancer;6};78var model_intervention = function() {9 var lungCancer = flip(0.01);10 var cold = flip(0.2);11 var cough = true;12 return lungCancer;13};1415var model_conditioning = function() {16 var lungCancer = flip(0.01);17 var cold = flip(0.2);18 var cough = (cold && flip(0.5)) || (lungCancer && flip(0.3));19 condition(cough);20 return lungCancer;21};2223var ANSWER = {24 original: Infer({method: 'enumerate'}, model_original),25 intervention: Infer({method: 'enumerate'}, model_intervention),26 conditioning: Infer({method: 'enumerate'}, model_conditioning)27};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Here is a model of how niceness affects whether a person smiles:
1var smilesModel = function() {2 var nice = mem(function(person) { flip(.7) });3 var smiles = function(person) {4 return nice(person) ? flip(.8) : flip(.5);5 }6 condition(smiles('alice') && smiles('bob') && smiles('alice'));7 return nice('alice');8}
Extend this into a function extendedSmilesModel that captures two additional factors:
1. People smile 80% of the time if they want something from you, and 50% otherwise.
2. Nice people want something from you 20% of the time; non-nice people 50%.
Nice people should still smile more often regardless of whether they want something. Niceness is a stable property of a person; whether they want something can vary. Have extendedSmilesModel return whether Alice smiles today, and end your program with Infer({method: 'enumerate'}, extendedSmilesModel) returning that distribution.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var extendedSmilesModel = function() {2 var nice = mem(function(person) { flip(.7) });3 var wantsSomething = function(person) {4 return flip(nice(person) ? .2 : .5);5 }6 var smiles = function(person, wants) {7 return (wants ? flip(.8) : flip(.5))8 || (nice(person) ? flip(.8) : flip(.5));9 }10 var wants = wantsSomething('alice');11 return smiles('alice', wants);12};13var ANSWER = (Infer({method: "enumerate"}, extendedSmilesModel));14
1var extendedSmilesModel = function() {2 var nice = mem(function(person) { return flip(.7); });34 var wants = function(person) {5 return nice(person) ? flip(.2) : flip(.5);6 };78 var smiles = function(person) {9 var w = wants(person);10 var n = nice(person);11 // Nice people smile more regardless of wanting something12 // wants -> 80% smile; not wants -> 50% smile13 // nice adds extra smile probability14 if (n && w) {15 return flip(.8);16 } else if (n && !w) {17 return flip(.8);18 } else if (!n && w) {19 return flip(.8);20 } else {21 return flip(.5);22 }23 };2425 return smiles('alice');26};2728// Wait, re-reading: "Nice people should still smile more often regardless of whether they want something"29// So niceness affects smile rate independently of wanting.30// Let me re-interpret:31// - wants -> 80% smile, not wants -> 50% smile (base from wanting)32// - nice people smile more: perhaps nice AND wants -> higher, nice AND not wants -> still higher than non-nice33// A natural interpretation: smile probability depends on BOTH niceness and wanting.34// nice: flip(.8) if nice else flip(.5) [original]35// wants: flip(.8) if wants else flip(.5)36// Combined: nice AND wants -> flip(.8) (both push up), nice AND not wants -> flip(.8) (nice dominates),37// not nice AND wants -> flip(.8) (wants dominates), not nice AND not wants -> flip(.5)38// But that doesn't differentiate niceness well.39// Better: use additive/multiplicative model or just define 4 cases explicitly:40// nice+wants: .9, nice+!wants: .8, !nice+wants: .8, !nice+!wants: .54142var extendedSmilesModel = function() {43 var nice = mem(function(person) { return flip(.7); });4445 var wants = function(person) {46 return nice(person) ? flip(.2) : flip(.5);47 };4849 var smiles = function(person) {50 var n = nice(person);51 var w = wants(person);52 if (n && w) {53 return flip(.9);54 } else if (n && !w) {55 return flip(.8);56 } else if (!n && w) {57 return flip(.8);58 } else {59 return flip(.5);60 }61 };6263 return smiles('alice');64};6566var ANSWER = Infer({method: 'enumerate'}, extendedSmilesModel);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◐TV<.5 | 0.119 | 0.0559 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◐TV<.5 | 0.119 | 0.0559 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◐TV<.5 | 0.184 | 0.1229 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◐TV<.5 | 0.062 | 0.0168 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◐TV<.5 | 0.119 | 0.0559 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◐TV<.5 | 0.117 | 0.0541 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◐TV<.5 | 0.102 | 0.0421 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.102 | 0.0421 | · |
Given this extendedSmilesModel skeleton:
1var extendedSmilesModel = function() {2 var nice = mem(function(person) { flip(.7) });3 var wantsSomething = function(person) {4 return flip(nice(person) ? .2 : .5);5 }6 var smiles = function(person, wants) {7 return (wants ? flip(.8) : flip(.5))8 || (nice(person) ? flip(.8) : flip(.5));9 }10 // ... your code here ...11}
Suppose you've seen Bob five times this week and each time he was *not* smiling. Today, you see him smiling. Modify the model body to compute the posterior probability that Bob wants something from you today. Niceness is stable (use mem as given); whether he wants something varies day-to-day. End your program with Infer({method: 'enumerate'}, extendedSmilesModel).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var extendedSmilesModel = function() {2 var nice = mem(function(person) { flip(.7) });3 var wantsSomething = function(person) {4 return flip(nice(person) ? .2 : .5);5 }6 var smiles = function(person, wants) {7 return (wants ? flip(.8) : flip(.5))8 || (nice(person) ? flip(.8) : flip(.5));9 }10 var wantsToday = wantsSomething('bob');11 condition(!smiles('bob', wantsSomething('bob')));12 condition(!smiles('bob', wantsSomething('bob')));13 condition(!smiles('bob', wantsSomething('bob')));14 condition(!smiles('bob', wantsSomething('bob')));15 condition(!smiles('bob', wantsSomething('bob')));16 condition(smiles('bob', wantsToday));17 return wantsToday;18};19var ANSWER = (Infer({method: "enumerate"}, extendedSmilesModel));20
1var extendedSmilesModel = function() {2 var nice = mem(function(person) { flip(.7) });3 var wantsSomething = function(person) {4 return flip(nice(person) ? .2 : .5);5 };6 var smiles = function(person, wants) {7 return (wants ? flip(.8) : flip(.5))8 || (nice(person) ? flip(.8) : flip(.5));9 };1011 // Observe Bob not smiling five times (niceness is stable, wants varies each day)12 var bobNice = nice("Bob");1314 // Each of the 5 past days: wants something varies, smiles is false15 map(function(i) {16 var wants = wantsSomething("Bob");17 condition(smiles("Bob", wants) === false);18 }, _.range(5));1920 // Today: observe Bob smiling21 var wantsToday = wantsSomething("Bob");22 condition(smiles("Bob", wantsToday) === true);2324 return wantsToday;25};2627var ANSWER = Infer({method: 'enumerate'}, extendedSmilesModel);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
I have a sprinkler in my garden that turns on each morning at random - half the time, independently each day. I live in a city where it rains on 30% of mornings. The lawn gets wet whenever the sprinkler turns on, it rains, or both. One morning, I notice my lawn is wet. Return an object literal with two keys:
- rain: posterior distribution over whether it rained
- sprinkler: posterior distribution over whether the sprinkler turned on
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (({2 rain: Infer({method: "enumerate"}, function() {3 var sprinkler = flip();4 var rain = flip(0.3);5 var wetLawn = sprinkler || rain;6 condition(wetLawn);7 return rain;8 }),9 sprinkler: Infer({method: "enumerate"}, function() {10 var sprinkler = flip();11 var rain = flip(0.3);12 var wetLawn = sprinkler || rain;13 condition(wetLawn);14 return sprinkler;15 })16}));17
1var model = function() {2 var rained = flip(0.3);3 var sprinkler = flip(0.5);4 var wet = rained || sprinkler;5 condition(wet);6 return { rain: rained, sprinkler: sprinkler };7};89var joint = Infer({method: 'enumerate'}, model);1011var rainDist = Infer({method: 'enumerate'}, function() {12 var s = sample(joint);13 return s.rain;14});1516var sprinklerDist = Infer({method: 'enumerate'}, function() {17 var s = sample(joint);18 return s.sprinkler;19});2021var ANSWER = { rain: rainDist, sprinkler: sprinklerDist };
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◉TV<.05 | 2.8e-17 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◉TV<.05 | 2.8e-17 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◉TV<.05 | 2.8e-17 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 2.8e-17 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◉TV<.05 | 2.8e-17 | 0.0000 | · |
I have a sprinkler in my garden that turns on each morning at random - half the time, independently each day. I live in a city where it rains on 30% of mornings. The lawn gets wet whenever the sprinkler turns on, it rains, or both. My neighbor Kelsey has the same kind of sprinkler (independent of mine, also turning on with probability 0.5 each morning). One morning, both my lawn and Kelsey's lawn are wet. End your program with Infer(...) returning the posterior distribution over whether it rained.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (Infer({method: "enumerate"}, function() {2 var rain = flip(0.3);3 var mySprinkler = flip();4 var herSprinkler = flip();5 var myLawnIsWet = mySprinkler || rain;6 var herLawnIsWet = herSprinkler || rain;7 condition(myLawnIsWet && herLawnIsWet);8 return rain;9}));10
1var model = function() {2 var rain = flip(0.3);3 var sprinkler = flip(0.5);4 var kelseySprinkler = flip(0.5);56 var myLawnWet = rain || sprinkler;7 var kelseyLawnWet = rain || kelseySprinkler;89 condition(myLawnWet && kelseyLawnWet);1011 return rain;12};1314var ANSWER = Infer({method: 'enumerate'}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
I have a sprinkler in my garden that turns on each morning at random - half the time, independently each day. I live in a city where it rains on 30% of mornings. Lawns are wet whenever the sprinkler turns on, it rains, or both. Five people in the area - me, Kelsey, Kevin, Manu, and Josh - all have the same kind of sprinkler (each independent, p=0.5). One morning all five lawns are wet. Use mem so each person's sprinkler is modeled independently. End your program with Infer(...) returning the posterior over whether it rained.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (Infer({method: "enumerate"}, function() {2 var rain = flip(0.3);3 var sprinkler = mem(function(person) { return flip() });4 var wetLawn = function(person) { return rain || sprinkler(person) };5 condition(wetLawn("me"));6 condition(wetLawn("Kelsey"));7 condition(wetLawn("Kevin"));8 condition(wetLawn("Manu"));9 condition(wetLawn("Josh"));10 return rain;11}));12
1var model = function() {2 var rained = flip(0.3);34 var sprinklerOn = mem(function(person) {5 return flip(0.5);6 });78 var lawnWet = function(person) {9 return sprinklerOn(person) || rained;10 };1112 var people = ["me", "Kelsey", "Kevin", "Manu", "Josh"];1314 map(function(person) {15 condition(lawnWet(person));16 }, people);1718 return rained;19};2021var ANSWER = Infer({method: 'enumerate'}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◔TV<1 | 0.632 | 1.2924 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 6.9e-18 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◉TV<.05 | 6.9e-18 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◉TV<.05 | 6.9e-18 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 6.9e-18 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◉TV<.05 | 6.9e-18 | 0.0000 | · |
A machine randomly draws a letter of the word "game" with probabilities {g: 0.05, a: 0.45, m: 0.05, e: 0.45}. Bob's probability of winning given letter h is 1/k^2 where k is the position of that letter in the word "game" (so g=1, a=2, m=3, e=4). We observe that Bob won, but don't know which letter he drew. Fill in the ...'s in the program below to compute p(letter | win), and end your program with the resulting distribution:
1var checkVowel = function(letter) { _.includes(['a', 'e', 'i', 'o', 'u'], letter) };2var letterVals = ['g', 'a', 'm', 'e'];3var letterProbs = map(function(letter) { checkVowel(letter) ? 0.45 : 0.05 }, letterVals);4var letters = Categorical({vs: letterVals, ps: letterProbs});56Infer({method: 'enumerate'}, function() {7 var letter = sample(letters);8 var position = letterVals.indexOf(letter) + 1;9 var winProb = 1 / Math.pow(position, 2);10 condition(...);11 return ...;12});
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var checkVowel = function(letter) { _.includes(['a', 'e', 'i', 'o', 'u'], letter) };2var letterVals = ['g', 'a', 'm', 'e'];3var letterProbs = map(function(letter) { checkVowel(letter) ? 0.45 : 0.05 }, letterVals);4var letters = Categorical({vs: letterVals, ps: letterProbs});5var ANSWER = (Infer({method: 'enumerate'}, function() {6 var letter = sample(letters);7 var position = letterVals.indexOf(letter) + 1;8 var winProb = 1 / Math.pow(position, 2);9 condition(flip(winProb));10 return letter;11}));12
1var checkVowel = function(letter) { _.includes(['a', 'e', 'i', 'o', 'u'], letter) };2var letterVals = ['g', 'a', 'm', 'e'];3var letterProbs = map(function(letter) { checkVowel(letter) ? 0.45 : 0.05 }, letterVals);4var letters = Categorical({vs: letterVals, ps: letterProbs});56var ANSWER = Infer({method: 'enumerate'}, function() {7 var letter = sample(letters);8 var position = _.indexOf(letterVals, letter) + 1;9 var winProb = 1 / Math.pow(position, 2);10 condition(flip(winProb));11 return letter;12});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Same casino-game setup: letters {g, a, m, e} drawn with probs {0.05, 0.45, 0.05, 0.45}, win prob = 1/k^2 with k = position in "game". Fill in the program below so the posterior distribution is over vowel vs consonant (instead of over the letter itself), given that Bob won:
1var checkVowel = function(letter) { _.includes(['a', 'e', 'i', 'o', 'u'], letter) };2var letterVals = ['g', 'a', 'm', 'e'];3var letterProbs = map(function(letter) { checkVowel(letter) ? 0.45 : 0.05 }, letterVals);4var letters = Categorical({vs: letterVals, ps: letterProbs});56Infer({method: 'enumerate'}, function() {7 var letter = sample(letters);8 var position = letterVals.indexOf(letter) + 1;9 var winProb = 1 / Math.pow(position, 2);10 condition(...);11 return ...;12});
End your program with the resulting Infer(...) distribution.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var checkVowel = function(letter) { _.includes(['a', 'e', 'i', 'o', 'u'], letter) };2var letterVals = ['g', 'a', 'm', 'e'];3var letterProbs = map(function(letter) { checkVowel(letter) ? 0.45 : 0.05 }, letterVals);4var letters = Categorical({vs: letterVals, ps: letterProbs});5var ANSWER = (Infer({method: 'enumerate'}, function() {6 var letter = sample(letters);7 var position = letterVals.indexOf(letter) + 1;8 var winProb = 1 / Math.pow(position, 2);9 condition(flip(winProb));10 return checkVowel(letter) ? 'vowel' : 'consonant';11}));12
1var checkVowel = function(letter) { _.includes(['a', 'e', 'i', 'o', 'u'], letter) };2var letterVals = ['g', 'a', 'm', 'e'];3var letterProbs = map(function(letter) { checkVowel(letter) ? 0.45 : 0.05 }, letterVals);4var letters = Categorical({vs: letterVals, ps: letterProbs});56var ANSWER = Infer({method: 'enumerate'}, function() {7 var letter = sample(letters);8 var position = _.indexOf(letterVals, letter) + 1;9 var winProb = 1 / Math.pow(position, 2);10 condition(flip(winProb));11 return checkVowel(letter) ? 'vowel' : 'consonant';12});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Given this binomial model where predictive distributions are computed for the same number of attempts as the observed data:
1var k = 1; var n = 20;2var priorDist = Uniform({a: 0, b: 1});3var model = function() {4 var p = sample(priorDist);5 observe(Binomial({p: p, n: n}), k);6 var posteriorPredictive = binomial(p, n);7 var prior_p = sample(priorDist);8 var priorPredictive = binomial(prior_p, n);9 return {prior: prior_p, priorPredictive, posterior: p, posteriorPredictive};10}11Infer({method: "MCMC", samples: 2500, lag: 50}, model);
Predictive distributions can be used to predict the outcome of a *different* experiment with new_n != n. Modify the model so the posterior predictive uses new_n = 5 attempts (while observed data is still k=1 success in n=20 attempts). Use Beta(a=1, b=1) as the prior. End with the Infer(...) call.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var k = 1;2var n = 20;3var new_n = 5;4var priorDist = Beta({a: 1, b: 1});56var model = function() {7 var p = sample(priorDist);8 observe(Binomial({p : p, n: n}), k);9 var posteriorPredictive = binomial(p, new_n);10 var prior_p = sample(priorDist);11 var priorPredictive = binomial(prior_p, n);12 return {13 prior: prior_p, priorPredictive : priorPredictive,14 posterior : p, posteriorPredictive : posteriorPredictive15 };16};17var ANSWER = (Infer({method: "MCMC", samples: 2500, lag: 50}, model));18
1var k = 1;2var n = 20;3var new_n = 5;4var priorDist = Beta({a: 1, b: 1});56var model = function() {7 var p = sample(priorDist);8 observe(Binomial({p: p, n: n}), k);9 var posteriorPredictive = binomial(p, new_n);10 var prior_p = sample(priorDist);11 var priorPredictive = binomial(prior_p, n);12 return {prior: prior_p, priorPredictive, posterior: p, posteriorPredictive};13};1415var ANSWER = Infer({method: "MCMC", samples: 2500, lag: 50}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 1.000 | 15.2029 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Imagine that you are an epidemiologist determining people's cause of death. There are two main diseases: cancer (rare, P = 0.00001, often fatal P(death|cancer)=0.9) and the common cold (P = 0.2, rarely fatal P(death|cold)=0.00006). Very rarely, people die of other causes (P = 0.000000001). Return an object literal with four marginals over cancer:
- prior: unconditional
- death: given death
- deathAndCold: given death && cold
- deathAndNoCold: given death && !cold
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (({2 prior: Infer({method: 'enumerate'}, function() {3var cancer = flip(0.00001);4var cold = flip(0.2);5var death_by_cancer = cancer && flip(0.9);6var death_by_cold = cold && flip(0.00006);7var other_death = flip(0.000000001);8var death = death_by_cancer || death_by_cold || other_death;9 return cancer;10 }),11 death: Infer({method: 'enumerate'}, function() {12var cancer = flip(0.00001);13var cold = flip(0.2);14var death_by_cancer = cancer && flip(0.9);15var death_by_cold = cold && flip(0.00006);16var other_death = flip(0.000000001);17var death = death_by_cancer || death_by_cold || other_death;18 condition(death);19 return cancer;20 }),21 deathAndCold: Infer({method: 'enumerate'}, function() {22var cancer = flip(0.00001);23var cold = flip(0.2);24var death_by_cancer = cancer && flip(0.9);25var death_by_cold = cold && flip(0.00006);26var other_death = flip(0.000000001);27var death = death_by_cancer || death_by_cold || other_death;28 condition(death && cold);29 return cancer;30 }),31 deathAndNoCold: Infer({method: 'enumerate'}, function() {32var cancer = flip(0.00001);33var cold = flip(0.2);34var death_by_cancer = cancer && flip(0.9);35var death_by_cold = cold && flip(0.00006);36var other_death = flip(0.000000001);37var death = death_by_cancer || death_by_cold || other_death;38 condition(death && !cold);39 return cancer;40 })41}));42
1var model = function() {2 var cancer = flip(0.00001);3 var cold = flip(0.2);45 var pDeath = (cancer ? 0.9 : 0) +6 (cold ? 0.00006 : 0) +7 0.000000001;8 // Clamp to [0,1]9 var pDeathClamped = Math.min(1, pDeath);1011 var death = flip(pDeathClamped);1213 return {cancer: cancer, cold: cold, death: death};14};1516var prior = Infer({method: 'enumerate'}, function() {17 var cancer = flip(0.00001);18 return cancer;19});2021var death = Infer({method: 'enumerate'}, function() {22 var r = model();23 condition(r.death);24 return r.cancer;25});2627var deathAndCold = Infer({method: 'enumerate'}, function() {28 var r = model();29 condition(r.death && r.cold);30 return r.cancer;31});3233var deathAndNoCold = Infer({method: 'enumerate'}, function() {34 var r = model();35 condition(r.death && !r.cold);36 return r.cancer;37});3839var ANSWER = {40 prior: prior,41 death: death,42 deathAndCold: deathAndCold,43 deathAndNoCold: deathAndNoCold44};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◉TV<.05 | 4.2e-16 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 3.8e-6 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◉TV<.05 | 6.8e-6 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◉TV<.05 | 6.8e-6 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Imagine that you are an epidemiologist determining people's cause of death. There are two main diseases: cancer (rare, P = 0.00001, often fatal P(death|cancer)=0.9) and the common cold (P = 0.2, rarely fatal P(death|cold)=0.00006). Very rarely, people die of other causes (P = 0.000000001). Return an object literal with four marginals over cold:
- prior: unconditional
- death: given death
- deathAndCancer: given death && cancer
- deathAndNoCancer: given death && !cancer
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (({2 prior: Infer({method: 'enumerate'}, function() {3var cancer = flip(0.00001);4var cold = flip(0.2);5var death_by_cancer = cancer && flip(0.9);6var death_by_cold = cold && flip(0.00006);7var other_death = flip(0.000000001);8var death = death_by_cancer || death_by_cold || other_death;9 return cold;10 }),11 death: Infer({method: 'enumerate'}, function() {12var cancer = flip(0.00001);13var cold = flip(0.2);14var death_by_cancer = cancer && flip(0.9);15var death_by_cold = cold && flip(0.00006);16var other_death = flip(0.000000001);17var death = death_by_cancer || death_by_cold || other_death;18 condition(death);19 return cold;20 }),21 deathAndCancer: Infer({method: 'enumerate'}, function() {22var cancer = flip(0.00001);23var cold = flip(0.2);24var death_by_cancer = cancer && flip(0.9);25var death_by_cold = cold && flip(0.00006);26var other_death = flip(0.000000001);27var death = death_by_cancer || death_by_cold || other_death;28 condition(death && cancer);29 return cold;30 }),31 deathAndNoCancer: Infer({method: 'enumerate'}, function() {32var cancer = flip(0.00001);33var cold = flip(0.2);34var death_by_cancer = cancer && flip(0.9);35var death_by_cold = cold && flip(0.00006);36var other_death = flip(0.000000001);37var death = death_by_cancer || death_by_cold || other_death;38 condition(death && !cancer);39 return cold;40 })41}));42
1var model = function() {2 var cancer = flip(0.00001);3 var cold = flip(0.2);45 var pDeath = (cancer ? 0.9 : 0) + (cold ? 0.00006 : 0) + 0.000000001;6 // Cap at 1 to be safe7 var pDeathCapped = Math.min(pDeath, 1);8 var death = flip(pDeathCapped);910 return {cancer: cancer, cold: cold, death: death};11};1213var prior = Infer({method: 'enumerate'}, function() {14 var r = model();15 return r.cold;16});1718var death = Infer({method: 'enumerate'}, function() {19 var r = model();20 condition(r.death);21 return r.cold;22});2324var deathAndCancer = Infer({method: 'enumerate'}, function() {25 var r = model();26 condition(r.death && r.cancer);27 return r.cold;28});2930var deathAndNoCancer = Infer({method: 'enumerate'}, function() {31 var r = model();32 condition(r.death && !r.cancer);33 return r.cold;34});3536var ANSWER = {37 prior: prior,38 death: death,39 deathAndCancer: deathAndCancer,40 deathAndNoCancer: deathAndNoCancer41};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◉TV<.05 | 3.9e-16 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◉TV<.05 | 8.0e-16 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◉TV<.05 | 3.9e-16 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 0.800 | 7.1953 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◉TV<.05 | 9.6e-6 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 4.2e-17 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◉TV<.05 | 4.2e-17 | 0.0000 | · |
Given this 'fair-vs-uniform' coin model:
1var weightPosterior = function(observedData){2 return Infer({method: 'MCMC', burn:1000, samples: 10000}, function() {3 var isFair = flip(0.9);4 var realWeight = isFair ? 0.5 : uniform({a:0, b:1});5 var coin = Bernoulli({p: realWeight});6 var obsFn = function(datum){ observe(coin, datum=='h') };7 mapData({data: observedData}, obsFn);8 return realWeight;9 })10}
This implies a two-faced coin and any other biased coin are equally likely. Adjust the model so that within the biased class (probability 0.1 in the prior), the coin is two-faced with probability 0.7 and otherwise uniform on (0, 1). End your program with var fullDataSet = repeat(50, function() { 'h' }); var observedDataSizes = [0,1,2,4,6,8,10,12,15,20,25,30,40,50]; map(function(N) { expectation(weightPosterior(fullDataSet.slice(0, N))) }, observedDataSizes) (an array of expected coin weights at each data size).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var weightPosterior = function(observedData) {2 return Infer({method: 'MCMC', burn:1000, samples: 10000}, function() {3 var isFair = flip(0.9);4 var isTwoFaced = flip(0.7);5 var realWeight = isFair ? 0.5 : (isTwoFaced ? 1 : uniform({a:0, b:1}));6 var coin = Bernoulli({p: realWeight});7 var obsFn = function(datum) { observe(coin, datum=='h') };8 mapData({data: observedData}, obsFn);9 return realWeight;10 })11};1213var fullDataSet = repeat(50, function() { 'h' });14var observedDataSizes = [0,1,2,4,6,8,10,12,15,20,25,30,40,50];15var ANSWER = (map(function(N) { expectation(weightPosterior(fullDataSet.slice(0, N))) }, observedDataSizes));16
1var weightPosterior = function(observedData){2 return Infer({method: 'MCMC', burn:1000, samples: 10000}, function() {3 var isFair = flip(0.9);4 var realWeight = isFair ? 0.5 : (flip(0.7) ? 1.0 : uniform({a:0, b:1}));5 var coin = Bernoulli({p: realWeight});6 var obsFn = function(datum){ observe(coin, datum=='h') };7 mapData({data: observedData}, obsFn);8 return realWeight;9 })10}1112var fullDataSet = repeat(50, function() { return 'h'; });13var observedDataSizes = [0,1,2,4,6,8,10,12,15,20,25,30,40,50];14var ANSWER = map(function(N) { expectation(weightPosterior(fullDataSet.slice(0, N))) }, observedDataSizes);
[0.5355, 0.5659, 0.6160, 0.7918, 0.9311, 0.9643, 0.9920, 0.9960, 0.9982, 0.9990, 0.9992, 0.9996, 0.9998, 0.9998]
[0.5356, 0.5717, 0.6224, 0.7914, 0.9264, 0.9726, 0.9933, 0.9964, 0.9982, 0.9988, 0.9993, 0.9996, 0.9998, 0.9998]
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ✓val+ | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ✗val- | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ✗val- | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ✓val+ | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ✓val+ | — | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ⚠fail | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ✗val- | — | — | · |
Given this Beta(10,10)-prior coin model and a data set alternating heads/tails 50 times each:
1var pseudoCounts = {a: 10, b: 10};2var weightPosterior = function(observedData){3 return Infer({method: 'MCMC', burn:1000, samples: 1000}, function() {4 var coinWeight = sample(Beta(pseudoCounts));5 var coinDist = Bernoulli({p: coinWeight});6 var obsFn = function(datum){ observe(coinDist, datum=='h') };7 mapData({data: observedData}, obsFn);8 return coinWeight;9 })10}11var fullDataSet = repeat(50, function() { ['h', 't'] }).flat();
Compute the prior distribution and the posterior after observing the full data set. Return an object literal with two distributions:
- prior: the Beta(10,10) prior
- post: the posterior after all observations
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var pseudoCounts = {a: 10, b: 10};23var weightPosterior = function(observedData){4 return Infer({method: 'MCMC', burn:1000, samples: 1000}, function() {5 var coinWeight = sample(Beta(pseudoCounts));6 var coinDist = Bernoulli({p: coinWeight});7 var obsFn = function(datum){ observe(coinDist, datum=='h') };8 mapData({data: observedData}, obsFn);9 return coinWeight;10 })11};1213var fullDataSet = repeat(50, function() { ['h', 't'] }).flat();14var ANSWER = (({15 prior: Beta(pseudoCounts),16 post: weightPosterior(fullDataSet)17}));18
1var pseudoCounts = {a: 10, b: 10};23var weightPosterior = function(observedData){4 return Infer({method: 'MCMC', burn:1000, samples: 5000}, function() {5 var coinWeight = sample(Beta(pseudoCounts));6 var coinDist = Bernoulli({p: coinWeight});7 var obsFn = function(datum){ observe(coinDist, datum=='h'); };8 mapData({data: observedData}, obsFn);9 return coinWeight;10 });11};1213var fullDataSet = _.flatten(repeat(50, function() { return ['h', 't']; }));1415var prior = Infer({method: 'forward', samples: 5000}, function() {16 return sample(Beta(pseudoCounts));17});1819var post = weightPosterior(fullDataSet);2021var ANSWER = {prior: prior, post: post};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◔TV<1 | 1.000 | 17.0246 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◔TV<1 | 1.000 | 15.4358 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 1.000 | 15.4358 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 0.800 | 13.6814 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◔TV<1 | 0.800 | 13.6814 | · |
Given this Beta(10,10)-prior coin model:
1var pseudoCounts = {a: 10, b: 10};2var weightPosterior = function(observedData){3 return Infer({method: 'MCMC', burn:1000, samples: 1000}, function() {4 var coinWeight = sample(Beta(pseudoCounts));5 var coinDist = Bernoulli({p: coinWeight});6 var obsFn = function(datum){ observe(coinDist, datum=='h') };7 mapData({data: observedData}, obsFn);8 return coinWeight;9 })10}
Compute the *variance* of the posterior at each of these data sizes: [0,2,4,8,16,32,64,128,256,512], using the data set repeat(256, function(){['h','t']}).flat() (alternating). Variance is expectation(posterior, function(x) { Math.pow(x - mean, 2) }) where mean = expectation(posterior). End with the array of variances.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var pseudoCounts = {a: 10, b: 10};23var weightPosterior = function(observedData){4 return Infer({method: 'MCMC', burn:1000, samples: 1000}, function() {5 var coinWeight = sample(Beta(pseudoCounts));6 var coinDist = Bernoulli({p: coinWeight});7 var obsFn = function(datum){ observe(coinDist, datum=='h') };8 mapData({data: observedData}, obsFn);9 return coinWeight;10 })11};1213var fullDataSet = repeat(256, function(){['h', 't']}).flat();14var observedDataSizes = [0,2,4,8,16,32,64,128,256,512];15var ANSWER = (map(function(N) {16 var posterior = weightPosterior(fullDataSet.slice(0,N));17 var mean = expectation(posterior);18 return expectation(posterior, function(x) { Math.pow(x - mean, 2) });19}, observedDataSizes));20
1var pseudoCounts = {a: 10, b: 10};23var weightPosterior = function(observedData){4 return Infer({method: 'MCMC', burn:1000, samples: 1000}, function() {5 var coinWeight = sample(Beta(pseudoCounts));6 var coinDist = Bernoulli({p: coinWeight});7 var obsFn = function(datum){ observe(coinDist, datum=='h') };8 mapData({data: observedData}, obsFn);9 return coinWeight;10 })11};1213// Build the alternating dataset of length 51214var fullData = _.flatten(repeat(256, function(){ return ['h','t']; }));1516var dataSizes = [0,2,4,8,16,32,64,128,256,512];1718var variances = map(function(n) {19 var data = fullData.slice(0, n);20 var posterior = weightPosterior(data);21 var mean = expectation(posterior, function(x){ return x; });22 var variance = expectation(posterior, function(x){ return Math.pow(x - mean, 2); });23 return variance;24}, dataSizes);2526var ANSWER = variances;
[0.0119, 0.0107, 0.0101, 0.0088, 0.0066, 0.0045, 0.0029, 0.0018, 0.0009, 0.0005]
[0.0119, 0.0107, 0.0101, 0.0088, 0.0066, 0.0045, 0.0029, 0.0018, 0.0009, 0.0005]
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ✓val+ | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ✓val+ | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ✓val+ | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ✓val+ | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ✓val+ | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ✓val+ | — | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ✓val+ | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ✓val+ | — | — | · |
You visit an alien planet and observe 10 aliens, each with three binary properties (antennae, green, blarghNoise). Build a mixture model with two latent kinds of aliens, each with its own per-property probability. Use Beta(.5, .5) priors independently on each of the three probabilities for each group, and a 50/50 prior over which group each alien belongs to. Memoize the per-group prototype with mem (so within one inference run, the group's prototype is consistent across aliens).
Data:
1var data = [2 {antennae: false, green: false, blarghNoise: false},3 {antennae: true, green: true, blarghNoise: true},4 {antennae: true, green: true, blarghNoise: true},5 {antennae: true, green: true, blarghNoise: true},6 {antennae: false, green: false, blarghNoise: false},7 {antennae: true, green: true, blarghNoise: true},8 {antennae: false, green: false, blarghNoise: false},9 {antennae: true, green: true, blarghNoise: true},10 {antennae: false, green: false, blarghNoise: false},11 {antennae: false, green: false, blarghNoise: false}12];
End your program with Infer({method: 'MCMC', kernel: {HMC: {steps: 10, stepSize: .01}}, samples: 3000}, ...) returning the joint posterior {group1, group2} where each is the per-property probability object {antennae, green, blarghNoise}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var properties = ['antennae', 'green', 'blarghNoise'];2var data = [3 {antennae : false, green: false, blarghNoise: false},4 {antennae : true, green: true, blarghNoise: true},5 {antennae : true, green: true, blarghNoise: true},6 {antennae : true, green: true, blarghNoise: true},7 {antennae : false, green: false, blarghNoise: false},8 {antennae : true, green: true, blarghNoise: true},9 {antennae : false, green: false, blarghNoise: false},10 {antennae : true, green: true, blarghNoise: true},11 {antennae : false, green: false, blarghNoise: false},12 {antennae : false, green: false, blarghNoise: false}13];1415var sampleGroupPrototype = mem(function(groupName) {16 var probs = repeat(3, function(){ beta(.5, .5)});17 return _.zipObject(properties, probs);18});19var ANSWER = (Infer({method: 'MCMC', kernel: {HMC: {steps: 10, stepSize: .01}}, samples: 3000},20 function(){21 mapData({data: data}, function(datum) {22 var group = flip() ? 'group1' : 'group2';23 var prototype = sampleGroupPrototype(group);24 mapData({data: properties}, function(property) {25 observe(Bernoulli({p: prototype[property]}), datum[property]);26 });27 });28 return {group1: sampleGroupPrototype('group1'),29 group2: sampleGroupPrototype('group2')};30}));31
1var data = [2 {antennae: false, green: false, blarghNoise: false},3 {antennae: true, green: true, blarghNoise: true},4 {antennae: true, green: true, blarghNoise: true},5 {antennae: true, green: true, blarghNoise: true},6 {antennae: false, green: false, blarghNoise: false},7 {antennae: true, green: true, blarghNoise: true},8 {antennae: false, green: false, blarghNoise: false},9 {antennae: true, green: true, blarghNoise: true},10 {antennae: false, green: false, blarghNoise: false},11 {antennae: false, green: false, blarghNoise: false}12];1314var model = function() {15 var prototype = mem(function(group) {16 return {17 antennae: beta(0.5, 0.5),18 green: beta(0.5, 0.5),19 blarghNoise: beta(0.5, 0.5)20 };21 });2223 mapData({data: data}, function(alien) {24 var group = flip(0.5) ? 1 : 2;25 var proto = prototype(group);26 observe(Bernoulli({p: proto.antennae}), alien.antennae);27 observe(Bernoulli({p: proto.green}), alien.green);28 observe(Bernoulli({p: proto.blarghNoise}), alien.blarghNoise);29 });3031 return {32 group1: prototype(1),33 group2: prototype(2)34 };35};3637var ANSWER = Infer({method: 'MCMC', kernel: {HMC: {steps: 10, stepSize: .01}}, samples: 3000}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 1.000 | 18.8515 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ⚠fail | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◔TV<1 | 1.000 | 19.1046 | · |
Alien-planet setup with 10 aliens (three binary properties: antennae, green, blarghNoise). Two latent kinds of aliens with Beta(.5, .5) priors per property, 50/50 prior on group membership, mem'd group prototypes, MCMC + HMC inference (steps:10, stepSize:.01, samples:3000).
Data:
1var data = [2 {antennae: false, green: false, blarghNoise: false},3 {antennae: true, green: true, blarghNoise: true},4 {antennae: true, green: true, blarghNoise: true},5 {antennae: true, green: true, blarghNoise: true},6 {antennae: false, green: false, blarghNoise: false},7 {antennae: true, green: true, blarghNoise: true},8 {antennae: false, green: false, blarghNoise: false},9 {antennae: true, green: true, blarghNoise: true},10 {antennae: false, green: false, blarghNoise: false},11 {antennae: false, green: false, blarghNoise: false}12];
Extend the model with a new observation: you hear a 'blargh' sound from a crater but cannot see the alien. Add a latent mysteryGroup (50/50 over the two groups), sample its prototype, and observe blarghNoise=true for that prototype. End your program with the Infer(...) returning {group1, group2, mysteryGroup}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var properties = ['antennae', 'green', 'blarghNoise'];2var data = [3 {antennae : false, green: false, blarghNoise: false},4 {antennae : true, green: true, blarghNoise: true},5 {antennae : true, green: true, blarghNoise: true},6 {antennae : true, green: true, blarghNoise: true},7 {antennae : false, green: false, blarghNoise: false},8 {antennae : true, green: true, blarghNoise: true},9 {antennae : false, green: false, blarghNoise: false},10 {antennae : true, green: true, blarghNoise: true},11 {antennae : false, green: false, blarghNoise: false},12 {antennae : false, green: false, blarghNoise: false}13];14var sampleGroupPrototype = mem(function(groupName) {15 var probs = repeat(3, function(){ beta(.5, .5)});16 return _.zipObject(properties, probs);17});18var ANSWER = (Infer({method: 'MCMC', kernel: {HMC: {steps: 10, stepSize: .01}}, samples: 3000},19 function(){20 mapData({data: data}, function(datum) {21 var group = flip() ? 'group1' : 'group2';22 var prototype = sampleGroupPrototype(group);23 mapData({data: properties}, function(property) {24 observe(Bernoulli({p: prototype[property]}), datum[property]);25 });26 });27 var mysteryGroup = flip() ? 'group1' : 'group2';28 var mysteryPrototype = sampleGroupPrototype(mysteryGroup);29 observe(Bernoulli({p: mysteryPrototype['blarghNoise']}), true);30 return {group1: sampleGroupPrototype('group1'),31 group2: sampleGroupPrototype('group2'),32 mysteryGroup: mysteryGroup};33}));34
1var data = [2 {antennae: false, green: false, blarghNoise: false},3 {antennae: true, green: true, blarghNoise: true},4 {antennae: true, green: true, blarghNoise: true},5 {antennae: true, green: true, blarghNoise: true},6 {antennae: false, green: false, blarghNoise: false},7 {antennae: true, green: true, blarghNoise: true},8 {antennae: false, green: false, blarghNoise: false},9 {antennae: true, green: true, blarghNoise: true},10 {antennae: false, green: false, blarghNoise: false},11 {antennae: false, green: false, blarghNoise: false}12];1314var model = function() {15 // Group prototypes: for each group, sample Beta(.5,.5) for each property16 var groupPrototype = mem(function(group) {17 return {18 antennae: beta(0.5, 0.5),19 green: beta(0.5, 0.5),20 blarghNoise: beta(0.5, 0.5)21 };22 });2324 // For each alien in data, assign to a group and observe properties25 mapData({data: data}, function(alien) {26 var group = flip(0.5) ? 1 : 2;27 var proto = groupPrototype(group);28 observe(Bernoulli({p: proto.antennae}), alien.antennae);29 observe(Bernoulli({p: proto.green}), alien.green);30 observe(Bernoulli({p: proto.blarghNoise}), alien.blarghNoise);31 });3233 // Mystery alien: heard blargh but can't see it34 var mysteryGroup = flip(0.5) ? 1 : 2;35 var mysteryProto = groupPrototype(mysteryGroup);36 observe(Bernoulli({p: mysteryProto.blarghNoise}), true);3738 var proto1 = groupPrototype(1);39 var proto2 = groupPrototype(2);4041 return {42 group1: {43 antennae: proto1.antennae,44 green: proto1.green,45 blarghNoise: proto1.blarghNoise46 },47 group2: {48 antennae: proto2.antennae,49 green: proto2.green,50 blarghNoise: proto2.blarghNoise51 },52 mysteryGroup: mysteryGroup53 };54};5556var ANSWER = Infer({57 method: 'MCMC',58 samples: 3000,59 kernel: {HMC: {steps: 10, stepSize: 0.01}}60}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◔TV<1 | 1.000 | 17.8546 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 1.000 | 17.8546 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 1.000 | 17.8546 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◔TV<1 | 1.000 | 17.8546 | · |
Suppose 22 participants take a memory test scored 0..45. Some are bona fide; others are malingerers (deliberately scoring low). Implement a mixture model inferring which group each participant belongs to.
Fill in the blanks and end with the Infer(...):
1var scores = [45, 45, 44, 45, 44, 45, 45, 45, 45, 45, 30, 20, 6, 44, 44, 27, 25, 17, 14, 27, 35, 30]2var subjIDs = _.range(scores.length)3var data = map(function(datum) {return _.zipObject(['subjID', 'score'], datum)}, _.zip(subjIDs, scores));45Infer({method: 'MCMC', samples: 10000}, function() {6 // Your code here: define group success probs, per-participant group memership7 var obsFn = function(datum){8 observe(// Your code here: Binomial({p, n: 45}))9 }10 mapData({data: data}, obsFn)11 // Your code here12 return // Your code here13});
Use Binomial({p, n: 45}) for each participant's score, with two latent group success probabilities (use uniform(0.5, 1) for the bona-fide group and uniform(0, group_1_p) for malingerers, ensuring p_malingerer < p_bona-fide). Return participant memberships plus group rates.
Use the strings 'group1' (bona fide) and 'group2' (malingerer) as group labels in your output. Use uniform(0.5, 1) for the bona-fide group success probability and uniform(0, group_1_p) for the malingerer probability.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var scores = [45, 45, 44, 45, 44, 45, 45, 45, 45, 45, 30, 20, 6, 44, 44, 27, 25, 17, 14, 27, 35, 30];2var subjIDs = _.range(scores.length);3var data = map(function(datum) {return _.zipObject(['subjID', 'score'], datum)}, _.zip(subjIDs, scores));4var ANSWER = (Infer({method: 'MCMC', samples: 10000}, function() {5 var group_1_p = uniform(0.5, 1);6 var group_2_p = uniform(0, group_1_p);7 var participant2Group = mem(function(participantID) {8 return flip() ? 'group1' : 'group2';9 });10 var group2Prob = mem(function(group) {11 return group == 'group1' ? group_1_p : group_2_p;12 });1314 var obsFn = function(datum){15 var p = group2Prob(participant2Group(datum.subjID));16 observe(Binomial({p: p, n: 45}), datum.score);17 };18 mapData({data: data}, obsFn);1920 var participantResults_ = map(function(datum) {return participant2Group(datum.subjID)}, data);21 var participantResults = _.zipObject(_.range(participantResults_.length), participantResults_);22 return _.merge(participantResults, {group_1_p: group_1_p, group_2_p: group_2_p});23}));24
1var scores = [45, 45, 44, 45, 44, 45, 45, 45, 45, 45, 30, 20, 6, 44, 44, 27, 25, 17, 14, 27, 35, 30];2var subjIDs = _.range(scores.length);3var data = map(function(datum) { return _.zipObject(['subjID', 'score'], datum); }, _.zip(subjIDs, scores));45var ANSWER = Infer({method: 'MCMC', samples: 10000}, function() {6 // Group success probabilities7 var group_1_p = uniform(0.5, 1); // bona fide8 var group_2_p = uniform(0, group_1_p); // malingerer910 // Per-participant group membership11 var groupMembership = map(function(id) {12 return flip(0.5) ? 'group1' : 'group2';13 }, subjIDs);1415 var obsFn = function(datum) {16 var id = datum.subjID;17 var score = datum.score;18 var group = groupMembership[id];19 var p = group === 'group1' ? group_1_p : group_2_p;20 observe(Binomial({p: p, n: 45}), score);21 };2223 mapData({data: data}, obsFn);2425 return {26 group_1_p: group_1_p,27 group_2_p: group_2_p,28 memberships: groupMembership29 };30});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◔TV<1 | 1.000 | 19.5553 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◔TV<1 | 1.000 | 19.5553 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◔TV<1 | 1.000 | 19.5553 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 1.000 | 18.2312 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◔TV<1 | 1.000 | 19.5771 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 1.000 | 19.5553 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 1.000 | 19.5771 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◔TV<1 | 1.000 | 19.5553 | · |
Number-game model. The hypothesis space contains rule-based concepts (multiples_of_N, powers_of_N for N=1..11; evens; odds) over the integers in [1, 20]. Implement similarity-based interval hypotheses (integers in [a, b]) so that:
- genSetFromInterval(a, b) returns all integers from a through b inclusive,
- makeIntervalHypothesisSpace(start, end) returns names 'interval_a_b' for every (a, b) with start <= a < b <= end,
- getSetFromHypothesis(rule) dispatches on the rule's prefix to produce the set of integers it denotes (handling multiples_, powers_, evens, odds, and interval_).
These helpers are given (include them or equivalent in your program):
1var maxNumber = 20;2var filterByInRange = function(set) {3 var inRange = function(v) { v <= maxNumber && v >= 0 };4 return _.uniq(filter(inRange, set));5};6var genEvens = function() {7 return filter(function(v) { return v % 2 == 0 }, _.range(1, maxNumber));8};9var genOdds = function() {10 return filter(function(v) { return (v + 1) % 2 == 0 }, _.range(1, maxNumber));11};12var genMultiples = function(base) {13 var multiples = map(function(v) { return base * v }, _.range(maxNumber));14 return filterByInRange(multiples);15};16var genPowers = function(base) {17 var powers = map(function(v) { return Math.pow(base, v) }, _.range(maxNumber));18 return filterByInRange(powers);19};20var inSet = function(val, set) { return _.includes(set, val); };21var makeRuleHypothesisSpace = function() {22 var multipleRules = map(function(b) { return 'multiples_of_' + b }, _.range(1, 12));23 var powerRules = map(function(b) { return 'powers_of_' + b }, _.range(1, 12));24 return multipleRules.concat(powerRules).concat(['evens', 'odds']);25};
Build learnConcept(examples, testQuery) that mixes rule and interval hypotheses 50/50 in the prior, draws a hypothesis, observes each example via Categorical({vs: set}), and returns {hypothesis, testQueryResponse: inSet(testQuery, set)}. End your program with learnConcept([3, 10], 12) returning the joint posterior.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var maxNumber = 20;2var filterByInRange = function(set) {3 var inRange = function(v) {v <= maxNumber && v >= 0};4 return _.uniq(filter(inRange, set));5};6var genEvens = function() {7 return filter(function(v) {return v % 2 == 0}, _.range(1, maxNumber));8};9var genOdds = function() {10 return filter(function(v) {return (v + 1) % 2 == 0}, _.range(1, maxNumber));11};12var genMultiples = function(base) {13 var multiples = map(function(v) {return base * v}, _.range(maxNumber));14 return filterByInRange(multiples);15};16var genPowers = function(base) {17 var powers = map(function(v) {return Math.pow(base, v)}, _.range(maxNumber));18 return filterByInRange(powers);19};20var inSet = function(val, set) { return _.includes(set, val); };21var makeRuleHypothesisSpace = function() {22 var multipleRules = map(function(base) {return 'multiples_of_' + base}, _.range(1, 12));23 var powerRules = map(function(base) {return 'powers_of_' + base}, _.range(1, 12));24 return multipleRules.concat(powerRules).concat(['evens', 'odds']);25};26var genSetFromInterval = function(a, b) { return _.range(a, b+1); };2728var makeIntervalHypothesisSpace = function(start, end) {29 var allIntervals = _.flatten(map(function(s) {30 return map(function(e) { [s, e] }, genSetFromInterval(s+1, end));31 }, genSetFromInterval(start, end)));32 return map(function(x) { 'interval_' + x[0] + '_' + x[1] }, allIntervals);33};3435var getSetFromHypothesis = function(rule) {36 var parts = rule.split('_');37 return (parts[0] == 'multiples' ? genMultiples(_.parseInt(parts[2])) :38 parts[0] == 'powers' ? genPowers(_.parseInt(parts[2])) :39 parts[0] == 'evens' ? genEvens() :40 parts[0] == 'odds' ? genOdds() :41 parts[0] == 'interval' ? genSetFromInterval(_.parseInt(parts[1]), _.parseInt(parts[2])) :42 console.error('unknown rule' + rule));43};4445var learnConcept = function(examples, testQuery) {46 return Infer({method: 'enumerate'}, function() {47 var rules = makeRuleHypothesisSpace();48 var intervals = makeIntervalHypothesisSpace(1, maxNumber);49 var hypothesis = flip(0.5) ? uniformDraw(rules) : uniformDraw(intervals);50 var set = getSetFromHypothesis(hypothesis);51 mapData({data: examples}, function(example) {52 observe(Categorical({vs: set}), example);53 });54 return {hypothesis: hypothesis,55 testQueryResponse: inSet(testQuery, set)};56 });57};58var ANSWER = (learnConcept([3, 10], 12));59
1var maxNumber = 20;23var filterByInRange = function(set) {4 var inRange = function(v) { return v <= maxNumber && v >= 1; };5 return _.uniq(filter(inRange, set));6};78var genEvens = function() {9 return filter(function(v) { return v % 2 == 0; }, _.range(1, maxNumber + 1));10};1112var genOdds = function() {13 return filter(function(v) { return (v + 1) % 2 == 0; }, _.range(1, maxNumber + 1));14};1516var genMultiples = function(base) {17 var multiples = map(function(v) { return base * v; }, _.range(1, maxNumber + 1));18 return filterByInRange(multiples);19};2021var genPowers = function(base) {22 var powers = map(function(v) { return Math.pow(base, v); }, _.range(0, maxNumber + 1));23 return filterByInRange(powers);24};2526var inSet = function(val, set) { return _.includes(set, val); };2728var makeRuleHypothesisSpace = function() {29 var multipleRules = map(function(b) { return 'multiples_of_' + b; }, _.range(1, 12));30 var powerRules = map(function(b) { return 'powers_of_' + b; }, _.range(1, 12));31 return multipleRules.concat(powerRules).concat(['evens', 'odds']);32};3334var genSetFromInterval = function(a, b) {35 return _.range(a, b + 1);36};3738var makeIntervalHypothesisSpace = function(start, end) {39 var pairs = [];40 var as = _.range(start, end);41 return _.flatten(map(function(a) {42 var bs = _.range(a + 1, end + 1);43 return map(function(b) {44 return 'interval_' + a + '_' + b;45 }, bs);46 }, as));47};4849var getSetFromHypothesis = function(rule) {50 var isMultiples = rule.indexOf('multiples_of_') === 0;51 var isPowers = rule.indexOf('powers_of_') === 0;52 var isInterval = rule.indexOf('interval_') === 0;53 return isMultiples ? (function() {54 var base = _.parseInt(rule.slice('multiples_of_'.length));55 return genMultiples(base);56 })() :57 isPowers ? (function() {58 var base = _.parseInt(rule.slice('powers_of_'.length));59 return genPowers(base);60 })() :61 isInterval ? (function() {62 var rest = rule.slice('interval_'.length);63 // rest is like "a_b"64 // find the underscore separating a and b65 // a and b could be multi-digit, split on first underscore66 var firstUnderscore = rest.indexOf('_');67 var a = _.parseInt(rest.slice(0, firstUnderscore));68 var b = _.parseInt(rest.slice(firstUnderscore + 1));69 return genSetFromInterval(a, b);70 })() :71 rule === 'evens' ? genEvens() :72 rule === 'odds' ? genOdds() :73 [];74};7576var ruleHypotheses = makeRuleHypothesisSpace();77var intervalHypotheses = makeIntervalHypothesisSpace(1, maxNumber);7879var numRules = ruleHypotheses.length;80var numIntervals = intervalHypotheses.length;8182var learnConcept = function(examples, testQuery) {83 return Infer({method: 'enumerate'}, function() {84 // 50/50 prior between rule-based and interval hypotheses85 var useRule = flip(0.5);86 var hypothesis = useRule87 ? ruleHypotheses[randomInteger(numRules)]88 : intervalHypotheses[randomInteger(numIntervals)];8990 var set = getSetFromHypothesis(hypothesis);9192 // Each example is observed as drawn uniformly from the set93 var setSize = set.length;94 condition(setSize > 0);9596 // Observe each example97 mapData({data: examples}, function(ex) {98 observe(Categorical({vs: set}), ex);99 });100101 var testQueryResponse = inSet(testQuery, set);102103 return {hypothesis: hypothesis, testQueryResponse: testQueryResponse};104 });105};106107var ANSWER = learnConcept([3, 10], 12);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ⚠fail | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Using the number-game model with rule + interval hypotheses (50/50 mix of rule-based and interval_a_b hypotheses) over integers [1, 20]: compute for each query in [1, 20] the expected probability that the query is in the inferred concept, given examples = [3, 6, 9].
Helpers (include them in your program):
1var maxNumber = 20;2var filterByInRange = function(set) {3 var inRange = function(v) { v <= maxNumber && v >= 0 };4 return _.uniq(filter(inRange, set));5};6var genEvens = function() {7 return filter(function(v) { return v % 2 == 0 }, _.range(1, maxNumber));8};9var genOdds = function() {10 return filter(function(v) { return (v + 1) % 2 == 0 }, _.range(1, maxNumber));11};12var genMultiples = function(base) {13 var multiples = map(function(v) { return base * v }, _.range(maxNumber));14 return filterByInRange(multiples);15};16var genPowers = function(base) {17 var powers = map(function(v) { return Math.pow(base, v) }, _.range(maxNumber));18 return filterByInRange(powers);19};20var inSet = function(val, set) { return _.includes(set, val); };21var makeRuleHypothesisSpace = function() {22 var multipleRules = map(function(b) { return 'multiples_of_' + b }, _.range(1, 12));23 var powerRules = map(function(b) { return 'powers_of_' + b }, _.range(1, 12));24 return multipleRules.concat(powerRules).concat(['evens', 'odds']);25};26var genSetFromInterval = function(a, b) { return _.range(a, b+1); };27var makeIntervalHypothesisSpace = function(start, end) {28 var allIntervals = _.flatten(map(function(s) {29 return map(function(e) { [s, e] }, genSetFromInterval(s+1, end));30 }, genSetFromInterval(start, end)));31 return map(function(x) { 'interval_' + x[0] + '_' + x[1] }, allIntervals);32};33var getSetFromHypothesis = function(rule) {34 var parts = rule.split('_');35 return parts[0] == 'multiples' ? genMultiples(_.parseInt(parts[2])) :36 parts[0] == 'powers' ? genPowers(_.parseInt(parts[2])) :37 parts[0] == 'evens' ? genEvens() :38 parts[0] == 'odds' ? genOdds() :39 parts[0] == 'interval' ? genSetFromInterval(_.parseInt(parts[1]), _.parseInt(parts[2])) :40 null;41};42var learnConcept = function(examples, testQuery) {43 return Infer({method: 'enumerate'}, function() {44 var rules = makeRuleHypothesisSpace();45 var intervals = makeIntervalHypothesisSpace(1, maxNumber);46 var hypothesis = flip(0.5) ? uniformDraw(rules) : uniformDraw(intervals);47 var set = getSetFromHypothesis(hypothesis);48 mapData({data: examples}, function(example) {49 observe(Categorical({vs: set}), example);50 });51 return {hypothesis: hypothesis, testQueryResponse: inSet(testQuery, set)};52 });53};
End your program with the array pQueries of length 20: for each query 1..20, compute expectation(marginalize(learnConcept([3, 6, 9], query), function(x) { x.testQueryResponse })).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var maxNumber = 20;2var filterByInRange = function(set) {3 var inRange = function(v) {v <= maxNumber && v >= 0};4 return _.uniq(filter(inRange, set));5};6var genEvens = function() {7 return filter(function(v) {return v % 2 == 0}, _.range(1, maxNumber));8};9var genOdds = function() {10 return filter(function(v) {return (v + 1) % 2 == 0}, _.range(1, maxNumber));11};12var genMultiples = function(base) {13 var multiples = map(function(v) {return base * v}, _.range(maxNumber));14 return filterByInRange(multiples);15};16var genPowers = function(base) {17 var powers = map(function(v) {return Math.pow(base, v)}, _.range(maxNumber));18 return filterByInRange(powers);19};20var inSet = function(val, set) { return _.includes(set, val); };21var makeRuleHypothesisSpace = function() {22 var multipleRules = map(function(base) {return 'multiples_of_' + base}, _.range(1, 12));23 var powerRules = map(function(base) {return 'powers_of_' + base}, _.range(1, 12));24 return multipleRules.concat(powerRules).concat(['evens', 'odds']);25};26var genSetFromInterval = function(a, b) { return _.range(a, b+1); };27var makeIntervalHypothesisSpace = function(start, end) {28 var allIntervals = _.flatten(map(function(s) {29 return map(function(e) { [s, e] }, genSetFromInterval(s+1, end));30 }, genSetFromInterval(start, end)));31 return map(function(x) { 'interval_' + x[0] + '_' + x[1] }, allIntervals);32};33var getSetFromHypothesis = function(rule) {34 var parts = rule.split('_');35 return (parts[0] == 'multiples' ? genMultiples(_.parseInt(parts[2])) :36 parts[0] == 'powers' ? genPowers(_.parseInt(parts[2])) :37 parts[0] == 'evens' ? genEvens() :38 parts[0] == 'odds' ? genOdds() :39 parts[0] == 'interval' ? genSetFromInterval(_.parseInt(parts[1]), _.parseInt(parts[2])) :40 console.error('unknown rule' + rule));41};42var learnConcept = function(examples, testQuery) {43 return Infer({method: 'enumerate'}, function() {44 var rules = makeRuleHypothesisSpace();45 var intervals = makeIntervalHypothesisSpace(1, maxNumber);46 var hypothesis = flip(0.5) ? uniformDraw(rules) : uniformDraw(intervals);47 var set = getSetFromHypothesis(hypothesis);48 mapData({data: examples}, function(example) {49 observe(Categorical({vs: set}), example);50 });51 return {hypothesis: hypothesis,52 testQueryResponse: inSet(testQuery, set)};53 });54};5556var examples = [3, 6, 9];57var queries = genSetFromInterval(1, maxNumber);58var ANSWER = (map(function(query) {59 var post = learnConcept(examples, query);60 return expectation(marginalize(post, function(x) { x.testQueryResponse }));61}, queries));62
1var maxNumber = 20;2var filterByInRange = function(set) {3 var inRange = function(v) { return v <= maxNumber && v >= 0; };4 return _.uniq(filter(inRange, set));5};6var genEvens = function() {7 return filter(function(v) { return v % 2 == 0; }, _.range(1, maxNumber + 1));8};9var genOdds = function() {10 return filter(function(v) { return (v + 1) % 2 == 0; }, _.range(1, maxNumber + 1));11};12var genMultiples = function(base) {13 var multiples = map(function(v) { return base * v; }, _.range(1, maxNumber + 1));14 return filterByInRange(multiples);15};16var genPowers = function(base) {17 var powers = map(function(v) { return Math.pow(base, v); }, _.range(1, maxNumber + 1));18 return filterByInRange(powers);19};20var inSet = function(val, set) { return _.includes(set, val); };21var makeRuleHypothesisSpace = function() {22 var multipleRules = map(function(b) { return 'multiples_of_' + b; }, _.range(1, 12));23 var powerRules = map(function(b) { return 'powers_of_' + b; }, _.range(1, 12));24 return multipleRules.concat(powerRules).concat(['evens', 'odds']);25};26var genSetFromInterval = function(a, b) { return _.range(a, b + 1); };27var makeIntervalHypothesisSpace = function(start, end) {28 var allIntervals = _.flatten(map(function(s) {29 return map(function(e) { return [s, e]; }, genSetFromInterval(s + 1, end));30 }, genSetFromInterval(start, end)));31 return map(function(x) { return 'interval_' + x[0] + '_' + x[1]; }, allIntervals);32};33var getSetFromHypothesis = function(rule) {34 var parts = rule.split('_');35 return parts[0] == 'multiples' ? genMultiples(_.parseInt(parts[2])) :36 parts[0] == 'powers' ? genPowers(_.parseInt(parts[2])) :37 parts[0] == 'evens' ? genEvens() :38 parts[0] == 'odds' ? genOdds() :39 parts[0] == 'interval' ? genSetFromInterval(_.parseInt(parts[1]), _.parseInt(parts[2])) :40 null;41};42var learnConcept = function(examples, testQuery) {43 return Infer({method: 'enumerate'}, function() {44 var rules = makeRuleHypothesisSpace();45 var intervals = makeIntervalHypothesisSpace(1, maxNumber);46 var hypothesis = flip(0.5) ? uniformDraw(rules) : uniformDraw(intervals);47 var set = getSetFromHypothesis(hypothesis);48 mapData({data: examples}, function(example) {49 observe(Categorical({vs: set, ps: map(function(x) { return 1/set.length; }, set)}), example);50 });51 return {hypothesis: hypothesis, testQueryResponse: inSet(testQuery, set)};52 });53};5455var queries = _.range(1, 21);56var pQueries = map(function(query) {57 var dist = learnConcept([3, 6, 9], query);58 var marginal = marginalize(dist, function(x) { return x.testQueryResponse; });59 return expectation(marginal);60}, queries);6162var ANSWER = pQueries;
[0.1406, 0.2989, 1.0000, 0.5153, 0.5153, 1.0000, 0.5153, 0.5153, 1.0000, 0.3843, 0.2934, 0.7125, 0.1789, 0.1414, 0.5968, 0.0886, 0.0696, 0.5387, 0.0410, 0.0093]
[0.1094, 0.2326, 1.0000, 0.4010, 0.4010, 1.0000, 0.4010, 0.4010, 1.0000, 0.2990, 0.2284, 0.7763, 0.1392, 0.1101, 0.6862, 0.0690, 0.0542, 0.6410, 0.0319, 0.0234]
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ✓val+ | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ✗val- | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ✓val+ | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ✓val+ | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ✓val+ | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ✗val- | — | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ✓val+ | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ✗val- | — | — | · |
Given this Causal Power model where C is a possible cause of E:
1var observedData = [{C:true, E:false}];2Infer({method: 'MCMC', samples: 10000, lag:2}, function() {3 var cp = uniform(0, 1); // Causal power of C to cause E4 var b = uniform(0, 1); // Background probability of E5 mapData({data: observedData}, function(datum) {6 var E = (datum.C && flip(cp)) || flip(b);7 condition(E == datum.E);8 })9 return {cp, b};10})
Modify it into a Causal Support model: also infer whether there is a causal relation at all (relation = flip()). When relation is false, C does not affect E. Return {relation, cp, b}. End with the Infer(...).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var observedData = [{C:true, E:false}];2var ANSWER = (Infer({method: 'MCMC', samples: 10000, lag:2}, function() {3 var relation = flip();4 var cp = uniform(0, 1);5 var b = uniform(0, 1);67 mapData({data: observedData}, function(datum) {8 var E = (relation && datum.C && flip(cp)) || flip(b);9 condition(E == datum.E);10 });1112 return {relation, cp, b};13}));14
1var observedData = [{C:true, E:false}];23var ANSWER = Infer({method: 'MCMC', samples: 10000, lag:2}, function() {4 var relation = flip();5 var cp = uniform(0, 1);6 var b = uniform(0, 1);7 mapData({data: observedData}, function(datum) {8 var E = (relation && datum.C && flip(cp)) || flip(b);9 condition(E == datum.E);10 });11 return {relation, cp, b};12});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◔TV<1 | 1.000 | 14.1448 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 0.691 | 9.3394 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Given the Causal Support model (relation, cp, b latent; observedData = [{C:true, E:false}]; noisy-OR effect E = (relation && C && flip(cp)) || flip(b)):
Single-site MH MCMC won't be efficient because changing relation alone rarely accepts. Improve efficiency by constructing the *marginal probability of E* directly via a small Infer, and use that in an observe (instead of sampling E and conditioning). Return {relation, cp, b}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var observedData = [{C:true, E:false}];2var ANSWER = (Infer({method: 'MCMC', samples: 10000, lag:2}, function() {3 var relation = flip();4 var cp = uniform(0, 1);5 var b = uniform(0, 1);67 var noisyOrMarginal = function(C) {8 return Infer({method: 'enumerate'}, function() {9 return (relation && C && flip(cp)) || flip(b);10 });11 };1213 mapData({data: observedData}, function(datum) {14 observe(noisyOrMarginal(datum.C), datum.E);15 });1617 return {relation, cp, b};18}));19
1var model = function() {2 var relation = flip(0.5);3 var cp = uniform(0, 1);4 var b = uniform(0, 1);56 // Compute marginal probability of E given C=true using a small inner Infer7 var eDistGivenCTrue = Infer({method: 'enumerate'}, function() {8 var E = (relation && flip(cp)) || flip(b);9 return E;10 });1112 // Observed data: {C: true, E: false}13 // P(E=false | C=true, relation, cp, b)14 observe(eDistGivenCTrue, false);1516 return {relation: relation, cp: cp, b: b};17};1819var ANSWER = Infer({20 method: 'MCMC',21 samples: 10000,22 burn: 2000,23 kernel: 'MH'24}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◔TV<1 | 0.827 | 11.4022 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◔TV<1 | 0.833 | 6.2019 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◔TV<1 | 0.968 | 13.6954 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 0.667 | 5.4863 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◔TV<1 | 0.677 | 8.3685 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 0.667 | 5.3785 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 0.779 | 10.1364 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◔TV<1 | 0.887 | 11.5619 | · |
Compare Causal Power (CP) and Causal Support (CS) models on data where E occurs varying amounts with and without C.
1var generateData = function(numEWithC, numEWithoutC) {2 var eWithC = repeat(numEWithC, function() {return {C: true, E: true}});3 var noEWithC = repeat(8 - numEWithC, function() {return {C: true, E: false}});4 var eWithoutC = repeat(numEWithoutC, function() {return {C: false, E: true}});5 var noEWithoutC = repeat(8 - numEWithoutC, function() {return {C: false, E: false}});6 return _.flatten([eWithC, noEWithC, eWithoutC, noEWithoutC]);7}8var dataParams = [[8,8],[6,6],[4,4],[2,2],[0,0],[8,6],[6,4],[4,2],[2,0],[8,4],[6,2],[4,0],[8,2],[6,0],[8,0]];
Implement cpPost(observedData) (returning posterior over cp) and csPost(observedData) (returning posterior over relation*cp), both using the marginal-noisy-OR observe from Ex 2.2. Return an object literal {cpValues, csValues} where each is an array of expectation(...) of the corresponding posterior across all 15 dataParams configurations.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var generateData = function(numEWithC, numEWithoutC) {2 var eWithC = repeat(numEWithC, function() {return {C: true, E: true}});3 var noEWithC = repeat(8 - numEWithC, function() {return {C: true, E: false}});4 var eWithoutC = repeat(numEWithoutC, function() {return {C: false, E: true}});5 var noEWithoutC = repeat(8 - numEWithoutC, function() {return {C: false, E: false}});6 return _.flatten([eWithC, noEWithC, eWithoutC, noEWithoutC]);7};89var dataParams = [[8, 8], [6, 6], [4, 4], [2, 2], [0, 0], [8, 6],10 [6, 4], [4, 2], [2, 0], [8, 4], [6, 2], [4, 0],11 [8, 2], [6, 0], [8, 0]];1213var data = map(function(x) { generateData(x[0], x[1]) }, dataParams);1415var cpPost = function(observedData) {16 return Infer({method: 'MCMC', burn: 2000, samples: 1000, lag:2}, function() {17 var cp = uniform(0, 1);18 var b = uniform(0, 1);19 var noisyOrMarginal = function(C) {20 return Infer({method: 'enumerate'}, function() {21 return (C && flip(cp)) || flip(b);22 });23 };24 mapData({data: observedData}, function(datum) {25 observe(noisyOrMarginal(datum.C), datum.E);26 });27 return cp;28 });29};3031var csPost = function(observedData) {32 return Infer({method: 'MCMC', burn: 2000, samples: 1000, lag:2}, function() {33 var relation = flip();34 var cp = uniform(0, 1);35 var b = uniform(0, 1);36 var noisyOrMarginal = function(C) {37 return Infer({method: 'enumerate'}, function() {38 return (relation && C && flip(cp)) || flip(b);39 });40 };41 mapData({data: observedData}, function(datum) {42 observe(noisyOrMarginal(datum.C), datum.E);43 });44 return relation * cp;45 });46};47var ANSWER = (({48 cpValues: map(function(d) { expectation(cpPost(d)) }, data),49 csValues: map(function(d) { expectation(csPost(d)) }, data)50}));51
1var cpPost = function(observedData) {2 return Infer({method: 'MCMC', samples: 5000, burn: 1000}, function() {3 var b = uniform(0, 1);4 var cp = uniform(0, 1);56 mapData({data: observedData}, function(d) {7 var pE = d.C ? (1 - (1 - b) * (1 - cp)) : b;8 observe(Bernoulli({p: pE}), d.E);9 });1011 return cp;12 });13};1415var csPost = function(observedData) {16 return Infer({method: 'MCMC', samples: 5000, burn: 1000}, function() {17 var b = uniform(0, 1);18 var relation = flip(0.5);19 var cp = relation ? uniform(0, 1) : 0;2021 mapData({data: observedData}, function(d) {22 var pE = d.C ? (1 - (1 - b) * (1 - cp)) : b;23 observe(Bernoulli({p: pE}), d.E);24 });2526 return relation * cp;27 });28};2930var generateData = function(numEWithC, numEWithoutC) {31 var eWithC = repeat(numEWithC, function() {return {C: true, E: true}});32 var noEWithC = repeat(8 - numEWithC, function() {return {C: true, E: false}});33 var eWithoutC = repeat(numEWithoutC, function() {return {C: false, E: true}});34 var noEWithoutC = repeat(8 - numEWithoutC, function() {return {C: false, E: false}});35 return _.flatten([eWithC, noEWithC, eWithoutC, noEWithoutC]);36};3738var dataParams = [[8,8],[6,6],[4,4],[2,2],[0,0],[8,6],[6,4],[4,2],[2,0],[8,4],[6,2],[4,0],[8,2],[6,0],[8,0]];3940var allData = map(function(params) {41 return generateData(params[0], params[1]);42}, dataParams);4344var cpValues = map(function(data) {45 var post = cpPost(data);46 return expectation(post);47}, allData);4849var csValues = map(function(data) {50 var post = csPost(data);51 return expectation(post);52}, allData);5354var ANSWER = {cpValues: cpValues, csValues: csValues};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ✗val- | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ✗val- | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ✗val- | — | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ✗val- | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ✗val- | — | — | · |
Sally chooses an action 'a', 'b', or 'c' that maps to a vending machine outcome (each action gives 80% chance of one specific food, 10% each for the others). If Sally is *deceptive* she chooses an action that does NOT lead to her goal food; otherwise she chooses one that does.
Fill in the condition(...) calls. End with the Infer(...) call returning the posterior over Sally's goal food given that she is deceptive AND chose 'b'.
1var actionPrior = Categorical({vs: ['a','b','c'], ps: [1/3,1/3,1/3]});2var foodPrior = Categorical({vs: ['bagel','cookie','doughnut'], ps: [1/3,1/3,1/3]});3var vendingMachine = /* maps action -> categorical food */;45var chooseAction = function(goal, transition, state, deceive) {6 return Infer({method: 'enumerate'}, function() {7 var action = sample(actionPrior);8 condition(...)9 return action;10 })11};1213Infer({method: 'enumerate'}, function() {14 var deceive = flip();15 var goalFood = sample(foodPrior);16 var goal = function(outcome) {return outcome == goalFood};17 var sallyActionDist = chooseAction(goal, vendingMachine, 'state', deceive);18 condition(...)19 return goalFood;20});
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var actionPrior = Categorical({vs: ['a', 'b', 'c'], ps: [1/3, 1/3, 1/3]});2var foodPrior = Categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [1/3, 1/3, 1/3]});34var vendingMachine = function(state, action) {5 return action == 'a' ? categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [.8, .1, .1]}) :6 action == 'b' ? categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [.1, .8, .1]}) :7 action == 'c' ? categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [.1, .1, .8]}) :8 'nothing';9};1011var chooseAction = function(goal, transition, state, deceive) {12 return Infer({method: 'enumerate'}, function() {13 var action = sample(actionPrior);14 var outcome = transition(state, action);15 condition(deceive ? !goal(outcome) : goal(outcome));16 return action;17 });18};19var ANSWER = (Infer({method: 'enumerate'}, function() {20 var deceive = flip();21 var goalFood = sample(foodPrior);22 var goal = function(outcome) {return outcome == goalFood};23 var sallyActionDist = chooseAction(goal, vendingMachine, 'state', deceive);24 condition(deceive);25 condition(sample(sallyActionDist) == 'b');26 return goalFood;27}));28
1var actionPrior = Categorical({vs: ['a','b','c'], ps: [1/3,1/3,1/3]});2var foodPrior = Categorical({vs: ['bagel','cookie','doughnut'], ps: [1/3,1/3,1/3]});34var vendingMachine = function(action) {5 return Categorical({6 vs: ['bagel','cookie','doughnut'],7 ps: action === 'a' ? [0.8, 0.1, 0.1] :8 action === 'b' ? [0.1, 0.8, 0.1] :9 [0.1, 0.1, 0.8]10 });11};1213var chooseAction = function(goal, transition, state, deceive) {14 return Infer({method: 'enumerate'}, function() {15 var action = sample(actionPrior);16 var outcome = sample(transition(action));17 condition(deceive ? !goal(outcome) : goal(outcome));18 return action;19 });20};2122var ANSWER = Infer({method: 'enumerate'}, function() {23 var deceive = flip();24 var goalFood = sample(foodPrior);25 var goal = function(outcome) { return outcome == goalFood; };26 var sallyActionDist = chooseAction(goal, vendingMachine, 'state', deceive);27 condition(deceive === true && sample(sallyActionDist) === 'b');28 return goalFood;29});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◐TV<.5 | 0.100 | 0.1054 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Sally chooses 'a', 'b', or 'c'. If deceptive she avoids her goal food; otherwise she picks an action whose outcome IS her goal food. The vending machine: action 'a' -> 80% bagel, 'b' -> 80% cookie, 'c' -> 80% doughnut (10% each for the others).
You observe Sally choose 'b' twice in a row. End with the Infer(...) returning the posterior over her goal food.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var actionPrior = Categorical({vs: ['a', 'b', 'c'], ps: [1/3, 1/3, 1/3]});2var foodPrior = Categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [1/3, 1/3, 1/3]});34var vendingMachine = function(state, action) {5 return action == 'a' ? categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [.8, .1, .1]}) :6 action == 'b' ? categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [.1, .8, .1]}) :7 action == 'c' ? categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [.1, .1, .8]}) :8 'nothing';9};1011var chooseAction = function(goal, transition, state, deceive) {12 return Infer({method: 'enumerate'}, function() {13 var action = sample(actionPrior);14 var outcome = transition(state, action);15 condition(deceive ? !goal(outcome) : goal(outcome));16 return action;17 });18};19var ANSWER = (Infer({method: 'enumerate'}, function() {20 var deceive = flip();21 var goalFood = sample(foodPrior);22 var goal = function(outcome) {return outcome == goalFood};23 var sallyActionDist = chooseAction(goal, vendingMachine, 'state', deceive);24 condition(sample(sallyActionDist) == 'b');25 condition(sample(sallyActionDist) == 'b');26 return goalFood;27}));28
1var model = function() {2 // Prior over goal food3 var goal = categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: [1/3, 1/3, 1/3]});45 // Prior over deceptive6 var deceptive = flip(0.5);78 // Vending machine outcomes given action9 var vendingMachine = function(action) {10 var ps = action === 'a' ? [0.8, 0.1, 0.1] :11 action === 'b' ? [0.1, 0.8, 0.1] :12 [0.1, 0.1, 0.8];13 return categorical({vs: ['bagel', 'cookie', 'doughnut'], ps: ps});14 };1516 // Sally picks action based on goal and deceptiveness17 // If not deceptive: picks action whose outcome IS her goal food (most likely)18 // If deceptive: avoids her goal food19 var chooseAction = function() {20 if (!deceptive) {21 // Pick the action most likely to give goal food22 // 'a' -> bagel, 'b' -> cookie, 'c' -> doughnut23 return goal === 'bagel' ? 'a' :24 goal === 'cookie' ? 'b' : 'c';25 } else {26 // Deceptive: avoid goal food, pick uniformly among non-goal actions27 var nonGoalActions = goal === 'bagel' ? ['b', 'c'] :28 goal === 'cookie' ? ['a', 'c'] :29 ['a', 'b'];30 return nonGoalActions[randomInteger(2)];31 }32 };3334 // Observe Sally choosing 'b' twice35 var action1 = chooseAction();36 condition(action1 === 'b');3738 var action2 = chooseAction();39 condition(action2 === 'b');4041 return goal;42};4344var ANSWER = Infer({method: 'enumerate'}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◐TV<.5 | 0.395 | 0.5031 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◐TV<.5 | 0.062 | 0.0082 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◐TV<.5 | 0.062 | 0.0082 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◐TV<.5 | 0.062 | 0.0082 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◐TV<.5 | 0.062 | 0.0082 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◐TV<.5 | 0.062 | 0.0082 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.062 | 0.0082 | · |
Monty Hall variant. Alice picks a door uniformly from {1, 2, 3} (one hides a prize, the others are empty). Monty opens a different door, and we condition on his door being neither Alice's nor the prize. In this variant, Monty picks a door uniformly at random from {1, 2, 3}, including possibly Alice's or the prize door (we filter those via the outer condition).
Helpers (include them in your program):
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL);4 }, l, badItems);5};6var doors = [1, 2, 3];
Build montyRandom(aliceDoor, prizeDoor) returning a Distribution over Monty's door under this variant. Build model(switches) that draws aliceDoor and prizeDoor uniformly, samples montyDoor from montyRandom, conditions on montyDoor != prizeDoor && montyDoor != aliceDoor, and returns aliceDoor == prizeDoor (whether Alice wins) under her strategy (switches true means switch to the remaining unopened door, false means stay). End your program with an object literal:
stay: P(win) when Alice doesn't switch
switch: P(win) when Alice switches
Each value should be Infer({method: 'enumerate'}, function() { return model(<flag>); }).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL)4 }, l, badItems);5};67var doors = [1, 2, 3];89var montyRandom = function(aliceDoor, prizeDoor) {10 return Infer({method: 'enumerate'}, function() {11 return categorical({vs: doors});12 });13};1415var model = function(switches) {16 var aliceDoor = categorical({vs: doors});17 var prizeDoor = categorical({vs: doors});18 var montyDoorDist = montyRandom(aliceDoor, prizeDoor);19 var montyDoor = sample(montyDoorDist);20 condition(montyDoor != prizeDoor);21 condition(montyDoor != aliceDoor);22 var aliceDoor = switches ? removeBadItems(doors, [aliceDoor, montyDoor])[0] : aliceDoor;23 return aliceDoor == prizeDoor;24};25var ANSWER = (({26 stay: Infer({method: 'enumerate'}, function() { return model(false); }),27 switch: Infer({method: 'enumerate'}, function() { return model(true); })28}));29
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL);4 }, l, badItems);5};6var doors = [1, 2, 3];78var montyRandom = function(aliceDoor, prizeDoor) {9 return Infer({method: 'enumerate'}, function() {10 var montyDoor = doors[randomInteger(3)];11 return montyDoor;12 });13};1415var model = function(switches) {16 var aliceDoor = doors[randomInteger(3)];17 var prizeDoor = doors[randomInteger(3)];18 var montyDoor = sample(montyRandom(aliceDoor, prizeDoor));19 condition(montyDoor != prizeDoor && montyDoor != aliceDoor);20 var finalDoor = switches21 ? removeBadItems(doors, [aliceDoor, montyDoor])[0]22 : aliceDoor;23 return finalDoor == prizeDoor;24};2526var ANSWER = {27 stay: Infer({method: 'enumerate'}, function() { return model(false); }),28 switch: Infer({method: 'enumerate'}, function() { return model(true); })29};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◔TV<1 | 0.500 | 0.6931 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◐TV<.5 | 0.167 | 0.0566 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◐TV<.5 | 0.167 | 0.0566 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◐TV<.5 | 0.167 | 0.0566 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.167 | 0.0566 | · |
Monty Hall variant. Alice picks a door uniformly from {1, 2, 3} (one hides a prize, the others are empty). Monty opens a different door, and we condition on his door being neither Alice's nor the prize. In this variant, Monty deliberately picks a door that is neither Alice's nor the prize (this is the original Monty Hall setup). Inside montyAvoidBoth, sample Monty's door uniformly and condition that it is not Alice's and not the prize.
Helpers (include them in your program):
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL);4 }, l, badItems);5};6var doors = [1, 2, 3];
Build montyAvoidBoth(aliceDoor, prizeDoor) returning a Distribution over Monty's door under this variant. Build model(switches) that draws aliceDoor and prizeDoor uniformly, samples montyDoor from montyAvoidBoth, conditions on montyDoor != prizeDoor && montyDoor != aliceDoor, and returns aliceDoor == prizeDoor (whether Alice wins) under her strategy (switches true means switch to the remaining unopened door, false means stay). End your program with an object literal:
stay: P(win) when Alice doesn't switch
switch: P(win) when Alice switches
Each value should be Infer({method: 'enumerate'}, function() { return model(<flag>); }).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL)4 }, l, badItems);5};67var doors = [1, 2, 3];89var montyAvoidBoth = function(aliceDoor, prizeDoor) {10 return Infer({method: 'enumerate'}, function() {11 var montyDoor = categorical({vs: doors});12 condition(montyDoor != aliceDoor);13 condition(montyDoor != prizeDoor);14 return montyDoor;15 });16};1718var model = function(switches) {19 var aliceDoor = categorical({vs: doors});20 var prizeDoor = categorical({vs: doors});21 var montyDoorDist = montyAvoidBoth(aliceDoor, prizeDoor);22 var montyDoor = sample(montyDoorDist);23 condition(montyDoor != prizeDoor);24 condition(montyDoor != aliceDoor);25 var aliceDoor = switches ? removeBadItems(doors, [aliceDoor, montyDoor])[0] : aliceDoor;26 return aliceDoor == prizeDoor;27};28var ANSWER = (({29 stay: Infer({method: 'enumerate'}, function() { return model(false); }),30 switch: Infer({method: 'enumerate'}, function() { return model(true); })31}));32
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL);4 }, l, badItems);5};6var doors = [1, 2, 3];78var montyAvoidBoth = function(aliceDoor, prizeDoor) {9 return Infer({method: 'enumerate'}, function() {10 var montyDoor = uniformDraw(doors);11 condition(montyDoor != aliceDoor && montyDoor != prizeDoor);12 return montyDoor;13 });14};1516var model = function(switches) {17 var aliceDoor = uniformDraw(doors);18 var prizeDoor = uniformDraw(doors);19 var montyDoor = sample(montyAvoidBoth(aliceDoor, prizeDoor));20 condition(montyDoor != prizeDoor && montyDoor != aliceDoor);21 var finalDoor = switches22 ? uniformDraw(removeBadItems(doors, [aliceDoor, montyDoor]))23 : aliceDoor;24 return finalDoor == prizeDoor;25};2627var ANSWER = {28 stay: Infer({method: 'enumerate'}, function() { return model(false); }),29 switch: Infer({method: 'enumerate'}, function() { return model(true); })30};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Monty Hall variant. Alice picks a door uniformly from {1, 2, 3} (one hides a prize, the others are empty). Monty opens a different door, and we condition on his door being neither Alice's nor the prize. In this variant, Monty picks a door uniformly at random but only avoids Alice's door (he may inadvertently reveal the prize). Inside montyAvoidAlice, sample Monty's door uniformly and condition only that it is not Alice's.
Helpers (include them in your program):
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL);4 }, l, badItems);5};6var doors = [1, 2, 3];
Build montyAvoidAlice(aliceDoor, prizeDoor) returning a Distribution over Monty's door under this variant. Build model(switches) that draws aliceDoor and prizeDoor uniformly, samples montyDoor from montyAvoidAlice, conditions on montyDoor != prizeDoor && montyDoor != aliceDoor, and returns aliceDoor == prizeDoor (whether Alice wins) under her strategy (switches true means switch to the remaining unopened door, false means stay). End your program with an object literal:
stay: P(win) when Alice doesn't switch
switch: P(win) when Alice switches
Each value should be Infer({method: 'enumerate'}, function() { return model(<flag>); }).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL)4 }, l, badItems);5};67var doors = [1, 2, 3];89var montyAvoidAlice = function(aliceDoor, prizeDoor) {10 return Infer({method: 'enumerate'}, function() {11 var montyDoor = categorical({vs: doors});12 condition(montyDoor != aliceDoor);13 return montyDoor;14 });15};1617var model = function(switches) {18 var aliceDoor = categorical({vs: doors});19 var prizeDoor = categorical({vs: doors});20 var montyDoorDist = montyAvoidAlice(aliceDoor, prizeDoor);21 var montyDoor = sample(montyDoorDist);22 condition(montyDoor != prizeDoor);23 condition(montyDoor != aliceDoor);24 var aliceDoor = switches ? removeBadItems(doors, [aliceDoor, montyDoor])[0] : aliceDoor;25 return aliceDoor == prizeDoor;26};27var ANSWER = (({28 stay: Infer({method: 'enumerate'}, function() { return model(false); }),29 switch: Infer({method: 'enumerate'}, function() { return model(true); })30}));31
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL);4 }, l, badItems);5};6var doors = [1, 2, 3];78var montyAvoidAlice = function(aliceDoor, prizeDoor) {9 return Infer({method: 'enumerate'}, function() {10 var montyDoor = uniformDraw(removeBadItems(doors, [aliceDoor]));11 return montyDoor;12 });13};1415var model = function(switches) {16 var aliceDoor = uniformDraw(doors);17 var prizeDoor = uniformDraw(doors);18 var montyDoor = sample(montyAvoidAlice(aliceDoor, prizeDoor));19 condition(montyDoor != prizeDoor && montyDoor != aliceDoor);20 var finalDoor = switches21 ? uniformDraw(removeBadItems(doors, [aliceDoor, montyDoor]))22 : aliceDoor;23 return finalDoor == prizeDoor;24};2526var ANSWER = {27 stay: Infer({method: 'enumerate'}, function() { return model(false); }),28 switch: Infer({method: 'enumerate'}, function() { return model(true); })29};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Monty Hall variant. Alice picks a door uniformly from {1, 2, 3} (one hides a prize, the others are empty). Monty opens a different door, and we condition on his door being neither Alice's nor the prize. In this variant, Monty picks a door uniformly at random but only avoids the prize door (he may inadvertently pick Alice's). Inside montyAvoidPrize, sample Monty's door uniformly and condition only that it is not the prize.
Helpers (include them in your program):
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL);4 }, l, badItems);5};6var doors = [1, 2, 3];
Build montyAvoidPrize(aliceDoor, prizeDoor) returning a Distribution over Monty's door under this variant. Build model(switches) that draws aliceDoor and prizeDoor uniformly, samples montyDoor from montyAvoidPrize, conditions on montyDoor != prizeDoor && montyDoor != aliceDoor, and returns aliceDoor == prizeDoor (whether Alice wins) under her strategy (switches true means switch to the remaining unopened door, false means stay). End your program with an object literal:
stay: P(win) when Alice doesn't switch
switch: P(win) when Alice switches
Each value should be Infer({method: 'enumerate'}, function() { return model(<flag>); }).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL)4 }, l, badItems);5};67var doors = [1, 2, 3];89var montyAvoidPrize = function(aliceDoor, prizeDoor) {10 return Infer({method: 'enumerate'}, function() {11 var montyDoor = categorical({vs: doors});12 condition(montyDoor != prizeDoor);13 return montyDoor;14 });15};1617var model = function(switches) {18 var aliceDoor = categorical({vs: doors});19 var prizeDoor = categorical({vs: doors});20 var montyDoorDist = montyAvoidPrize(aliceDoor, prizeDoor);21 var montyDoor = sample(montyDoorDist);22 condition(montyDoor != prizeDoor);23 condition(montyDoor != aliceDoor);24 var aliceDoor = switches ? removeBadItems(doors, [aliceDoor, montyDoor])[0] : aliceDoor;25 return aliceDoor == prizeDoor;26};27var ANSWER = (({28 stay: Infer({method: 'enumerate'}, function() { return model(false); }),29 switch: Infer({method: 'enumerate'}, function() { return model(true); })30}));31
1var removeBadItems = function(l, badItems) {2 return reduce(function(badItem, remainingL) {3 return remove(badItem, remainingL);4 }, l, badItems);5};6var doors = [1, 2, 3];78var montyAvoidPrize = function(aliceDoor, prizeDoor) {9 return Infer({method: 'enumerate'}, function() {10 var montyDoor = uniformDraw(doors);11 condition(montyDoor != prizeDoor);12 return montyDoor;13 });14};1516var model = function(switches) {17 var aliceDoor = uniformDraw(doors);18 var prizeDoor = uniformDraw(doors);19 var montyDoor = sample(montyAvoidPrize(aliceDoor, prizeDoor));20 condition(montyDoor != aliceDoor);21 var finalDoor = switches22 ? uniformDraw(removeBadItems(doors, [aliceDoor, montyDoor]))23 : aliceDoor;24 return finalDoor == prizeDoor;25};2627var ANSWER = {28 stay: Infer({method: 'enumerate'}, function() { return model(false); }),29 switch: Infer({method: 'enumerate'}, function() { return model(true); })30};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 0.500 | 0.6931 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Show that setting Dirichlet alpha = [2, 3, 1, 1, 1] is equivalent to setting alpha = [1, 1, 1, 1, 1] then observing the first category once and the second twice. Build the second model (with alpha = [2, 3, 1, 1, 1]) given the first.
1var colors = ['black', 'blue', 'green', 'orange', 'red'];2var observedData = [{bag: 'bag1', draw: 'blue'},3 {bag: 'bag1', draw: 'blue'},4 {bag: 'bag1', draw: 'black'}];56var observed = Infer({method: 'MCMC', samples: 20000}, function() {7 var makeBag = mem(function(bag) {8 var colorProbs = dirichlet(ones([colors.length, 1]));9 return Categorical({vs: colors, ps: colorProbs});10 })11 var obsFn = function(datum) { observe(makeBag(datum.bag), datum.draw); }12 mapData({data: observedData}, obsFn);13 return {bag1: sample(makeBag('bag1'))};14})1516var usealpha = Infer({method: 'MCMC', samples: 20000}, function () {17 // ...fill in: alpha = [2, 3, 1, 1, 1] without observation18})
Return an object literal {observed, usealpha} with both posteriors.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var colors = ['black', 'blue', 'green', 'orange', 'red'];2var observedData = [{bag: 'bag1', draw: 'blue'},3 {bag: 'bag1', draw: 'blue'},4 {bag: 'bag1', draw: 'black'}];56var observed = Infer({method: 'MCMC', samples: 20000}, function() {7 var makeBag = mem(function(bag) {8 var colorProbs = dirichlet(ones([colors.length, 1]));9 return Categorical({vs: colors, ps: colorProbs});10 });11 var obsFn = function(datum) { observe(makeBag(datum.bag), datum.draw); };12 mapData({data: observedData}, obsFn);13 return {bag1: sample(makeBag('bag1'))};14});1516var usealpha = Infer({method: 'MCMC', samples: 20000}, function () {17 var makeBag = mem(function(bag) {18 var colorProbs = dirichlet(Vector([2, 3, 1, 1, 1]));19 return Categorical({vs: colors, ps: colorProbs});20 });21 return {bag1: sample(makeBag('bag1'))};22});23var ANSWER = (({observed: observed, usealpha: usealpha}));24
1var colors = ['black', 'blue', 'green', 'orange', 'red'];2var observedData = [{bag: 'bag1', draw: 'blue'},3 {bag: 'bag1', draw: 'blue'},4 {bag: 'bag1', draw: 'black'}];56var observed = Infer({method: 'MCMC', samples: 20000}, function() {7 var makeBag = mem(function(bag) {8 var colorProbs = dirichlet(ones([colors.length, 1]));9 return Categorical({vs: colors, ps: colorProbs});10 });11 var obsFn = function(datum) { observe(makeBag(datum.bag), datum.draw); };12 mapData({data: observedData}, obsFn);13 return {bag1: sample(makeBag('bag1'))};14});1516var usealpha = Infer({method: 'MCMC', samples: 20000}, function() {17 var makeBag = mem(function(bag) {18 // alpha = [2, 3, 1, 1, 1] corresponds to black=2, blue=3, green=1, orange=1, red=119 // which is equivalent to uniform prior + 1 black observation + 2 blue observations20 var colorProbs = dirichlet(Vector([2, 3, 1, 1, 1]));21 return Categorical({vs: colors, ps: colorProbs});22 });23 return {bag1: sample(makeBag('bag1'))};24});2526var ANSWER = {observed: observed, usealpha: usealpha};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Apples in a barrel: each apple is rotten with probability p, where p ~ Beta(.1, .2) (mass at endpoints). Implement makeBarrel(barrelName) returning a function barrel(n) that returns an array of n booleans (whether each apple is rotten). End with the Infer(...) over Math.sum(barrel(10)).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var makeBarrel = mem(function(barrelName) {2 var pRotten = beta({a: .1, b: .2});3 var barrel = function(n) {4 return repeat(n, function() { flip(pRotten) });5 };6 return barrel;7});8var ANSWER = (Infer({method: 'forward'}, function() {9 var barrel = makeBarrel('barrel');10 return Math.sum(barrel(10));11}));12
1var makeBarrel = function(barrelName) {2 var p = mem(function(name) { return beta(0.1, 0.2); });3 var barrel = function(n) {4 return repeat(n, function() { return flip(p(barrelName)); });5 };6 return barrel;7};89var model = function() {10 var barrel = makeBarrel("myBarrel");11 return Math.sum(barrel(10));12};1314var ANSWER = Infer({method: 'forward', samples: 10000}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◔TV<1 | 0.528 | 0.7331 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◔TV<1 | 0.530 | 0.7380 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◐TV<.5 | 0.126 | 0.0614 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◐TV<.5 | 0.144 | 0.1057 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◐TV<.5 | 0.102 | 0.0448 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◐TV<.5 | 0.110 | 0.0475 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.106 | 0.0455 | · |
Apples in a barrel: each apple is rotten with probability pRotten, and pRotten is itself drawn from a Beta distribution. Different *stores* tend to be either mostly-rotten or mostly-fresh, captured by drawing the Beta hyperparameters from a discrete prior.
Implement makeStore(storeName) that returns a makeBarrel(barrelName) function. Each store's Beta hyperparameters come from a 50/50 mix: {a: .1, b: .3} (mostly fresh) vs {a: .3, b: .1} (mostly rotten). Within a store, all makeBarrel(...) calls share that store's Beta. Memoize at both store and barrel level. The barrel function takes n and returns an array of n booleans (rotten or not).
End your program with an object literal {sameStore, differentStore}, each a forward-sample distribution over the absolute difference in rotten counts between two barrels of size 10:
- sameStore: both barrels from the same store
- differentStore: barrels from two different stores
Use Infer({method: 'forward', samples: 10000}, ...) for each.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var makeStore = mem(function(storeName) {2 var storePrior = flip() ? {a: .1, b: .3} : {a: .3, b: .1};3 var makeBarrel = mem(function(barrelName) {4 var pRotten = beta(storePrior);5 var barrel = function(n) {6 return repeat(n, function() { flip(pRotten) });7 };8 return barrel;9 });10 return makeBarrel;11});12var ANSWER = (({13 sameStore: Infer({method: 'forward', samples: 10000}, function() {14 var S = makeStore('S');15 var B1 = S('B1');16 var B2 = S('B2');17 return Math.abs(Math.sum(B1(10)) - Math.sum(B2(10)));18 }),19 differentStore: Infer({method: 'forward', samples: 10000}, function() {20 var S1 = makeStore('S1');21 var S2 = makeStore('S2');22 var B1 = S1('B1');23 var B2 = S2('B2');24 return Math.abs(Math.sum(B1(10)) - Math.sum(B2(10)));25 })26}));27
1var makeStore = mem(function(storeName) {2 var hyperparams = flip(0.5) ? {a: 0.1, b: 0.3} : {a: 0.3, b: 0.1};34 var makeBarrel = mem(function(barrelName) {5 var pRotten = beta(hyperparams.a, hyperparams.b);67 var barrel = function(n) {8 return repeat(n, function() { return flip(pRotten); });9 };1011 return barrel;12 });1314 return makeBarrel;15});1617var countRotten = function(arr) {18 return reduce(function(x, acc) { return acc + (x ? 1 : 0); }, 0, arr);19};2021var sameStore = Infer({method: 'forward', samples: 10000}, function() {22 var makeBarrel = makeStore('store1');23 var barrel1 = makeBarrel('barrel1');24 var barrel2 = makeBarrel('barrel2');25 var count1 = countRotten(barrel1(10));26 var count2 = countRotten(barrel2(10));27 return Math.abs(count1 - count2);28});2930var differentStore = Infer({method: 'forward', samples: 10000}, function() {31 var makeBarrel1 = makeStore('storeA');32 var makeBarrel2 = makeStore('storeB');33 var barrel1 = makeBarrel1('barrel1');34 var barrel2 = makeBarrel2('barrel2');35 var count1 = countRotten(barrel1(10));36 var count2 = countRotten(barrel2(10));37 return Math.abs(count1 - count2);38});3940var ANSWER = {sameStore: sameStore, differentStore: differentStore};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◐TV<.5 | 0.418 | 0.5237 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◐TV<.5 | 0.418 | 0.5237 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 0.736 | 1.3314 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Three-level rotten-apple hierarchy: cities → stores → barrels. Each city has a cityPrior = beta({a: .25, b: .25}) controlling the probability that a store in that city is the 'fresh' kind (Beta(.1, .3) for pRotten) vs the 'rotten' kind (Beta(.3, .1)). Within a store, each barrel draws pRotten = beta(storePrior), and barrel(n) returns an array of n booleans (rotten or not).
Implement makeCity(cityName) returning a makeStore function returning a makeBarrel function as described. Memoize at all three levels.
For city C1, store S1, barrel B1, end your program with the Infer over Math.sum(B1(20)) using forward sampling.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var makeCity = mem(function(cityName){2 var cityPrior = beta({a: .25, b: .25});3 var makeStore = mem(function(storeName) {4 var storePrior = flip(cityPrior) ? {a: .1, b: .3} : {a: .3, b: .1};5 var makeBarrel = mem(function(barrelName) {6 var pRotten = beta(storePrior);7 var barrel = function(n) {8 return repeat(n, function() { flip(pRotten) });9 };10 return barrel;11 });12 return makeBarrel;13 });14 return makeStore;15});1617var C1 = makeCity("C1");18var S1 = C1("S1");19var B1 = S1("B1");20var ANSWER = (Infer({method: 'forward'}, function(){21 return Math.sum(B1(20));22}));23
1var makeCity = mem(function(cityName) {2 var cityPrior = beta({a: 0.25, b: 0.25});3 var makeStore = mem(function(storeName) {4 var storePrior = flip(cityPrior)5 ? {a: 0.1, b: 0.3}6 : {a: 0.3, b: 0.1};7 var makeBarrel = mem(function(barrelName) {8 var pRotten = beta(storePrior);9 var barrel = function(n) {10 return repeat(n, function() { return flip(pRotten); });11 };12 return barrel;13 });14 return makeBarrel;15 });16 return makeStore;17});1819var ANSWER = Infer({method: 'forward', samples: 10000}, function() {20 var makeStore = makeCity('C1');21 var makeBarrel = makeStore('S1');22 var B1 = makeBarrel('B1');23 return Math.sum(B1(20));24});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◔TV<1 | 0.893 | 18.3466 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◉TV<.05 | 0.006 | 0.0021 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◐TV<.5 | 0.437 | 7.3818 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 0.006 | 0.0021 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◉TV<.05 | 0.006 | 0.0021 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 0.669 | 12.6580 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 0.669 | 12.6580 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◉TV<.05 | 0.006 | 0.0021 | · |
Three-level rotten-apple hierarchy (cities → stores → barrels), as in the previous exercise: city has cityPrior = beta({a: .25, b: .25}); stores within a city pick Beta(.1, .3) (fresh) with prob cityPrior, else Beta(.3, .1) (rotten); barrels in a store sample pRotten = beta(storePrior), and barrel(n) returns n booleans. Memoized at all levels.
You visit a store in a city and observe a barrel of 10 apples, 7 of which are rotten. You then visit a *different* store in the *same* city. End your program with Infer({method: 'MCMC', samples: 5000, lag: 100}, ...) returning the posterior over the number of rotten apples in a 10-apple barrel from this second store.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var makeCity = mem(function(cityName){2 var cityPrior = beta({a: .25, b: .25});34 var makeStore = mem(function(storeName) {5 var storePrior = flip(cityPrior) ? {a: .1, b: .3} : {a: .3, b: .1};67 var makeBarrel = mem(function(barrelName) {8 var pRotten = beta(storePrior);9 var barrel = function(n) {10 return repeat(n, function() { flip(pRotten) });11 };12 return barrel;13 });1415 return makeBarrel;16 });1718 return makeStore;19});20var ANSWER = (Infer({method: 'MCMC', samples:5000, lag: 100}, function(){21 var C = makeCity("C");22 var S1 = C("S1");23 var B1 = S1("B1");24 var S2 = C("S2");25 var B2 = S2("B2");2627 condition(Math.sum(B1(10)) == 7);2829 return Math.sum(B2(10));30}));31
1var model = function() {2 // City-level prior3 var cityPrior = beta({a: 0.25, b: 0.25});45 // Store-level: given cityPrior, pick Beta params6 var makeStore = function() {7 var isFresh = flip(cityPrior);8 var storePrior = isFresh ? beta({a: 0.1, b: 0.3}) : beta({a: 0.3, b: 0.1});9 return storePrior;10 };1112 // Barrel: given store's rotten rate, sample n booleans13 var makeBarrel = function(storePrior, n) {14 var pRotten = beta({a: storePrior * 10, b: (1 - storePrior) * 10});15 return repeat(n, function() { return flip(pRotten); });16 };1718 // Store 1: observe 7 rotten out of 1019 var store1Prior = makeStore();20 var pRotten1 = beta({a: store1Prior * 10, b: (1 - store1Prior) * 10});21 // Observe 10 apples, 7 rotten22 observe(Binomial({p: pRotten1, n: 10}), 7);2324 // Store 2: different store, same city (same cityPrior)25 var store2Prior = makeStore();26 var pRotten2 = beta({a: store2Prior * 10, b: (1 - store2Prior) * 10});2728 // Return number of rotten apples in a 10-apple barrel from store 229 return sample(Binomial({p: pRotten2, n: 10}));30};3132var ANSWER = Infer({method: 'MCMC', samples: 5000, lag: 100}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◉TV<.05 | 0.021 | 0.0019 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◐TV<.5 | 0.144 | 0.0485 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◐TV<.5 | 0.170 | 0.0665 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 0.044 | 0.0043 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.134 | 0.0396 | · |
Hierarchical Bayesian data analysis on word reading times. Each data point has {group: "vowel"|"consonant", word: str, id: int, rt: number}.
Data:
1var data = [{group: "vowel", word: "abacus", id: 1, rt: 210},2 {group: "vowel", word: "abacus", id: 2, rt: 212},3 {group: "vowel", word: "abacus", id: 3, rt: 209},4 {group: "vowel", word: "aardvark", id: 1, rt: 200},5 {group: "vowel", word: "aardvark", id: 2, rt: 201},6 {group: "vowel", word: "aardvark", id: 3, rt: 198},7 {group: "vowel", word: "ellipse", id: 1, rt: 220},8 {group: "vowel", word: "ellipse", id: 2, rt: 222},9 {group: "vowel", word: "ellipse", id: 3, rt: 219},10 {group: "consonant", word: "proton", id: 1, rt: 190},11 {group: "consonant", word: "proton", id: 2, rt: 191},12 {group: "consonant", word: "proton", id: 3, rt: 189},13 {group: "consonant", word: "folder", id: 1, rt: 180},14 {group: "consonant", word: "folder", id: 2, rt: 182},15 {group: "consonant", word: "folder", id: 3, rt: 178},16 {group: "consonant", word: "fedora", id: 1, rt: 230},17 {group: "consonant", word: "fedora", id: 2, rt: 231},18 {group: "consonant", word: "fedora", id: 3, rt: 228},19 {group: "consonant", word: "fedora", id: 1, rt: 231},20 {group: "consonant", word: "fedora", id: 2, rt: 233},21 {group: "consonant", word: "fedora", id: 3, rt: 230},22 {group: "consonant", word: "fedora", id: 1, rt: 230},23 {group: "consonant", word: "fedora", id: 2, rt: 232},24 {group: "consonant", word: "fedora", id: 3, rt: 228}];
Simple BDA model (without per-word random effects):
1var opts = {method: "MCMC", burn: 10000, lag: 5, samples: 5000};2Infer(opts, function() {3 var groupMeans = {vowel: gaussian(200, 100),4 consonant: gaussian(200, 100)};5 var obsFn = function(d) {6 observe(Gaussian({mu: groupMeans[d.group], sigma: 10}), d.rt);7 };8 mapData({data: data}, obsFn);9 return groupMeans['vowel'] - groupMeans['consonant'];10});
Adjust the model so each *word* has its own mean reading time drawn from its group's mean: use mem(function(word, group) { gaussian(groupMeans[group], 20) }) for the per-word effect, and observe each rt as Gaussian({mu: wordMean(d.word, d.group), sigma: 10}). End your program with the Infer(...) returning the posterior over groupMeans['vowel'] - groupMeans['consonant'].
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var data = [{group: "vowel", word: "abacus", id: 1, rt: 210},2 {group: "vowel", word: "abacus", id: 2, rt: 212},3 {group: "vowel", word: "abacus", id: 3, rt: 209},4 {group: "vowel", word: "aardvark", id: 1, rt: 200},5 {group: "vowel", word: "aardvark", id: 2, rt: 201},6 {group: "vowel", word: "aardvark", id: 3, rt: 198},7 {group: "vowel", word: "ellipse", id: 1, rt: 220},8 {group: "vowel", word: "ellipse", id: 2, rt: 222},9 {group: "vowel", word: "ellipse", id: 3, rt: 219},10 {group: "consonant", word: "proton", id: 1, rt: 190},11 {group: "consonant", word: "proton", id: 2, rt: 191},12 {group: "consonant", word: "proton", id: 3, rt: 189},13 {group: "consonant", word: "folder", id: 1, rt: 180},14 {group: "consonant", word: "folder", id: 2, rt: 182},15 {group: "consonant", word: "folder", id: 3, rt: 178},16 {group: "consonant", word: "fedora", id: 1, rt: 230},17 {group: "consonant", word: "fedora", id: 2, rt: 231},18 {group: "consonant", word: "fedora", id: 3, rt: 228},19 {group: "consonant", word: "fedora", id: 1, rt: 231},20 {group: "consonant", word: "fedora", id: 2, rt: 233},21 {group: "consonant", word: "fedora", id: 3, rt: 230},22 {group: "consonant", word: "fedora", id: 1, rt: 230},23 {group: "consonant", word: "fedora", id: 2, rt: 232},24 {group: "consonant", word: "fedora", id: 3, rt: 228}];2526var opts = {method: "MCMC", burn: 10000, lag: 5, samples: 5000};27var ANSWER = (Infer(opts, function() {28 var groupMeans = {vowel: gaussian(200, 100),29 consonant: gaussian(200, 100)};3031 var wordMean = mem(function(word, group) {32 return gaussian(groupMeans[group], 20);33 });3435 var obsFn = function(d) {36 observe(Gaussian({mu: wordMean(d.word, d.group),37 sigma: 10}), d.rt);38 };3940 mapData({data: data}, obsFn);4142 return groupMeans['vowel'] - groupMeans['consonant'];43}));44
1var data = [{group: "vowel", word: "abacus", id: 1, rt: 210},2 {group: "vowel", word: "abacus", id: 2, rt: 212},3 {group: "vowel", word: "abacus", id: 3, rt: 209},4 {group: "vowel", word: "aardvark", id: 1, rt: 200},5 {group: "vowel", word: "aardvark", id: 2, rt: 201},6 {group: "vowel", word: "aardvark", id: 3, rt: 198},7 {group: "vowel", word: "ellipse", id: 1, rt: 220},8 {group: "vowel", word: "ellipse", id: 2, rt: 222},9 {group: "vowel", word: "ellipse", id: 3, rt: 219},10 {group: "consonant", word: "proton", id: 1, rt: 190},11 {group: "consonant", word: "proton", id: 2, rt: 191},12 {group: "consonant", word: "proton", id: 3, rt: 189},13 {group: "consonant", word: "folder", id: 1, rt: 180},14 {group: "consonant", word: "folder", id: 2, rt: 182},15 {group: "consonant", word: "folder", id: 3, rt: 178},16 {group: "consonant", word: "fedora", id: 1, rt: 230},17 {group: "consonant", word: "fedora", id: 2, rt: 231},18 {group: "consonant", word: "fedora", id: 3, rt: 228},19 {group: "consonant", word: "fedora", id: 1, rt: 231},20 {group: "consonant", word: "fedora", id: 2, rt: 233},21 {group: "consonant", word: "fedora", id: 3, rt: 230},22 {group: "consonant", word: "fedora", id: 1, rt: 230},23 {group: "consonant", word: "fedora", id: 2, rt: 232},24 {group: "consonant", word: "fedora", id: 3, rt: 228}];2526var opts = {method: "MCMC", burn: 10000, lag: 5, samples: 5000};2728var ANSWER = Infer(opts, function() {29 var groupMeans = {vowel: gaussian(200, 100),30 consonant: gaussian(200, 100)};31 var wordMean = mem(function(word, group) {32 return gaussian(groupMeans[group], 20);33 });34 var obsFn = function(d) {35 observe(Gaussian({mu: wordMean(d.word, d.group), sigma: 10}), d.rt);36 };37 mapData({data: data}, obsFn);38 return groupMeans['vowel'] - groupMeans['consonant'];39});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Hierarchical Bayesian data analysis on word reading times. Each data point is {group: 'vowel'|'consonant', word: str, id: int, rt: number}. Group means are drawn from Gaussian(200, 100); each word has its own mean mem(function(word, group) { gaussian(groupMeans[group], 20) }); individual reading times are Gaussian({mu: wordMean(d.word, d.group), sigma: 10}).
Extend this model with a per-participant random effect: a Gaussian(0, 2) additive offset for each participant id. Each rt is now Gaussian({mu: wordMean(d.word, d.group) + participantMean(d.id), sigma: 10}).
Data:
1var data = [{group: "vowel", word: "abacus", id: 1, rt: 210},2 {group: "vowel", word: "abacus", id: 2, rt: 212},3 {group: "vowel", word: "abacus", id: 3, rt: 209},4 {group: "vowel", word: "aardvark", id: 1, rt: 200},5 {group: "vowel", word: "aardvark", id: 2, rt: 201},6 {group: "vowel", word: "aardvark", id: 3, rt: 198},7 {group: "vowel", word: "ellipse", id: 1, rt: 220},8 {group: "vowel", word: "ellipse", id: 2, rt: 222},9 {group: "vowel", word: "ellipse", id: 3, rt: 219},10 {group: "consonant", word: "proton", id: 1, rt: 190},11 {group: "consonant", word: "proton", id: 2, rt: 191},12 {group: "consonant", word: "proton", id: 3, rt: 189},13 {group: "consonant", word: "folder", id: 1, rt: 180},14 {group: "consonant", word: "folder", id: 2, rt: 182},15 {group: "consonant", word: "folder", id: 3, rt: 178},16 {group: "consonant", word: "fedora", id: 1, rt: 230},17 {group: "consonant", word: "fedora", id: 2, rt: 231},18 {group: "consonant", word: "fedora", id: 3, rt: 228},19 {group: "consonant", word: "fedora", id: 1, rt: 231},20 {group: "consonant", word: "fedora", id: 2, rt: 233},21 {group: "consonant", word: "fedora", id: 3, rt: 230},22 {group: "consonant", word: "fedora", id: 1, rt: 230},23 {group: "consonant", word: "fedora", id: 2, rt: 232},24 {group: "consonant", word: "fedora", id: 3, rt: 228}];
Use Infer({method: 'MCMC', burn: 10000, lag: 5, samples: 5000}, ...). End your program with the joint posterior {diff, p1, p2, p3} where diff = groupMeans['vowel'] - groupMeans['consonant'] and p1, p2, p3 are participantMean(1), participantMean(2), participantMean(3).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var data = [{group: "vowel", word: "abacus", id: 1, rt: 210},2 {group: "vowel", word: "abacus", id: 2, rt: 212},3 {group: "vowel", word: "abacus", id: 3, rt: 209},4 {group: "vowel", word: "aardvark", id: 1, rt: 200},5 {group: "vowel", word: "aardvark", id: 2, rt: 201},6 {group: "vowel", word: "aardvark", id: 3, rt: 198},7 {group: "vowel", word: "ellipse", id: 1, rt: 220},8 {group: "vowel", word: "ellipse", id: 2, rt: 222},9 {group: "vowel", word: "ellipse", id: 3, rt: 219},10 {group: "consonant", word: "proton", id: 1, rt: 190},11 {group: "consonant", word: "proton", id: 2, rt: 191},12 {group: "consonant", word: "proton", id: 3, rt: 189},13 {group: "consonant", word: "folder", id: 1, rt: 180},14 {group: "consonant", word: "folder", id: 2, rt: 182},15 {group: "consonant", word: "folder", id: 3, rt: 178},16 {group: "consonant", word: "fedora", id: 1, rt: 230},17 {group: "consonant", word: "fedora", id: 2, rt: 231},18 {group: "consonant", word: "fedora", id: 3, rt: 228},19 {group: "consonant", word: "fedora", id: 1, rt: 231},20 {group: "consonant", word: "fedora", id: 2, rt: 233},21 {group: "consonant", word: "fedora", id: 3, rt: 230},22 {group: "consonant", word: "fedora", id: 1, rt: 230},23 {group: "consonant", word: "fedora", id: 2, rt: 232},24 {group: "consonant", word: "fedora", id: 3, rt: 228}];2526var opts = {method: "MCMC", burn: 10000, lag: 5, samples: 5000};27var ANSWER = (Infer(opts, function() {28 var groupMeans = {vowel: gaussian(200, 100),29 consonant: gaussian(200, 100)};3031 var participantMean = mem(function(pid) {32 return gaussian(0, 2);33 });3435 var wordMean = mem(function(word, group) {36 return gaussian(groupMeans[group], 20);37 });3839 var obsFn = function(d) {40 observe(Gaussian({mu: wordMean(d.word, d.group) + participantMean(d.id),41 sigma: 10}), d.rt);42 };4344 mapData({data: data}, obsFn);4546 return {diff: groupMeans['vowel'] - groupMeans['consonant'],47 p1: participantMean(1),48 p2: participantMean(2),49 p3: participantMean(3)};50}));51
1var data = [{group: "vowel", word: "abacus", id: 1, rt: 210},2 {group: "vowel", word: "abacus", id: 2, rt: 212},3 {group: "vowel", word: "abacus", id: 3, rt: 209},4 {group: "vowel", word: "aardvark", id: 1, rt: 200},5 {group: "vowel", word: "aardvark", id: 2, rt: 201},6 {group: "vowel", word: "aardvark", id: 3, rt: 198},7 {group: "vowel", word: "ellipse", id: 1, rt: 220},8 {group: "vowel", word: "ellipse", id: 2, rt: 222},9 {group: "vowel", word: "ellipse", id: 3, rt: 219},10 {group: "consonant", word: "proton", id: 1, rt: 190},11 {group: "consonant", word: "proton", id: 2, rt: 191},12 {group: "consonant", word: "proton", id: 3, rt: 189},13 {group: "consonant", word: "folder", id: 1, rt: 180},14 {group: "consonant", word: "folder", id: 2, rt: 182},15 {group: "consonant", word: "folder", id: 3, rt: 178},16 {group: "consonant", word: "fedora", id: 1, rt: 230},17 {group: "consonant", word: "fedora", id: 2, rt: 231},18 {group: "consonant", word: "fedora", id: 3, rt: 228},19 {group: "consonant", word: "fedora", id: 1, rt: 231},20 {group: "consonant", word: "fedora", id: 2, rt: 233},21 {group: "consonant", word: "fedora", id: 3, rt: 230},22 {group: "consonant", word: "fedora", id: 1, rt: 230},23 {group: "consonant", word: "fedora", id: 2, rt: 232},24 {group: "consonant", word: "fedora", id: 3, rt: 228}];2526var model = function() {27 var groupMeans = {28 vowel: gaussian(200, 100),29 consonant: gaussian(200, 100)30 };3132 var wordMean = mem(function(word, group) {33 return gaussian(groupMeans[group], 20);34 });3536 var participantMean = mem(function(id) {37 return gaussian(0, 2);38 });3940 mapData({data: data}, function(d) {41 observe(Gaussian({mu: wordMean(d.word, d.group) + participantMean(d.id), sigma: 10}), d.rt);42 });4344 return {45 diff: groupMeans['vowel'] - groupMeans['consonant'],46 p1: participantMean(1),47 p2: participantMean(2),48 p3: participantMean(3)49 };50};5152var ANSWER = Infer({method: 'MCMC', burn: 10000, lag: 5, samples: 5000}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
We have a vocabulary {dogs, cats, chase, sleep, stop} and a bigram model with Dirichlet priors over per-word transition distributions.
Given the scaffold:
1Infer({method:'MCMC', burn:10000, samples: 50000, onlyMAP:false}, function() {2 let vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];3 var wordToDistribution = mem(function(word) {4 return dirichletDrift({alpha:ones([vocab.length,1]), concentration:10})5 })6 var transition = function(word) {7 return categorical({ps: wordToDistribution(word), vs: vocab})8 }9 // ...your code here...10})
Someone says 'dogs chase cats'. Determine how likely 'chase' is to be followed by each word. Use a recursive generateSentence that emits until 'stop'. Condition on the observation. End with the Infer(...) returning the posterior over transition('chase').
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var comparray = function(arr1,arr2){2 return (JSON.stringify(arr1) === JSON.stringify(arr2));3};4var ANSWER = (Infer({method:'MCMC', burn:10000, samples: 50000, onlyMAP:false}, function() {5 let vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];6 var wordToDistribution = mem(function(word) {7 return dirichletDrift({alpha:ones([vocab.length,1]), concentration:10});8 });9 var transition = function(word) {10 return categorical({ps: wordToDistribution(word), vs: vocab});11 };12 let obs = ['dogs', 'chase', 'cats'];13 let generateSentence = function(lastState, sentence) {14 let word = transition(lastState);15 if (word == 'stop') return [];16 return [word].concat(generateSentence(word, sentence));17 };18 condition(comparray(obs, generateSentence('start')));19 return transition('chase');20}));21
1var ANSWER = Infer({method:'MCMC', burn:10000, samples: 50000, onlyMAP:false}, function() {2 var vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];3 var wordToDistribution = mem(function(word) {4 return dirichletDrift({alpha:ones([vocab.length,1]), concentration:10});5 });6 var transition = function(word) {7 return categorical({ps: wordToDistribution(word), vs: vocab});8 };910 var generateSentence = function(word, sentence) {11 var nextWord = transition(word);12 var newSentence = sentence.concat([nextWord]);13 return (nextWord === 'stop') ? newSentence : generateSentence(nextWord, newSentence);14 };1516 var observeSentence = function(words) {17 // words is the sequence after the start, ending with 'stop'18 // We observe each transition: start -> words[0], words[0] -> words[1], ...19 // The observed sentence is 'dogs chase cats stop'20 // transitions: dogs->chase, chase->cats, cats->stop21 observe(Categorical({ps: wordToDistribution('dogs'), vs: vocab}), 'chase');22 observe(Categorical({ps: wordToDistribution('chase'), vs: vocab}), 'cats');23 observe(Categorical({ps: wordToDistribution('cats'), vs: vocab}), 'stop');24 };2526 observeSentence(['chase', 'cats', 'stop']);2728 return wordToDistribution('chase');29});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◉TV<.05 | 0.028 | 0.0036 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◐TV<.5 | 0.084 | 0.0167 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◉TV<.05 | 0.045 | 0.0079 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◉TV<.05 | 0.045 | 0.0079 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 1.000 | 15.5874 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 0.028 | 0.0036 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◉TV<.05 | 0.035 | 0.0030 | · |
Bigram sentence model over vocabulary ['dogs', 'cats', 'chase', 'sleep', 'stop']. Each word has its own transition distribution drawn from a dirichletDrift({alpha: ones([5,1]), concentration: 10}). A sentence is generated by starting at the special token 'start', repeatedly applying transition(prevWord) to draw the next word, terminating when 'stop' is drawn (and emitting 'stop' so that the observed sentence's length matches without undefined).
Helper:
1var comparray = function(arr1, arr2) {2 return JSON.stringify(arr1) === JSON.stringify(arr2);3};
Use Infer({method: 'MCMC', burn: 10000, samples: 50000, onlyMAP: false}, ...). Observe the sentence ['dogs', 'chase', 'cats', 'stop']. Then, in a second sentence, the first word is 'dogs'. End your program with the Infer(...) returning the marginal distribution over the second word of this new sentence.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var comparray = function(arr1,arr2){2 return (JSON.stringify(arr1) === JSON.stringify(arr2));3};4var ANSWER = (Infer({method:'MCMC', burn:10000, samples: 50000, onlyMAP: false}, function() {5 let vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];6 var wordToDistribution = mem(function(word) {7 return dirichletDrift({alpha:ones([vocab.length,1]), concentration:10});8 });9 var transition = function(word) {10 return categorical({ps: wordToDistribution(word), vs: vocab});11 };12 let generateSentence = function(lastState, sentence) {13 let word = transition(lastState);14 if (word == 'stop') return ['stop'];15 return [word].concat(generateSentence(word, sentence));16 };17 let obs = ['dogs', 'chase', 'cats', 'stop'];18 condition(comparray(obs, generateSentence('start')));19 let newSentence = generateSentence('start');20 condition(newSentence[0] == 'dogs');21 return newSentence[1];22}));23
1var vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];2var n = vocab.length;34var comparray = function(arr1, arr2) {5 return JSON.stringify(arr1) === JSON.stringify(arr2);6};78var model = function() {9 // Each word (including 'start') has its own transition distribution10 // Words that can be "previous": start, dogs, cats, chase, sleep, stop11 var prevWords = ['start', 'dogs', 'cats', 'chase', 'sleep', 'stop'];1213 // Draw transition distributions for each previous word14 var transitionDists = mem(function(word) {15 return dirichletDrift({alpha: ones([n, 1]), concentration: 10});16 });1718 var transition = function(prevWord) {19 var probs = transitionDists(prevWord);20 var idx = sample(Categorical({vs: vocab, ps: probs}));21 return idx;22 };2324 // Generate a sentence starting from 'start'25 var generateSentence = function(prevWord, acc) {26 var nextWord = transition(prevWord);27 var newAcc = acc.concat([nextWord]);28 if (nextWord === 'stop') {29 return newAcc;30 } else {31 return generateSentence(nextWord, newAcc);32 }33 };3435 // Observe the sentence ['dogs', 'chase', 'cats', 'stop']36 var observed = ['dogs', 'chase', 'cats', 'stop'];37 var generated = generateSentence('start', []);38 condition(comparray(generated, observed));3940 // Second sentence: first word is 'dogs', return second word41 var secondWord = transition('dogs');42 return secondWord;43};4445var ANSWER = Infer({method: 'MCMC', burn: 10000, samples: 50000, onlyMAP: false}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 0.047 | 0.0064 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◐TV<.5 | 0.117 | 0.0373 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ⚠fail | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Bigram sentence model over vocabulary ['dogs', 'cats', 'chase', 'sleep', 'stop']. Each word has its own transition distribution drawn from a dirichletDrift({alpha: ones([5,1]), concentration: 10}). A sentence is generated by starting at the special token 'start', repeatedly applying transition(prevWord) to draw the next word, terminating when 'stop' is drawn (and emitting 'stop' so that the observed sentence's length matches without undefined).
Helper:
1var comparray = function(arr1, arr2) {2 return JSON.stringify(arr1) === JSON.stringify(arr2);3};
Use Infer({method: 'MCMC', burn: 10000, samples: 50000, onlyMAP: false}, ...). Observe the sentence ['dogs', 'chase', 'cats', 'stop']. Then, in a second sentence, the second word is 'chase'. End your program with the Infer(...) returning the marginal distribution over the FIRST word of this new sentence.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var comparray = function(arr1,arr2){2 return (JSON.stringify(arr1) === JSON.stringify(arr2));3};4var ANSWER = (Infer({method:'MCMC', burn:10000, samples: 50000, onlyMAP: false}, function() {5 let vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];6 var wordToDistribution = mem(function(word) {7 return dirichletDrift({alpha:ones([vocab.length,1]), concentration:10});8 });9 var transition = function(word) {10 return categorical({ps: wordToDistribution(word), vs: vocab});11 };12 let generateSentence = function(lastState, sentence) {13 let word = transition(lastState);14 if (word == 'stop') return ['stop'];15 return [word].concat(generateSentence(word, sentence));16 };17 let obs = ['dogs', 'chase', 'cats', 'stop'];18 condition(comparray(obs, generateSentence('start')));19 let newSentence = generateSentence('start');20 condition(newSentence[1] == 'chase');21 return newSentence[0];22}));23
1var vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];2var vocabSize = vocab.length;34var comparray = function(arr1, arr2) {5 return JSON.stringify(arr1) === JSON.stringify(arr2);6};78var indexOf = function(word) {9 return _.indexOf(vocab, word);10};1112var model = function() {13 // Each word (including 'start') has a transition distribution over vocab14 // Words that can be "previous": start, dogs, cats, chase, sleep, stop15 // We need transition from 'start' and from each vocab word16 var allPrev = ['start'].concat(vocab);1718 // Draw transition distributions for each previous word19 var transitionDists = mem(function(prevWord) {20 return dirichletDrift({alpha: ones([5, 1]), concentration: 10});21 });2223 var transition = function(prevWord) {24 var probs = transitionDists(prevWord);25 var idx = sample(Categorical({vs: vocab, ps: probs}));26 return idx;27 };2829 // Generate a sentence starting from 'start'30 var generateSentence = function(prevWord, acc) {31 var nextWord = transition(prevWord);32 var newAcc = acc.concat([nextWord]);33 if (nextWord === 'stop') {34 return newAcc;35 } else {36 return generateSentence(nextWord, newAcc);37 }38 };3940 // Observe the first sentence: ['dogs', 'chase', 'cats', 'stop']41 var observed1 = ['dogs', 'chase', 'cats', 'stop'];42 var generated1 = generateSentence('start', []);43 condition(comparray(generated1, observed1));4445 // Generate a second sentence and condition on second word being 'chase'46 var generated2 = generateSentence('start', []);47 condition(generated2.length >= 2 && generated2[1] === 'chase');4849 // Return the first word of the second sentence50 return generated2[0];51};5253var ANSWER = Infer({method: 'MCMC', burn: 10000, samples: 50000, onlyMAP: false}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◐TV<.5 | 0.183 | 2.7039 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.167 | 2.9909 | · |
Bigram sentence model over vocabulary ['dogs', 'cats', 'chase', 'sleep', 'stop']. Each word has its own transition distribution drawn from a dirichletDrift({alpha: ones([5,1]), concentration: 10}). A sentence is generated by starting at the special token 'start', repeatedly applying transition(prevWord) to draw the next word, terminating when 'stop' is drawn (and emitting 'stop' so that the observed sentence's length matches without undefined).
Helper:
1var comparray = function(arr1, arr2) {2 return JSON.stringify(arr1) === JSON.stringify(arr2);3};
Use Infer({method: 'MCMC', burn: 10000, samples: 50000, onlyMAP: false}, ...). Observe the sentence ['dogs', 'chase', 'cats', 'stop']. Then, in a second sentence, the first word is 'cats'. End your program with the Infer(...) returning the marginal distribution over the second word of this new sentence.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var comparray = function(arr1,arr2){2 return (JSON.stringify(arr1) === JSON.stringify(arr2));3};4var ANSWER = (Infer({method:'MCMC', burn:10000, samples: 50000, onlyMAP: false}, function() {5 let vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];6 var wordToDistribution = mem(function(word) {7 return dirichletDrift({alpha:ones([vocab.length,1]), concentration:10});8 });9 var transition = function(word) {10 return categorical({ps: wordToDistribution(word), vs: vocab});11 };12 let generateSentence = function(lastState, sentence) {13 let word = transition(lastState);14 if (word == 'stop') return ['stop'];15 return [word].concat(generateSentence(word, sentence));16 };17 let obs = ['dogs', 'chase', 'cats', 'stop'];18 condition(comparray(obs, generateSentence('start')));19 let newSentence = generateSentence('start');20 condition(newSentence[0] == 'cats');21 return newSentence[1];22}));23
1var vocab = ['dogs', 'cats', 'chase', 'sleep', 'stop'];2var n = vocab.length;34var comparray = function(arr1, arr2) {5 return JSON.stringify(arr1) === JSON.stringify(arr2);6};78var model = function() {9 // Each word (including 'start') has a transition distribution over vocab10 var allWords = ['start', 'dogs', 'cats', 'chase', 'sleep', 'stop'];1112 // Memoized transition distributions13 var transitionDist = mem(function(word) {14 return dirichletDrift({alpha: ones([n, 1]), concentration: 10});15 });1617 var transition = function(prevWord) {18 var probs = transitionDist(prevWord);19 return vocab[discrete(probs)];20 };2122 // Generate a sentence starting from 'start'23 var generateSentence = function(prevWord, acc) {24 var nextWord = transition(prevWord);25 var newAcc = acc.concat([nextWord]);26 if (nextWord === 'stop') {27 return newAcc;28 } else {29 return generateSentence(nextWord, newAcc);30 }31 };3233 // Observe the sentence ['dogs', 'chase', 'cats', 'stop']34 var observed = ['dogs', 'chase', 'cats', 'stop'];35 var generated = generateSentence('start', []);36 condition(comparray(generated, observed));3738 // Second sentence: first word is 'cats', return second word39 var secondWord = transition('cats');40 return secondWord;41};4243var ANSWER = Infer({method: 'MCMC', burn: 10000, samples: 50000, onlyMAP: false}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◐TV<.5 | 0.084 | 0.0159 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◐TV<.5 | 0.100 | 0.0239 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.084 | 0.0159 | · |
Define a hidden Markov model for sentence generation: words have parts of speech (N for nouns 'dogs','cats'; V for verbs 'chase','sleep'; 'stop'). Markov transitions are between POS, not words; words are then drawn given POS. Use Dirichlet-drift priors for the per-POS transition distributions. End your program with generateSentence('start') (a single sample - the harness will rerun your program multiple times to estimate the distribution).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var drawWord = function(pos){2 return (pos=="N") ? uniformDraw(['dogs','cats']) :3 (pos=="V") ? uniformDraw(['chase','sleep']) :4 'stop';5};6var POS = ["N", "V", "stop"];78var posToDistribution = mem(function(pos) {9 return dirichletDrift({alpha:ones([POS.length,1]), concentration:10});10});1112var transition = function(pos) {13 return categorical({ps: posToDistribution(pos), vs: POS});14};1516var generateSentence = function(lastPOS) {17 var nextPOS = transition(lastPOS);18 var word = drawWord(nextPOS);19 return (word == 'stop') ? [word] : [word].concat(generateSentence(nextPOS));20};21var ANSWER = (generateSentence("start"));22
1// HMC/MCMC with Dirichlet-drift priors for HMM sentence generation23// POS tags and vocabulary4var posStates = ['N', 'V', 'stop'];5var words = {6 'N': ['dogs', 'cats'],7 'V': ['chase', 'sleep'],8 'stop': ['stop']9};1011// Dirichlet priors for transitions from each POS (including 'start')12// From 'start': can go to N or V (not stop directly)13// From 'N': can go to V or stop14// From 'V': can go to N or stop1516// We use mem to get consistent parameters within a run17var transitionDist = mem(function(fromState) {18 if (fromState === 'start') {19 // Can transition to N or V20 var ps = dirichlet(Vector([1, 1]));21 return Categorical({vs: ['N', 'V'], ps: ps});22 } else if (fromState === 'N') {23 // Can transition to V or stop24 var ps = dirichlet(Vector([1, 1]));25 return Categorical({vs: ['V', 'stop'], ps: ps});26 } else if (fromState === 'V') {27 // Can transition to N or stop28 var ps = dirichlet(Vector([1, 1]));29 return Categorical({vs: ['N', 'stop'], ps: ps});30 } else {31 // stop -> stop (absorbing)32 return Categorical({vs: ['stop'], ps: Vector([1])});33 }34});3536var emissionDist = mem(function(pos) {37 if (pos === 'stop') {38 return Categorical({vs: ['stop'], ps: Vector([1])});39 }40 var vocabSize = words[pos].length;41 var ps = dirichlet(ones([vocabSize, 1]));42 return Categorical({vs: words[pos], ps: ps});43});4445var generateSentence = function(currentState) {46 if (currentState === 'stop') {47 return [];48 }49 var nextState = sample(transitionDist(currentState));50 if (nextState === 'stop') {51 return [];52 }53 var word = sample(emissionDist(nextState));54 return [word].concat(generateSentence(nextState));55};5657var ANSWER = generateSentence('start');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◔TV<1 | 0.960 | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 0.860 | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◔TV<1 | 1.000 | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 1.000 | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 0.750 | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◔TV<1 | 0.765 | — | · |
Hidden Markov sentence model. Words have parts of speech: N for nouns {'dogs', 'cats'}; V for verbs {'chase', 'sleep'}; plus 'stop'. Each POS has its own transition distribution drawn from dirichletDrift({alpha: ones([3,1]), concentration: 10}), memoized. A sentence is generated by starting at 'start' POS, transitioning to successive POS tags, drawing a word given each POS via drawWord(pos) (which returns uniformDraw([...]) for N or V, else 'stop'), and appending 'stop' when reached.
Helper:
1var comparray = function(arr1, arr2) {2 return JSON.stringify(arr1) === JSON.stringify(arr2);3};
Observe the sentence ['dogs', 'chase', 'cats', 'stop']. Then, in a second sentence whose first word is 'cats', end your program with the Infer(...) (MCMC, burn: 10000, samples: 1000, lag: 10, onlyMAP: false) returning the marginal distribution over the second word.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var comparray = function(arr1,arr2){2 return (JSON.stringify(arr1) === JSON.stringify(arr2));3};45var drawWord = function(pos){6 return (pos=="N") ? uniformDraw(['dogs','cats']) :7 (pos=="V") ? uniformDraw(['chase','sleep']) :8 'stop';9};10var POS = ["N", "V", "stop"];11var ANSWER = (Infer({method:'MCMC', burn:10000, samples: 1000, lag:10, onlyMAP: false}, function() {12 var posToDistribution = mem(function(pos) {13 return dirichletDrift({alpha:ones([POS.length,1]), concentration:10});14 });1516 var transition = function(pos) {17 return categorical({ps: posToDistribution(pos), vs: POS});18 };1920 let generateSentence = function(lastPOS) {21 let nextPOS = transition(lastPOS);22 let word = drawWord(nextPOS);23 return (word == 'stop') ? [word] : [word].concat(generateSentence(nextPOS));24 };25 let obs = ['dogs', 'chase', 'cats', 'stop'];26 condition(comparray(obs, generateSentence('start')));2728 let newSentence = generateSentence('start');29 condition(newSentence[0] == 'cats');30 return newSentence[1];31}));32
1var comparray = function(arr1, arr2) {2 return JSON.stringify(arr1) === JSON.stringify(arr2);3};45var posLabels = ['N', 'V', 'stop'];6var posIndex = {N: 0, V: 1, stop: 2};78var nouns = ['dogs', 'cats'];9var verbs = ['chase', 'sleep'];1011var drawWord = function(pos) {12 if (pos === 'N') return uniformDraw(nouns);13 if (pos === 'V') return uniformDraw(verbs);14 return 'stop';15};1617var getPosOfWord = function(word) {18 if (word === 'dogs' || word === 'cats') return 'N';19 if (word === 'chase' || word === 'sleep') return 'V';20 return 'stop';21};2223var model = function() {24 // Memoized transition distributions for each POS (including 'start')25 var transitionDist = mem(function(pos) {26 return dirichletDrift({alpha: ones([3, 1]), concentration: 10});27 });2829 var transition = function(pos) {30 var probs = transitionDist(pos);31 var idx = sample(Categorical({vs: posLabels, ps: [T.get(probs, 0), T.get(probs, 1), T.get(probs, 2)]}));32 return idx;33 };3435 // Generate a sentence from a given starting POS36 var generateSentence = function(currentPos, acc) {37 var nextPos = transition(currentPos);38 var word = drawWord(nextPos);39 if (nextPos === 'stop') {40 return acc.concat(['stop']);41 } else {42 return generateSentence(nextPos, acc.concat([word]));43 }44 };4546 // Observe sentence 1: ['dogs', 'chase', 'cats', 'stop']47 // We need to condition on this sentence being generated48 // The sentence is: start -> N(dogs) -> V(chase) -> N(cats) -> stop49 // We observe the transitions and words5051 // Sentence 1 observations52 var s1 = ['dogs', 'chase', 'cats', 'stop'];53 var s1pos = map(getPosOfWord, s1); // ['N', 'V', 'N', 'stop']5455 // Observe transitions for sentence 156 // start -> N57 var t1_0 = transitionDist('start');58 observe(Categorical({vs: posLabels, ps: [T.get(t1_0, 0), T.get(t1_0, 1), T.get(t1_0, 2)]}), 'N');59 // N -> V60 var t1_1 = transitionDist('N');61 observe(Categorical({vs: posLabels, ps: [T.get(t1_1, 0), T.get(t1_1, 1), T.get(t1_1, 2)]}), 'V');62 // V -> N63 var t1_2 = transitionDist('V');64 observe(Categorical({vs: posLabels, ps: [T.get(t1_2, 0), T.get(t1_2, 1), T.get(t1_2, 2)]}), 'N');65 // N -> stop (second time N is used, but transitionDist('N') is memoized so same dist)66 observe(Categorical({vs: posLabels, ps: [T.get(t1_1, 0), T.get(t1_1, 1), T.get(t1_1, 2)]}), 'stop');6768 // Sentence 2: first word is 'cats' (POS = N)69 // start -> N(cats) -> ?70 var t2_0 = transitionDist('start');71 observe(Categorical({vs: posLabels, ps: [T.get(t2_0, 0), T.get(t2_0, 1), T.get(t2_0, 2)]}), 'N');7273 // Now sample the second POS (transition from N)74 var secondPos = transition('N');75 var secondWord = drawWord(secondPos);7677 return secondWord;78};7980var ANSWER = Infer({method: 'MCMC', burn: 10000, samples: 1000, lag: 10, onlyMAP: false}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 0.523 | 0.7402 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◐TV<.5 | 0.095 | 0.0217 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ⚠fail | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.054 | 0.0064 | · |
Hidden Markov sentence model with extended vocabulary. POS tags: N for nouns {'dog', 'cat'}, V for verbs {'chases', 'sleeps'}, D for determiners {'the', 'a'}, A for adverbs ({'dilligently'}), plus 'stop'. Per-POS transitions drawn from dirichletDrift({alpha: ones([5,1]), concentration: 10}), memoized. drawWord(pos) returns uniformDraw(...) for the respective list, or 'dilligently' for A, or 'stop' for stop.
Helper:
1var comparray = function(arr1, arr2) {2 return JSON.stringify(arr1) === JSON.stringify(arr2);3};
Use factor(comparray(['the', 'dog', 'chases', 'a', 'cat', 'stop'], generateSentence('start')) * 5) to softly condition on the observed sentence. Then sample five new sentences from generateSentence('start'). End your program with Infer({method: 'MCMC', burn: 10000, samples: 1000, lag: 10, onlyMAP: true}, ...) returning a record {sent1, sent2, sent3, sent4, sent5}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var comparray = function(arr1,arr2){2 return (JSON.stringify(arr1) === JSON.stringify(arr2));3};45var drawWord = function(pos){6 return (pos=="N") ? uniformDraw(['dog','cat']) :7 (pos=="V") ? uniformDraw(['chases','sleeps']) :8 (pos=="D") ? uniformDraw(['the','a']) :9 (pos=="A") ? 'dilligently' :10 'stop';11};12var POS = ["N", "V", "D", "A", "stop"];13var ANSWER = (Infer({method:'MCMC', burn:10000, samples: 1000, lag:10, onlyMAP: true}, function() {14 var posToDistribution = mem(function(pos) {15 return dirichletDrift({alpha:ones([POS.length,1]), concentration:10});16 });1718 var transition = function(pos) {19 return categorical({ps: posToDistribution(pos), vs: POS});20 };2122 let generateSentence = function(lastPOS) {23 let nextPOS = transition(lastPOS);24 let word = drawWord(nextPOS);25 return (word == 'stop') ? [word] : [word].concat(generateSentence(nextPOS));26 };27 let obs = ['the', 'dog', 'chases', 'a', 'cat', 'stop'];2829 factor(comparray(obs, generateSentence('start'))*5);3031 var sent1 = generateSentence('start');32 var sent2 = generateSentence('start');33 var sent3 = generateSentence('start');34 var sent4 = generateSentence('start');35 var sent5 = generateSentence('start');36 return {sent1: sent1, sent2: sent2, sent3: sent3, sent4: sent4, sent5: sent5};37}));38
1var tags = ['N', 'V', 'D', 'A', 'stop'];23var tagIndex = function(tag) {4 return _.indexOf(tags, tag);5};67var transitionDist = mem(function(fromTag) {8 return dirichletDrift({alpha: ones([5, 1]), concentration: 10});9});1011var nextTag = function(fromTag) {12 var probs = transitionDist(fromTag);13 var idx = sample(Categorical({vs: tags, ps: [T.get(probs, 0), T.get(probs, 1), T.get(probs, 2), T.get(probs, 3), T.get(probs, 4)]}));14 return idx;15};1617var drawWord = function(pos) {18 return pos === 'N' ? uniformDraw(['dog', 'cat']) :19 pos === 'V' ? uniformDraw(['chases', 'sleeps']) :20 pos === 'D' ? uniformDraw(['the', 'a']) :21 pos === 'A' ? 'dilligently' :22 'stop';23};2425var generateSentence = function(currentTag) {26 var next = nextTag(currentTag);27 return next === 'stop' ? ['stop'] : [drawWord(next)].concat(generateSentence(next));28};2930var comparray = function(arr1, arr2) {31 return JSON.stringify(arr1) === JSON.stringify(arr2);32};3334var model = function() {35 factor(comparray(['the', 'dog', 'chases', 'a', 'cat', 'stop'], generateSentence('start')) * 5);3637 var sent1 = generateSentence('start');38 var sent2 = generateSentence('start');39 var sent3 = generateSentence('start');40 var sent4 = generateSentence('start');41 var sent5 = generateSentence('start');4243 return {sent1: sent1, sent2: sent2, sent3: sent3, sent4: sent4, sent5: sent5};44};4546var ANSWER = Infer({method: 'MCMC', burn: 10000, samples: 1000, lag: 10, onlyMAP: true}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ○TV=1 | 1.000 | 23.0259 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ⚠fail | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Use a phrase structure grammar instead of an HMM. Vocabulary: D='the','a'; N='cat','dog'; V='chases','sleeps'; A='diligently'. Production rules: AP -> A; NP -> [D, N]; VP -> [V, AP] | [V, NP]; S -> [NP, VP]. Condition on [['the', 'dog'], ['chases', ['a', 'cat']]] being a sentence (using comparray and S()), then sample 5 new sentences. Use MCMC with onlyMAP=true. End with the Infer(...) returning {sent1...sent5}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var comparray = function(arr1,arr2){2 return (JSON.stringify(arr1) === JSON.stringify(arr2));3};45var uniformDraw = function (xs) {return xs[randomInteger(xs.length)]};67var D = function() {return uniformDraw(['the', 'a'])};8var N = function() {return uniformDraw(['cat', 'dog'])};9var V = function() {return uniformDraw(['chases', 'sleeps'])};10var A = function() {return uniformDraw(['diligently'])};11var AP = function() {return uniformDraw([A()])};12var NP = function() {return [D(), N()]};13var VP = function() {return uniformDraw([[V(), AP()],14 [V(), NP()]])};15var S = function() {return [NP(), VP()]};16var ANSWER = (Infer({method:'MCMC', burn:10000, samples: 1000, onlyMAP: true}, function() {17 let obs = [['the', 'dog'], ['chases', ['a', 'cat']]];18 condition(comparray(obs, S()));1920 var sent1 = S();21 var sent2 = S();22 var sent3 = S();23 var sent4 = S();24 var sent5 = S();25 return {sent1: sent1, sent2: sent2, sent3: sent3, sent4: sent4, sent5: sent5};26}));27
1var D = ['the', 'a'];2var N = ['cat', 'dog'];3var V = ['chases', 'sleeps'];4var A = ['diligently'];56var AP = function() {7 return sample(Categorical({vs: A, ps: [1/A.length, 1/A.length].slice(0, A.length)}));8};910var apSample = function() {11 return A[randomInteger(A.length)];12};1314var npSample = function() {15 return [D[randomInteger(D.length)], N[randomInteger(N.length)]];16};1718var vpSample = function() {19 var choice = flip(0.5);20 return choice ? [V[randomInteger(V.length)], apSample()] : [V[randomInteger(V.length)], npSample()];21};2223var sSample = function() {24 return [npSample(), vpSample()];25};2627var comparray = function(a, b) {28 if (Array.isArray(a) && Array.isArray(b)) {29 if (a.length !== b.length) return false;30 return _.every(map2(function(x, y) { return comparray(x, y); }, a, b));31 } else {32 return a === b;33 }34};3536var observed = [['the', 'dog'], ['chases', ['a', 'cat']]];3738var model = function() {39 condition(comparray(sSample(), observed));40 var sent1 = sSample();41 var sent2 = sSample();42 var sent3 = sSample();43 var sent4 = sSample();44 var sent5 = sSample();45 return {sent1: sent1, sent2: sent2, sent3: sent3, sent4: sent4, sent5: sent5};46};4748var ANSWER = Infer({method: 'MCMC', samples: 1000, onlyMAP: true}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ○TV=1 | 1.000 | 23.0259 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ○TV=1 | 1.000 | 23.0259 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ○TV=1 | 1.000 | 23.0259 | · |
Modify the factor(...) line in the coin-flipping model so that the soft condition makes heads happen with approximately 95% probability:
1Infer({method: 'enumerate'}, function () {2 var A = flip()3 factor(A) // edit this line4 return A5});
Hint: a factor of c gives weight exp(c). End your program with the Infer(...).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (Infer({method: 'enumerate'}, function () {2 var A = flip();3 factor(A*3);4 return A;5}));6
1var ANSWER = Infer({method: 'enumerate'}, function () {2 var A = flip();3 factor(A ? Math.log(19) : 0);4 return A;5});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◉TV<.05 | 0.003 | 0.0001 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◉TV<.05 | 0.045 | 0.0378 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◉TV<.05 | 0.003 | 0.0001 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 0.003 | 0.0001 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◉TV<.05 | 0.003 | 0.0001 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◉TV<.05 | 0.003 | 0.0001 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 0.003 | 0.0001 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◉TV<.05 | 0.003 | 0.0001 | · |
Three coins are flipped. Use factor to favor outcomes where the number of heads (true's) equals 2:
1Infer({}, function() {2 var a = flip(0.5);3 var b = flip(0.5);4 var c = flip(0.5);5 factor(...); // fill in6 return a;7})
End your program with the Infer(...).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (Infer({}, function() {2 var a = flip(0.5);3 var b = flip(0.5);4 var c = flip(0.5);5 factor(1*((a+b+c)==2));6 return a;7}));8
1var ANSWER = Infer({}, function() {2 var a = flip(0.5);3 var b = flip(0.5);4 var c = flip(0.5);5 var numHeads = (a ? 1 : 0) + (b ? 1 : 0) + (c ? 1 : 0);6 factor(numHeads === 2 ? 0 : -Infinity);7 return a;8});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◐TV<.5 | 0.052 | 0.0056 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◐TV<.5 | 0.101 | 0.0214 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◐TV<.5 | 0.101 | 0.0214 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◐TV<.5 | 0.101 | 0.0214 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◐TV<.5 | 0.101 | 0.0214 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◐TV<.5 | 0.101 | 0.0214 | · |
Ultimatum game: a proposer allocates $10 between themselves and a responder in $1 increments. The responder accepts or rejects. If accepted, both get the split; if rejected, both get $0.
Assume the responder is a strict utilitarian (accepts any offer >= $1). The proposer is a soft maximizer who wants to keep as much of the $10 as possible. End your program with the Infer(...) over the proposer's offer.
1var responder = function(offer) { /* fill in */ }2Infer({method: "enumerate"}, function(){3 // sample offer, compute reward = responder(offer) ? (10-offer) : 04 // factor(reward)5 return offer6})
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var responder = function(offer) {2 return (offer>0 ? true : false);3};4var ANSWER = (Infer({method: "enumerate"}, function(){5 var offer = uniformDraw([0,1,2,3,4,5,6,7,8,9,10]);6 var reward = responder(offer) ? (10 - offer) : 0;7 factor(reward);8 return offer;9}));10
1var responder = function(offer) {2 return offer >= 1;3};45var ANSWER = Infer({method: "enumerate"}, function() {6 var offer = randomInteger(10) + 1; // offers from 1 to 107 var reward = responder(offer) ? (10 - offer) : 0;8 factor(reward);9 return offer;10});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◉TV<.05 | 7.8e-5 | 0.0001 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Modified ultimatum game: the responder accepts in proportion to the fraction of $10 allocated to her, raised to a power alpha = 2 (a spitefulness parameter):
1var alpha = 2;2var responder = function(offer, alpha) {3 var p = Math.pow(offer/10, alpha);4 return flip(p);5}
Use offers 0..10 in $1 increments. End with the Infer(...) over the proposer's offer (factor on reward).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var alpha = 2;23var responder = function(offer, alpha) {4 var p = Math.pow(offer/10,alpha);5 return flip(p);6};7var ANSWER = (Infer({method: "enumerate"}, function(){8 var offer = uniformDraw([0,1,2,3,4,5,6,7,8,9,10]);9 var reward = responder(offer,alpha) ? (10 - offer) : 0;10 factor(reward);11 return offer;12}));13
1var alpha = 2;23var responder = function(offer, alpha) {4 var p = Math.pow(offer / 10, alpha);5 return flip(p);6};78var model = function() {9 var offer = randomInteger(11); // 0..1010 var accepted = responder(offer, alpha);11 var reward = accepted ? (10 - offer) : 0;12 factor(reward);13 return offer;14};1516var ANSWER = Infer({method: 'enumerate'}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
In the ultimatum game (responder accepts with probability (offer/10)^alpha), the proposer doesn't know alpha but believes it is uniformly distributed on [0.5, 5]. The proposer offered $2 and the responder rejected it. End with the Infer(...) returning the posterior over alpha. Use MCMC with 50000 samples.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var responder = function(offer, alpha) {2 var p = Math.pow(offer/10,alpha);3 return flip(p);4};5var ANSWER = (Infer({method: "MCMC", samples:50000}, function(){6 var alpha = uniform(0.5,5);7 var offer = 2;8 var reward = responder(offer, alpha) ? (10 - offer) : 0;9 condition(reward==0);10 return alpha;11}));12
1var model = function() {2 var alpha = uniform(0.5, 5);34 // Probability that responder accepts offer of $2 out of $105 var pAccept = Math.pow(2 / 10, alpha);67 // Responder rejected, so observe false (not accepted)8 observe(Bernoulli({p: pAccept}), false);910 return alpha;11};1213var ANSWER = Infer({method: 'MCMC', samples: 50000}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◐TV<.5 | 0.144 | 1.1342 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◔TV<1 | 0.833 | 10.1402 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 0.833 | 10.1402 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 0.853 | 10.3914 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◔TV<1 | 0.833 | 10.1402 | · |
Ultimatum game with uncertain alpha. Responder accepts with probability Math.pow(offer/10, alpha); the proposer doesn't know alpha but believes it is uniform on [0.5, 5].
Setup: in round 1, the proposer offered $2 and the responder rejected. In round 2, what should the proposer offer to maximize expected payoff?
Two-stage inference:
1. proposer1: Infer (MCMC, 50000 samples) the posterior over alpha given that round 1's offer of $2 was rejected.
2. End your program with an outer Infer (forward, 1000 samples) that samples an alpha2 from proposer1, then runs an inner Infer (MCMC, 5000 samples) over offers 0..10 with factor(reward2) where reward2 = responder(offer2, alpha2) ? (10 - offer2) : 0. Sample one round-2 offer from that inner posterior and return it. The outer Infer is the distribution over the chosen round-2 offer.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var responder = function(offer, alpha) {2 var p = Math.pow(offer/10,alpha);3 return flip(p);4};56var proposer1 = Infer({method: "MCMC", samples:50000}, function(){7 var alpha = uniform(0.5,5);8 var offer1 = 2;9 var reward1 = responder(offer1, alpha) ? (10 - offer1) : 0;10 condition(reward1==0);11 return alpha;12});13var ANSWER = (Infer({method: "forward", samples:1000}, function(){14 var alpha2 = sample(proposer1);15 var proposer2 = Infer({method: "MCMC", samples:5000}, function(){16 var offer2 = uniformDraw([0,1,2,3,4,5,6,7,8,9,10]);17 var reward2 = responder(offer2, alpha2) ? (10 - offer2) : 0;18 factor(reward2);19 return offer2;20 });21 return sample(proposer2);22}));23
1var responder = function(offer, alpha) {2 return flip(Math.pow(offer / 10, alpha));3};45// Stage 1: Infer posterior over alpha given rejection of offer=26var proposer1 = Infer({method: 'MCMC', samples: 50000}, function() {7 var alpha = uniform(0.5, 5);8 var accepted = responder(2, alpha);9 condition(!accepted);10 return alpha;11});1213// Stage 2: For each alpha2 sampled from posterior, find best offer14var ANSWER = Infer({method: 'forward', samples: 1000}, function() {15 var alpha2 = sample(proposer1);1617 var innerDist = Infer({method: 'MCMC', samples: 5000}, function() {18 var offer2 = randomInteger(11); // 0..1019 var accepted = responder(offer2, alpha2);20 var reward2 = accepted ? (10 - offer2) : 0;21 factor(reward2);22 return offer2;23 });2425 var chosenOffer = sample(innerDist);26 return chosenOffer;27});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◐TV<.5 | 0.356 | 0.3853 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ⚠fail | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◉TV<.05 | 0.048 | 0.0083 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 0.048 | 0.0083 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 0.048 | 0.0083 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◉TV<.05 | 0.048 | 0.0083 | · |
Prisoner's Dilemma. Two thieves are interrogated separately. If a thief confesses she gets a lenient sentence (lenient = 6 years). If she doesn't but the other does, she gets 10 years. If neither confesses, both go free. With lenient = 6, use factor(percentYearsFreedom) (where percentYearsFreedom = (10 - years_in_jail) / 10). The other thief flips uniformly to decide whether to rat. End your program with the Infer(...) returning the posterior over the focal thief's rat decision.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var thiefRats = function(){2 return flip();3};45var lenient = 6;6var ANSWER = (Infer({}, function(){7 var otherThiefRats = thiefRats();8 var IRat = thiefRats();9 var years = (otherThiefRats?10 (IRat? lenient : 10) :11 (IRat? lenient : 0));12 var percentYearsFreedom = (10-years)/10;13 factor(percentYearsFreedom);14 return IRat;15}));16
1var model = function() {2 // Focal thief's decision: rat or not3 var rat = flip(0.5);45 // Other thief flips uniformly6 var otherRat = flip(0.5);78 var lenient = 6;910 // Compute years in jail for focal thief11 var years = rat ? lenient : (otherRat ? 10 : 0);1213 // percentYearsFreedom = (10 - years) / 1014 var percentYearsFreedom = (10 - years) / 10;1516 factor(percentYearsFreedom);1718 return rat;19};2021var ANSWER = Infer({method: 'enumerate'}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 0.006 | 0.0001 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ○TV=1 | 1.000 | 22.3387 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ○TV=1 | 1.000 | 22.3387 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Implement the Frank & Goodman RSA model. The world has three objects represented as records {shape, color}: {shape: "square", color: "blue"}, {shape: "circle", color: "blue"}, {shape: "square", color: "green"}, drawn uniformly. Possible utterances: 'blue', 'green', 'square', 'circle'. Truth function: color/shape utterances must match the corresponding attribute, otherwise true.
Build:
- literalListener(utterance): Infer over meaningPrior conditional on truth
- speaker(obj, alpha): uniformDraw utterance, factor alpha * literalListener(utterance).score(obj)
- pragmaticListener(utterance, alpha): Infer over meaningPrior conditional on speaker(obj, alpha)
Return an object literal with pragmaticListener('blue', alpha) for four alphas, keyed as:
- alpha_001 (alpha=0.01)
- alpha_1 (alpha=1)
- alpha_4 (alpha=4)
- alpha_10 (alpha=10)
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var meaningPrior = function() {2 uniformDraw([3 {shape: "square", color: "blue"},4 {shape: "circle", color: "blue"},5 {shape: "square", color: "green"}6 ])7};89var utterances = ["blue","green","square","circle"];1011var meaning = function(utterance, obj){12 (utterance === "blue" || utterance === "green") ? utterance === obj.color :13 (utterance === "circle" || utterance === "square") ? utterance === obj.shape :14 true15};1617var literalListener = function(utterance){18 return Infer({model: function(){19 var obj = meaningPrior();20 condition(meaning(utterance, obj));21 return obj;22 }});23};2425var speaker = function(obj,alpha){26 return Infer({model: function(){27 var utterance = uniformDraw(utterances);28 factor(alpha * literalListener(utterance).score(obj));29 return utterance;30 }});31};3233var pragmaticListener = function(utterance,alpha){34 return Infer({model: function(){35 var obj = meaningPrior();36 observe(speaker(obj,alpha),utterance);37 return obj;38 }});39};40var ANSWER = (({41 alpha_001: pragmaticListener("blue", 0.01),42 alpha_1: pragmaticListener("blue", 1),43 alpha_4: pragmaticListener("blue", 4),44 alpha_10: pragmaticListener("blue", 10)45}));46
1var objects = [2 {shape: "square", color: "blue"},3 {shape: "circle", color: "blue"},4 {shape: "square", color: "green"}5];67var utterances = ['blue', 'green', 'square', 'circle'];89var meaningPrior = function() {10 return uniformDraw(objects);11};1213var meaning = function(utterance, obj) {14 if (utterance === 'blue' || utterance === 'green') {15 return obj.color === utterance;16 } else if (utterance === 'square' || utterance === 'circle') {17 return obj.shape === utterance;18 } else {19 return true;20 }21};2223var literalListener = mem(function(utterance) {24 return Infer({method: 'enumerate'}, function() {25 var obj = meaningPrior();26 condition(meaning(utterance, obj));27 return obj;28 });29});3031var speaker = function(obj, alpha) {32 return Infer({method: 'enumerate'}, function() {33 var utterance = uniformDraw(utterances);34 factor(alpha * literalListener(utterance).score(obj));35 return utterance;36 });37};3839var pragmaticListener = function(utterance, alpha) {40 return Infer({method: 'enumerate'}, function() {41 var obj = meaningPrior();42 observe(speaker(obj, alpha), utterance);43 return obj;44 });45};4647var ANSWER = {48 alpha_001: pragmaticListener('blue', 0.01),49 alpha_1: pragmaticListener('blue', 1),50 alpha_4: pragmaticListener('blue', 4),51 alpha_10: pragmaticListener('blue', 10)52};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 1.1e-16 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Frank & Goodman pragmatic listener / pragmatic speaker / RSA model. The world has three objects represented as records {shape, color}: {shape: "square", color: "blue"}, {shape: "circle", color: "blue"}, {shape: "square", color: "green"}, drawn uniformly. Possible utterances: 'blue', 'green', 'square', 'circle'. Truth function: a color/shape utterance must match the corresponding attribute, otherwise the utterance is true.
Build the level-1 stack with alpha = 1:
- literalListener(utterance): Infer over meaningPrior conditioning on meaning(utterance, obj) being true.
- speaker(obj): Infer over utterances drawn uniformly with factor(alpha * literalListener(utterance).score(obj)).
- pragmaticListener(utterance) = L1: Infer over meaningPrior conditioning on observe(speaker(obj), utterance).
Build the level-2 stack with alpha = 1:
- speaker2(obj): Infer over utterances drawn uniformly with factor(alpha * pragmaticListener(utterance).score(obj)).
- listener3(utterance) = L2: Infer over meaningPrior conditioning on observe(speaker2(obj), utterance).
End your program with an object literal {L1, L2} where L1 = pragmaticListener('blue') and L2 = listener3('blue').
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var meaningPrior = function() {2 uniformDraw([3 {shape: "square", color: "blue"},4 {shape: "circle", color: "blue"},5 {shape: "square", color: "green"}6 ])7};89var utterances = ["blue","green","square","circle"];1011var meaning = function(utterance, obj){12 (utterance === "blue" || utterance === "green") ? utterance === obj.color :13 (utterance === "circle" || utterance === "square") ? utterance === obj.shape :14 true15};1617var alpha = 1;1819var literalListener = function(utterance){20 return Infer({model: function(){21 var obj = meaningPrior();22 condition(meaning(utterance, obj));23 return obj;24 }});25};2627var speaker = function(obj){28 return Infer({model: function(){29 var utterance = uniformDraw(utterances);30 factor(alpha * literalListener(utterance).score(obj));31 return utterance;32 }});33};3435var pragmaticListener = function(utterance){36 return Infer({model: function(){37 var obj = meaningPrior();38 observe(speaker(obj),utterance);39 return obj;40 }});41};4243var speaker2 = function(obj){44 return Infer({model: function(){45 var utterance = uniformDraw(utterances);46 factor(alpha * pragmaticListener(utterance).score(obj));47 return utterance;48 }});49};5051var listener3 = function(utterance){52 return Infer({model: function(){53 var obj = meaningPrior();54 observe(speaker2(obj),utterance);55 return obj;56 }});57};58var ANSWER = (({59 L1: pragmaticListener("blue"),60 L2: listener3("blue")61}));62
1var objects = [2 {shape: "square", color: "blue"},3 {shape: "circle", color: "blue"},4 {shape: "square", color: "green"}5];67var utterances = ['blue', 'green', 'square', 'circle'];89var meaning = function(utterance, obj) {10 if (utterance === 'blue' || utterance === 'green') {11 return obj.color === utterance;12 } else {13 return obj.shape === utterance;14 }15};1617var meaningPrior = function() {18 return uniformDraw(objects);19};2021var alpha = 1;2223var literalListener = mem(function(utterance) {24 return Infer({method: 'enumerate'}, function() {25 var obj = meaningPrior();26 condition(meaning(utterance, obj));27 return obj;28 });29});3031var speaker = mem(function(obj) {32 return Infer({method: 'enumerate'}, function() {33 var utterance = uniformDraw(utterances);34 factor(alpha * literalListener(utterance).score(obj));35 return utterance;36 });37});3839var pragmaticListener = mem(function(utterance) {40 return Infer({method: 'enumerate'}, function() {41 var obj = meaningPrior();42 observe(speaker(obj), utterance);43 return obj;44 });45});4647var speaker2 = mem(function(obj) {48 return Infer({method: 'enumerate'}, function() {49 var utterance = uniformDraw(utterances);50 factor(alpha * pragmaticListener(utterance).score(obj));51 return utterance;52 });53});5455var listener3 = mem(function(utterance) {56 return Infer({method: 'enumerate'}, function() {57 var obj = meaningPrior();58 observe(speaker2(obj), utterance);59 return obj;60 });61});6263var L1 = pragmaticListener('blue');64var L2 = listener3('blue');6566var ANSWER = {L1: L1, L2: L2};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Heart-shaped implicit curve: a point (x, y) is *on the curve* if x^2 + (y - x^(2/3))^2 - 1 is within 0.01 of 0. The reference model draws x and y from independent Gaussians around the bounding box center, and conditions on onCurve(x, y).
Helpers (include them or equivalent in your program):
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};7var xbounds = [-1, 1];8var ybounds = [-1, 1.6];9var xmu = 0.5 * (xbounds[0] + xbounds[1]);10var ymu = 0.5 * (ybounds[0] + ybounds[1]);11var xsigma = 0.5 * (xbounds[1] - xbounds[0]);12var ysigma = 0.5 * (ybounds[1] - ybounds[0]);
Use Metropolis-Hastings MCMC instead of rejection sampling on the independent-Gaussians model var x = gaussian(xmu, xsigma); var y = gaussian(ymu, ysigma); condition(onCurve(x, y)). End your program with Infer({method: 'MCMC', samples: 10000, lag: 10}, model) returning the posterior over {x, y}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};7var xbounds = [-1, 1];8var ybounds = [-1, 1.6];910var xmu = 0.5 * (xbounds[0] + xbounds[1]);11var ymu = 0.5 * (ybounds[0] + ybounds[1]);12var xsigma = 0.5 * (xbounds[1] - xbounds[0]);13var ysigma = 0.5 * (ybounds[1] - ybounds[0]);1415var model = function() {16 var x = gaussian(xmu, xsigma);17 var y = gaussian(ymu, ysigma);18 condition(onCurve(x, y));19 return {x: x, y: y};20};21var ANSWER = (Infer({method: 'MCMC',22 samples: 10000,23 lag: 10}, model));24
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};78var xbounds = [-1, 1];9var ybounds = [-1, 1.6];10var xmu = 0.5 * (xbounds[0] + xbounds[1]);11var ymu = 0.5 * (ybounds[0] + ybounds[1]);12var xsigma = 0.5 * (xbounds[1] - xbounds[0]);13var ysigma = 0.5 * (ybounds[1] - ybounds[0]);1415var model = function() {16 var x = gaussian(xmu, xsigma);17 var y = gaussian(ymu, ysigma);18 condition(onCurve(x, y));19 return {x: x, y: y};20};2122var ANSWER = Infer({method: 'MCMC', samples: 10000, lag: 10}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Heart-shaped implicit curve: a point (x, y) is *on the curve* if x^2 + (y - x^(2/3))^2 - 1 is within 0.01 of 0. The reference model draws x and y from independent Gaussians around the bounding box center, and conditions on onCurve(x, y).
Helpers (include them or equivalent in your program):
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};7var xbounds = [-1, 1];8var ybounds = [-1, 1.6];9var xmu = 0.5 * (xbounds[0] + xbounds[1]);10var ymu = 0.5 * (ybounds[0] + ybounds[1]);11var xsigma = 0.5 * (xbounds[1] - xbounds[0]);12var ysigma = 0.5 * (ybounds[1] - ybounds[0]);
Change the model to draw x and y *jointly* via a diagCovGaussian centered at (xmu, ymu) with diagonal covariance (xsigma, ysigma), so MH MCMC successfully traces the curve. Use T.get to extract the x and y components of the sample. End your program with Infer({method: 'MCMC', samples: 1000, lag: 100}, model) returning the posterior over {x, y}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};7var xbounds = [-1, 1];8var ybounds = [-1, 1.6];910var xmu = 0.5 * (xbounds[0] + xbounds[1]);11var ymu = 0.5 * (ybounds[0] + ybounds[1]);12var xsigma = 0.5 * (xbounds[1] - xbounds[0]);13var ysigma = 0.5 * (ybounds[1] - ybounds[0]);1415var model = function() {16 var xy = diagCovGaussian({mu: Vector([xmu, ymu]),17 sigma: Vector([xsigma, ysigma])});18 var x = T.get(xy, 0);19 var y = T.get(xy, 1);20 condition(onCurve(x, y));21 return {x: x, y: y};22};23var ANSWER = (Infer({method: 'MCMC',24 samples: 1000,25 lag: 100}, model));26
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};78var xbounds = [-1, 1];9var ybounds = [-1, 1.6];10var xmu = 0.5 * (xbounds[0] + xbounds[1]);11var ymu = 0.5 * (ybounds[0] + ybounds[1]);12var xsigma = 0.5 * (xbounds[1] - xbounds[0]);13var ysigma = 0.5 * (ybounds[1] - ybounds[0]);1415var model = function() {16 var mu = Vector([xmu, ymu]);17 var sigma = Vector([xsigma, ysigma]);18 var xy = sample(DiagCovGaussian({mu: mu, sigma: sigma}));19 var x = T.get(xy, 0);20 var y = T.get(xy, 1);21 condition(onCurve(x, y));22 return {x: x, y: y};23};2425var ANSWER = Infer({method: 'MCMC', samples: 1000, lag: 100}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◔TV<1 | 1.000 | 16.7784 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Heart-shaped implicit curve: a point (x, y) is *on the curve* if x^2 + (y - x^(2/3))^2 - 1 is within 0.01 of 0. The reference model draws x and y from independent Gaussians around the bounding box center, and conditions on onCurve(x, y).
Helpers (include them or equivalent in your program):
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};7var xbounds = [-1, 1];8var ybounds = [-1, 1.6];9var xmu = 0.5 * (xbounds[0] + xbounds[1]);10var ymu = 0.5 * (ybounds[0] + ybounds[1]);11var xsigma = 0.5 * (xbounds[1] - xbounds[0]);12var ysigma = 0.5 * (ybounds[1] - ybounds[0]);
Using the original independent-Gaussians model, use HMC instead of MH. End your program with Infer({method: 'MCMC', samples: 10000, kernel: {HMC: {steps: 10, stepSize: .5}}}, model) returning the posterior over {x, y}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};7var xbounds = [-1, 1];8var ybounds = [-1, 1.6];910var xmu = 0.5 * (xbounds[0] + xbounds[1]);11var ymu = 0.5 * (ybounds[0] + ybounds[1]);12var xsigma = 0.5 * (xbounds[1] - xbounds[0]);13var ysigma = 0.5 * (ybounds[1] - ybounds[0]);1415var model = function() {16 var x = gaussian(xmu, xsigma);17 var y = gaussian(ymu, ysigma);18 condition(onCurve(x, y));19 return {x: x, y: y};20};21var ANSWER = (Infer({method: 'MCMC',22 samples: 10000,23 kernel: {HMC : { steps: 10, stepSize: .5 }} }, model));24
1var onCurve = function(x, y) {2 var x2 = x*x;3 var term1 = y - Math.pow(x2, 1/3);4 var crossSection = x2 + term1*term1 - 1;5 return Math.abs(crossSection) < 0.01;6};7var xbounds = [-1, 1];8var ybounds = [-1, 1.6];9var xmu = 0.5 * (xbounds[0] + xbounds[1]);10var ymu = 0.5 * (ybounds[0] + ybounds[1]);11var xsigma = 0.5 * (xbounds[1] - xbounds[0]);12var ysigma = 0.5 * (ybounds[1] - ybounds[0]);1314var model = function() {15 var x = gaussian(xmu, xsigma);16 var y = gaussian(ymu, ysigma);17 condition(onCurve(x, y));18 return {x: x, y: y};19};2021var ANSWER = Infer({method: 'MCMC', samples: 10000, kernel: {HMC: {steps: 10, stepSize: .5}}}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Two-endpoint interpolation: point1 = -10 is fixed; point2 is uniform on [-100, 100]; interpolationWeight is uniform on [0, 1]; pointInMiddle = point1 * interpolationWeight + point2 * (1 - interpolationWeight). We observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle).
Helper:
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};
Run MCMC with samples: 5000, lag: 100. End your program with an object literal {point2, interpolationWeight} where each value is the marginal distribution of the corresponding latent (use marginalize).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var interpolate = function(point1, point2, interpolationWeight) {2 return (point1 * interpolationWeight +3 point2 * (1 - interpolationWeight));4};56var model = function(){7 var point1 = -10;8 var point2 = uniform(-100, 100);9 var interpolationWeight = uniform(0, 1);10 var pointInMiddle = interpolate(point1, point2, interpolationWeight);11 observe(Gaussian({mu: 0, sigma:0.1}), pointInMiddle);12 return {point2, interpolationWeight, pointInMiddle};13};1415var posterior = Infer({method: 'MCMC', samples: 5000, lag: 100}, model);16var ANSWER = (({17 point2: marginalize(posterior, function(x) {return x.point2}),18 interpolationWeight: marginalize(posterior, function(x) {return x.interpolationWeight})19}));20
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};45var model = function() {6 var point1 = -10;7 var point2 = uniform(-100, 100);8 var interpolationWeight = uniform(0, 1);9 var pointInMiddle = interpolate(point1, point2, interpolationWeight);10 observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle);11 return {point2: point2, interpolationWeight: interpolationWeight};12};1314var posterior = Infer({method: 'MCMC', samples: 5000, lag: 100}, model);1516var ANSWER = {17 point2: marginalize(posterior, function(x) { return x.point2; }),18 interpolationWeight: marginalize(posterior, function(x) { return x.interpolationWeight; })19};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Two-endpoint interpolation: point1 = -10 is fixed; point2 is uniform on [-100, 100]; interpolationWeight is uniform on [0, 1]; pointInMiddle = point1 * interpolationWeight + point2 * (1 - interpolationWeight). We observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle).
Helper:
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};
Run MCMC with samples: 5000, lag: 100. End your program with the joint marginal distribution over (point2, interpolationWeight) as a Distribution (use marginalize returning an object {point2, inter} for each sample).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var interpolate = function(point1, point2, interpolationWeight) {2 return (point1 * interpolationWeight +3 point2 * (1 - interpolationWeight));4};56var model = function(){7 var point1 = -10;8 var point2 = uniform(-100, 100);9 var interpolationWeight = uniform(0, 1);10 var pointInMiddle = interpolate(point1, point2, interpolationWeight);11 observe(Gaussian({mu: 0, sigma:0.1}), pointInMiddle);12 return {point2, interpolationWeight, pointInMiddle};13};1415var posterior = Infer({method: 'MCMC', samples: 5000, lag: 100}, model);16var ANSWER = (marginalize(posterior, function(x) {17 return {'point2': x.point2, 'inter': x.interpolationWeight};18}));19
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};45var model = function() {6 var point1 = -10;7 var point2 = uniform(-100, 100);8 var interpolationWeight = uniform(0, 1);9 var pointInMiddle = interpolate(point1, point2, interpolationWeight);10 observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle);11 return {point2: point2, inter: interpolationWeight};12};1314var dist = Infer({method: 'MCMC', samples: 5000, lag: 100}, model);1516var ANSWER = marginalize(dist, function(x) { return {point2: x.point2, inter: x.inter}; });
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◉TV<.05 | 4.0e-15 | -0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◉TV<.05 | 4.0e-15 | -0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◉TV<.05 | 4.0e-15 | -0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◉TV<.05 | 4.0e-15 | -0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Two-endpoint interpolation: point1 = -10 is fixed; point2 is uniform on [-100, 100]; interpolationWeight is uniform on [0, 1]; pointInMiddle = point1 * interpolationWeight + point2 * (1 - interpolationWeight). We observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle).
Helper:
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};
Run MCMC with samples: 100, lag: 0. From posterior.samples, extract the array of pointInMiddle values in order. End your program with that array of length 100.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var interpolate = function(point1, point2, interpolationWeight) {2 return (point1 * interpolationWeight +3 point2 * (1 - interpolationWeight));4};56var model = function(){7 var point1 = -10;8 var point2 = uniform(-100, 100);9 var interpolationWeight = uniform(0, 1);10 var pointInMiddle = interpolate(point1, point2, interpolationWeight);11 observe(Gaussian({mu: 0, sigma:0.1}), pointInMiddle);12 return {point2, interpolationWeight, pointInMiddle};13};1415var posterior = Infer({method: 'MCMC', samples: 100, lag: 0}, model);16var ANSWER = (map(function(d) { d["value"]["pointInMiddle"] }, posterior.samples));17
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};45var point1 = -10;67var model = function() {8 var point2 = uniform(-100, 100);9 var interpolationWeight = uniform(0, 1);10 var pointInMiddle = interpolate(point1, point2, interpolationWeight);11 observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle);12 return pointInMiddle;13};1415var posterior = Infer({method: 'MCMC', samples: 100, lag: 0}, model);1617var ANSWER = map(function(s) { return s.value; }, posterior.samples);
[-15.6980, -13.4220, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549]
[-15.6980, -13.4220, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -6.1144, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.7865, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.2616, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549, -1.1549]
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ✗val- | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ✗val- | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ✗val- | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ✗val- | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ✓val+ | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ✓val+ | — | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ✓val+ | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ✓val+ | — | — | · |
Two-endpoint interpolation: point1 = -10 is fixed; point2 is uniform on [-100, 100]; interpolationWeight is uniform on [0, 1]; pointInMiddle = point1 * interpolationWeight + point2 * (1 - interpolationWeight). We observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle).
Helper:
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};
Rewrite this as rejection sampling. Convert the observe into a condition(Math.abs(pointInMiddle) < 0.01). End your program with Infer({method: 'rejection', samples: 1000}, model) returning the posterior over {point2, interpolationWeight, pointInMiddle}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var interpolate = function(point1, point2, interpolationWeight) {2 return (point1 * interpolationWeight +3 point2 * (1 - interpolationWeight));4};56var model = function(){7 var point1 = -10;8 var point2 = uniform(-100, 100);9 var interpolationWeight = uniform(0, 1);10 var pointInMiddle = interpolate(point1, point2, interpolationWeight);11 condition(Math.abs(pointInMiddle) < 0.01);12 return {point2, interpolationWeight, pointInMiddle};13};14var ANSWER = (Infer({method: 'rejection', samples: 1000}, model));15
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};45var model = function() {6 var point1 = -10;7 var point2 = uniform(-100, 100);8 var interpolationWeight = uniform(0, 1);9 var pointInMiddle = interpolate(point1, point2, interpolationWeight);10 condition(Math.abs(pointInMiddle) < 0.01);11 return { point2: point2, interpolationWeight: interpolationWeight, pointInMiddle: pointInMiddle };12};1314var ANSWER = Infer({method: 'rejection', samples: 1000}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Two-endpoint interpolation: point1 = -10 is fixed; point2 is uniform on [-100, 100]; interpolationWeight is uniform on [0, 1]; pointInMiddle = point1 * interpolationWeight + point2 * (1 - interpolationWeight). We observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle).
Helper:
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};
Replace point2's prior with a drift kernel: uniformDrift({a: -100, b: 100, width: 0.1}). End your program with Infer({method: 'MCMC', samples: 500}, model) returning the posterior over {point2, interpolationWeight, pointInMiddle}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var interpolate = function(point1, point2, interpolationWeight) {2 return (point1 * interpolationWeight +3 point2 * (1 - interpolationWeight));4};56var model = function(){7 var point1 = -10;8 var point2 = uniformDrift({a: -100, b: 100, width: .1});9 var interpolationWeight = uniform(0, 1);10 var pointInMiddle = interpolate(point1, point2, interpolationWeight);11 observe(Gaussian({mu: 0, sigma:0.1}), pointInMiddle);12 return {point2, interpolationWeight, pointInMiddle};13};14var ANSWER = (Infer({method: 'MCMC', samples: 500}, model));15
1var interpolate = function(point1, point2, interpolationWeight) {2 return point1 * interpolationWeight + point2 * (1 - interpolationWeight);3};45var model = function() {6 var point1 = -10;7 var point2 = uniformDrift({a: -100, b: 100, width: 0.1});8 var interpolationWeight = uniform(0, 1);9 var pointInMiddle = interpolate(point1, point2, interpolationWeight);10 observe(Gaussian({mu: 0, sigma: 0.1}), pointInMiddle);11 return {point2: point2, interpolationWeight: interpolationWeight, pointInMiddle: pointInMiddle};12};1314var ANSWER = Infer({method: 'MCMC', samples: 500}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ⚠fail | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ⚠fail | — | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Implement a topic model with two latent topics over the vocabulary ['DNA', 'evolution', 'parsing', 'phonology']. For each word in each document, a topic is drawn from a per-document topic distribution, then a word is observed under that topic's distribution. Topic distributions over the vocabulary are drawn from Dirichlet(eta = ones); per-document topic distributions are drawn from Dirichlet(alpha = ones).
1var vocabulary = ['DNA', 'evolution', 'parsing', 'phonology'];2var eta = ones([vocabulary.length, 1])3var numTopics = 24var alpha = ones([numTopics, 1])56var corpus = /* 6 short documents */;78Infer({method: 'MCMC', samples: 200, lag: 50}, function() {9 var topics = repeat(numTopics, function() {10 return T.toScalars(dirichlet({alpha: eta}))11 })12 mapData({data: corpus}, function(doc) {13 // your code: per-doc topic dist, then per-word topic + observe14 })15 return topics16});
End with the Infer(...) returning the joint posterior over topic distributions.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var vocabulary = ['DNA', 'evolution', 'parsing', 'phonology'];2var eta = ones([vocabulary.length, 1]);34var numTopics = 2;5var alpha = ones([numTopics, 1]);67var corpus = [8 'DNA evolution DNA evolution DNA evolution DNA evolution DNA evolution'.split(' '),9 'DNA evolution DNA evolution DNA evolution DNA evolution DNA evolution'.split(' '),10 'DNA evolution DNA evolution DNA evolution DNA evolution DNA evolution'.split(' '),11 'parsing phonology parsing phonology parsing phonology parsing phonology parsing phonology'.split(' '),12 'parsing phonology parsing phonology parsing phonology parsing phonology parsing phonology'.split(' '),13 'parsing phonology parsing phonology parsing phonology parsing phonology parsing phonology'.split(' ')14];15var ANSWER = (Infer({method: 'MCMC', samples: 200, lag: 50}, function() {16 var topics = repeat(numTopics, function() {17 return T.toScalars(dirichlet({alpha: eta}));18 });1920 mapData({data: corpus}, function(doc) {21 var docTopicDist = dirichlet({alpha: alpha});22 mapData({data: doc}, function(word) {23 var z = discrete(docTopicDist);24 var topic = topics[z];25 observe(Categorical({vs: vocabulary, ps: topic}), word);26 });27 });28 return topics;29}));30
1var vocabulary = ['DNA', 'evolution', 'parsing', 'phonology'];2var eta = ones([vocabulary.length, 1]);3var numTopics = 2;4var alpha = ones([numTopics, 1]);56var corpus = [7 ['DNA', 'evolution', 'DNA', 'evolution'],8 ['DNA', 'DNA', 'evolution', 'evolution'],9 ['evolution', 'DNA', 'evolution', 'DNA'],10 ['parsing', 'phonology', 'parsing', 'phonology'],11 ['phonology', 'parsing', 'phonology', 'parsing'],12 ['parsing', 'parsing', 'phonology', 'phonology']13];1415var ANSWER = Infer({method: 'MCMC', samples: 200, lag: 50}, function() {16 var topics = repeat(numTopics, function() {17 return T.toScalars(dirichlet({alpha: eta}));18 });1920 mapData({data: corpus}, function(doc) {21 var topicDist = T.toScalars(dirichlet({alpha: alpha}));22 mapData({data: doc}, function(word) {23 var topicIdx = sample(Categorical({vs: _.range(numTopics), ps: topicDist}));24 var wordDist = topics[topicIdx];25 observe(Categorical({vs: vocabulary, ps: wordDist}), word);26 });27 });2829 return topics;30});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ⚠fail | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◔TV<1 | 0.995 | 18.1212 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ⚠fail | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◔TV<1 | 1.000 | 19.0535 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ◔TV<1 | 1.000 | 18.9206 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◔TV<1 | 1.000 | 19.0371 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ◔TV<1 | 1.000 | 18.9206 | · |
Verify by sampling 1000 times each that the following three programs have the same marginal distribution:
Program 1: flip() ? flip(.7) : flip(.1)
Program 2: flip(flip() ? .7 : .1)
Program 3: flip(.4)
Return an object literal {p1, p2, p3} where each value is an array of 1000 samples from the corresponding program (use repeat(1000, function() {...})).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (({2 p1: repeat(1000, function() { return flip() ? flip(.7) : flip(.1); }),3 p2: repeat(1000, function() { return flip(flip() ? .7 : .1); }),4 p3: repeat(1000, function() { return flip(.4); })5}));6
1var p1 = repeat(1000, function() { return flip() ? flip(.7) : flip(.1); });2var p2 = repeat(1000, function() { return flip(flip() ? .7 : .1); });3var p3 = repeat(1000, function() { return flip(.4); });45var ANSWER = {p1: p1, p2: p2, p3: p3};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | — | · |
Write a *new* WebPPL expression with the same marginal distribution as flip(.4) (i.e., true with probability 0.4) that looks structurally different from flip(.4), flip() ? flip(.7) : flip(.1), and flip(flip() ? .7 : .1). End with that expression - the harness will rerun your program multiple times to estimate its marginal.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (flip() ? false : flip(.8));2
1var ANSWER = sample(Bernoulli({p: 0.4}));
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | △shape! | — | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | △shape! | — | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | △shape! | — | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | △shape! | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | △shape! | — | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | △shape! | — | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◉TV<.05 | 0.010 | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | △shape! | — | — | · |
Given:
1var foo = function() { return flip(); };2[foo(), foo(), foo()];
Each call to foo() independently flips, so the list can have any combination of trues and falses. Modify the program using mem so that [foo(), foo(), foo()] is always either [true, true, true] or [false, false, false]. End with the list expression - the harness will rerun your program multiple times. Use a ; to terminate each top-level statement.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var foo = mem(function() { return flip(); });2var ANSWER = ([foo(), foo(), foo()]);3
1var foo = mem(function() { return flip(); });2var ANSWER = [foo(), foo(), foo()];
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ○TV=1 | 1.000 | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ○TV=1 | 1.000 | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ○TV=1 | 1.000 | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ⚠fail | — | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ○TV=1 | 1.000 | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ○TV=1 | 1.000 | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ○TV=1 | 1.000 | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ○TV=1 | 1.000 | — | · |
Given the memoized program:
1var foo = mem(function() { return flip(); });2[foo(), foo(), foo()];
Modify it so that the first two elements are always equal but the third can differ. Hint: pass an argument to foo that distinguishes the two calls you want to be the same. End with the list expression. Use a ; to terminate each top-level statement.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var foo = mem(function(x) { return flip(); });2var ANSWER = ([foo(0), foo(0), foo(1)]);3
1var foo = mem(function(id) { return flip(); });2var ANSWER = [foo('a'), foo('a'), foo('b')];
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ○TV=1 | 1.000 | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ○TV=1 | 1.000 | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ○TV=1 | 1.000 | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ○TV=1 | 1.000 | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ○TV=1 | 1.000 | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ○TV=1 | 1.000 | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ○TV=1 | 1.000 | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ○TV=1 | 1.000 | — | · |
Given a simple symptoms model:
1var allergies = flip(0.3)2var cold = flip(0.2)3var sneeze = cold || allergies4var fever = cold
End with Infer({method: 'forward', samples: 1000}, ...) returning the joint distribution over {sneeze, fever}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (Infer({method: "forward", samples: 1000}, function() {2 var allergies = flip(0.3);3 var cold = flip(0.2);45 var sneeze = cold || allergies;6 var fever = cold;78 return {"sneeze": sneeze, "fever": fever};9}));10
1var model = function() {2 var allergies = flip(0.3);3 var cold = flip(0.2);4 var sneeze = cold || allergies;5 var fever = cold;6 return {sneeze: sneeze, fever: fever};7};89var ANSWER = Infer({method: 'forward', samples: 1000}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Given a multi-patient symptoms model attempted via per-person functions:
1var allergies = function(person) { return flip(.3) }2var cold = function(person) { return flip(.2) }3var sneeze = function(person) { return cold(person) || allergies(person) }4[sneeze('bob'), sneeze('alice')]
Add fever, then end with Infer({method: 'forward', samples: 1000}, ...) returning the joint distribution over Bob's symptoms {sneeze, fever}. The raw program double-counts: sneeze('bob') flips a fresh cold, and so does fever('bob'). Fix this by using mem on cold and allergies so the same person has consistent disease state within a sample.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (Infer({method: "forward", samples: 1000}, function() {2 var allergies = mem(function(person) { return flip(.3); });3 var cold = mem(function(person) { return flip(.2); });45 var sneeze = function(person) { return cold(person) || allergies(person); };6 var fever = function(person) { return cold(person); };78 return {"sneeze": sneeze('bob'), "fever": fever('bob')};9}));10
1var allergies = mem(function(person) { return flip(0.3); });2var cold = mem(function(person) { return flip(0.2); });3var sneeze = function(person) { return cold(person) || allergies(person); };4var fever = function(person) { return cold(person); };56var model = function() {7 return { sneeze: sneeze('bob'), fever: fever('bob') };8};910var ANSWER = Infer({method: 'forward', samples: 1000}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◐TV<.5 | 0.109 | 0.0462 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◐TV<.5 | 0.109 | 0.0462 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◐TV<.5 | 0.079 | 1.0777 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◐TV<.5 | 0.128 | 0.0668 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Given:
1var makeCoin = function(weight) {2 return function() { return flip(weight) ? 'h' : 't' }3}4var bend = function(coin) {5 return function() {6 return coin() == 'h' ? makeCoin(.7)() : makeCoin(.1)()7 }8}9var fairCoin = makeCoin(.5)10var bentCoin = bend(fairCoin)
End with Infer({method: 'forward', samples: 10000}, bentCoin) returning the bent coin's distribution.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var makeCoin = function(weight) {2 return function() {3 return flip(weight) ? 'h' : 't';4 };5};6var bend = function(coin) {7 return function() {8 return coin() == 'h' ? makeCoin(.7)() : makeCoin(.1)();9 };10};1112var fairCoin = makeCoin(.5);13var bentCoin = bend(fairCoin);14var ANSWER = (Infer({method: 'forward', samples: 10000}, bentCoin));15
1var makeCoin = function(weight) {2 return function() { return flip(weight) ? 'h' : 't' }3}4var bend = function(coin) {5 return function() {6 return coin() == 'h' ? makeCoin(.7)() : makeCoin(.1)()7 }8}9var fairCoin = makeCoin(.5)10var bentCoin = bend(fairCoin)1112var ANSWER = Infer({method: 'forward', samples: 10000}, bentCoin);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
The recursive geometric distribution is defined as:
1var geometric = function() {2 return flip() ? 0 : 1 + geometric()3}
End with Infer({method: 'forward', samples: 10000}, geometric) returning its distribution.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var geometric = function() {2 return flip() ? 0 : 1 + geometric();3};4var ANSWER = (Infer({method: "forward", samples:10000}, geometric));5
1var geometric = function() {2 return flip() ? 0 : 1 + geometric();3};45var ANSWER = Infer({method: 'forward', samples: 10000}, geometric);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
Convert this joint probability table into a compact WebPPL program:
| A | B | P(A,B) | |---|---|--------| | F | F | 0.14 | | F | T | 0.06 | | T | F | 0.4 | | T | T | 0.4 |
Requirement: fix P(A) first, then define the probability of B as a function of A. Use flip(...) for both. End with the expression [a, b] - the harness will rerun your program multiple times. Use a ; to terminate each top-level statement.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var a = flip(0.8);2var b = flip(a ? 0.5 : 0.3);3var ANSWER = ([a, b]);4
1var a = flip(0.8);2var b = a ? flip(0.5) : flip(0.3);3var ANSWER = [a, b];
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ◔TV<1 | 1.000 | — | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ◔TV<1 | 1.000 | — | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ◔TV<1 | 1.000 | — | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ◔TV<1 | 1.000 | — | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ◐TV<.5 | 0.080 | — | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ○TV=1 | 1.000 | — | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ◐TV<.5 | 0.080 | — | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ○TV=1 | 1.000 | — | · |
Wrap the joint probability program for (A, B) - where P(A=T)=0.8 and P(B=T|A) is 0.5 if A=T else 0.3 - in Infer({method: 'forward', samples: 10000}, ...) to verify the joint distribution. End with the Infer(...) returning [a, b].
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var ANSWER = (Infer({method: "forward", samples: 10000}, function() {2 var a = flip(0.8);3 var b = flip(a ? 0.5 : 0.3);4 return [a, b];5}));6
1var model = function() {2 var a = flip(0.8);3 var b = flip(a ? 0.5 : 0.3);4 return [a, b];5};67var ANSWER = Infer({method: 'forward', samples: 10000}, model);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| haiku-45-noprimer-v3 | haiku-4.5 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-primer-v3 | haiku-4.5 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-noprimer-v3 | haiku-4.5 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| haiku-45-think-primer-v3 | haiku-4.5 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-noprimer-v3 | sonnet-4.6 no-primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-primer-v3 primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-noprimer-v3 | sonnet-4.6 no-primer thinking | ●TV=0 | 0.000 | 0.0000 | · |
| sonnet-46-think-primer-v3 | sonnet-4.6 +primer thinking | ●TV=0 | 0.000 | 0.0000 | · |