An RSA model combining vague adjectives with QUD uncertainty (Kao et al. hyperbole model + gradable adjectives vagueness model). A pragmatic listener hears "expensive" about a kettle and infers the price and whether the speaker has negative valence.
prices = [50, 500, 1000, 5000, 10000]. pricePrior: categorical({vs:prices, ps:[0.8070, 0.1070, 0.0434, 0.0223, 0.0203]}). valencePrior(state): lookup table {50:0.3173, 500:0.7920, 1000:0.8933, 5000:0.9524, 10000:0.9864}, returns flip(probs[state]). utterances = ["expensive", "notExpensive"]. utterancePrior: uniformDraw(utterances). thetaPrior: uniformDraw(prices). meaning(utterance, price, theta): expensive ? price>=theta : notExpensive ? price<=theta : true. cost(utterance): notExpensive->1, expensive->0. qudFns = {price: state->{price:state.price}, valence: state->{valence:state.valence}, priceValence: state->{price:state.price,valence:state.valence}}. qudPrior: categorical({vs:["price","valence","priceValence"], ps:[1,1,1]}). alpha = 1. literalListener(utterance, qud, theta): Infer, uniformDraw(prices), samples valence, fullState={price,valence}, qudAnswer=qudFns[qud](fullState), conditions on meaning(utterance,price,theta), returns qudAnswer. speaker(fullState, qud, theta): Infer, utterancePrior, factor(alpha*(L0.score(qudAnswer)-cost(utterance))), returns utterance. pragmaticListener(utterance): Infer, samples price from pricePrior, valence from valencePrior(price), qud from qudPrior, theta from thetaPrior, fullState={price,valence}, observes speaker(fullState,qud,theta), returns {price, valence}.
Compute pragmaticListener("expensive"). Return joint distribution over {price, valence}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// ADJECTIVES + QUD MODEL2// frankie + shane RSA project34// code adapted from the Kao et al. hyperbole model +5// gradable adjectives & vagueness resolution model678var utterances = ["expensive", "notExpensive"]910var utterancePrior = function() {11 return uniformDraw(utterances)12}1314var thetaPrior = function() {15 return uniformDraw(prices)16}1718// theta moderates interpretation of utterances19var meaning = function(utterance, price, theta) {20return utterance == "expensive" ? price >= theta :21 utterance == "notExpensive" ? price <= theta :22 true23}2425// more words = higher cost26var cost = function(utterance) {27 return utterance== 'notExpensive'? 1 :28 029};3031var prices = [32 50,33 500,34 1000,35 5000,36 1000037]3839var pricePrior = function() {40 return categorical({41 vs: prices,42 ps: [43 0.8070,44 0.1070,45 0.0434,46 0.0223,47 0.020348 ]49 })50}5152var valencePrior = function(state) {53 var probs = {54 50 : 0.3173,55 500 : 0.7920,56 1000 : 0.8933,57 5000 : 0.9524,58 10000 : 0.986459 }60 var tf = flip(probs[state])61 return tf62}6364var qudFns = {65 price : function(state) {return { price: state.price } },66 valence : function(state) {return { valence: state.valence } },67 priceValence : function(state) {68 return { price: state.price, valence: state.valence }69 }70 }7172var qudPrior = function() {73 categorical({74 vs: ["price", "valence", "priceValence"],75 ps: [1, 1, 1]76 })77}7879var literalListener = cache(function(utterance, qud, theta) {80 return Infer({model: function(){81 var price = uniformDraw(prices)82 var valence = valencePrior(price)83 var fullState = {price, valence}84 var qudFn = qudFns[qud]85 var qudAnswer = qudFn(fullState)86 condition( meaning(utterance, price, theta) )87 return qudAnswer88 }89 })})9091// speaker optimality92var alpha = 19394var speaker = cache(function(fullState, qud, theta) {95 return Infer({model: function(){96 var utterance = utterancePrior()97 var qudFn = qudFns[qud]98 var qudAnswer = qudFn(fullState)99 factor(alpha*(literalListener(utterance,qud,theta).score(qudAnswer)100 - cost(utterance)))101 return utterance102 }})103})104105var pragmaticListener = cache(function(utterance) {106 return Infer({model: function(){107 //////// priors ////////108 var price = pricePrior()109 var valence = valencePrior(price)110 var qud = qudPrior()111 var theta = thetaPrior()112 ////////////////////////113 var fullState = {price, valence}114 observe(speaker(fullState, qud, theta), utterance)115 return {price, valence}116117 }})118})119120var listenerPosterior1 = pragmaticListener("expensive")121var listenerPosterior2 = pragmaticListener('notExpensive')122123print('Pragmatic listener hears "expensive":')124viz(listenerPosterior1)125print('Pragmatic listener hears "not expensive":')126viz(listenerPosterior2)127128var ANSWER = (pragmaticListener("expensive"));129
1var prices = [50, 500, 1000, 5000, 10000];2var pricePrior = Categorical({vs: prices, ps: [0.8070, 0.1070, 0.0434, 0.0223, 0.0203]});34var valenceProbTable = {50: 0.3173, 500: 0.7920, 1000: 0.8933, 5000: 0.9524, 10000: 0.9864};5var valencePrior = function(state) {6 return flip(valenceProbTable[state]);7};89var utterances = ["expensive", "notExpensive"];10var utterancePrior = function() { return uniformDraw(utterances); };1112var thetaPrior = function() { return uniformDraw(prices); };1314var meaning = function(utterance, price, theta) {15 return utterance === "expensive" ? price >= theta :16 utterance === "notExpensive" ? price <= theta :17 true;18};1920var cost = function(utterance) {21 return utterance === "notExpensive" ? 1 : 0;22};2324var qudFns = {25 price: function(state) { return {price: state.price}; },26 valence: function(state) { return {valence: state.valence}; },27 priceValence: function(state) { return {price: state.price, valence: state.valence}; }28};2930var qudPrior = function() {31 return uniformDraw(["price", "valence", "priceValence"]);32};3334var alpha = 1;3536var literalListener = mem(function(utterance, qud, theta) {37 return Infer({method: 'enumerate'}, function() {38 var price = sample(pricePrior);39 var valence = valencePrior(price);40 var fullState = {price: price, valence: valence};41 var qudAnswer = qudFns[qud](fullState);42 condition(meaning(utterance, price, theta));43 return qudAnswer;44 });45});4647var speaker = mem(function(fullState, qud, theta) {48 return Infer({method: 'enumerate'}, function() {49 var utterance = utterancePrior();50 var L0 = literalListener(utterance, qud, theta);51 var qudAnswer = qudFns[qud](fullState);52 factor(alpha * (L0.score(qudAnswer) - cost(utterance)));53 return utterance;54 });55});5657var pragmaticListener = function(utterance) {58 return Infer({method: 'enumerate'}, function() {59 var price = sample(pricePrior);60 var valence = valencePrior(price);61 var qud = qudPrior();62 var theta = thetaPrior();63 var fullState = {price: price, valence: valence};64 var S1 = speaker(fullState, qud, theta);65 observe(S1, utterance);66 return {price: price, valence: valence};67 });68};6970var ANSWER = pragmaticListener("expensive");
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◐TV<.5 | 0.251 | 0.1640 | · |
An RSA model for teasing/insulting language combining non-literal language with antisocial utility. A pragmatic listener hears an utterance and infers the intelligence state of the action, the speaker's phi (epistemic vs. antisocial balance), goal, valence, and arousal.
states = [1,2,3,4] (1=very unintelligent, 4=very intelligent). utterances = ["", "dumb as rocks", "dumb", "f*cking idiot"]. uniformDraw over utterances. literalSemantics lookup table: "" -> [0.25,0.25,0.25,0.25], "dumb as rocks" -> [0.45,0.85,0.20,0.02], "dumb" -> [0.85,0.95,0.02,0.02], "f*cking idiot" -> [0.95,0.55,0.02,0.02]. meaning(utterance, state): flip(literalSemantics[utterance][state-1]). valencePrior(state): 1->flip(0.01)?"good":"bad", 2->flip(0.33)?"good":"bad", 3->flip(0.66)?"good":"bad", 4->flip(0.99)?"good":"bad". arousals = ["low","high"]. arousalPrior(state): 1->categorical([0.1,0.9],arousals), 2->categorical([0.7,0.3],arousals), 3->categorical([0.7,0.3],arousals), 4->categorical([0.1,0.9],arousals). goals = ["goalState","goalValence","goalArousal"]. goalPrior: categorical([1,1,1], goals). goalState(goal, state, valence, arousal): returns state/valence/arousal depending on goal. literalListener(utterance, goal): Infer over uniformDraw(states), conditions on meaning(utterance,state), returns {QUDanswer: goalState(...), state}. lambda = -1.25 (antisocial, negative). valueFunction(s) = lambda * s. alpha = 10. speaker1(state, phi, goal, valence, arousal): Infer over uniformDraw(utterances), epistemic utility = L0_qudAnswerMarginal.score(QUDanswer), antisocial utility = expectation(L0_stateMarginal, valueFunction), speakerUtility = phi*epistemic + (1-phi)*antisocial, factor(alpha*speakerUtility). pragmaticListener(utterance): Infer over uniformDraw(states), valencePrior, arousalPrior, goalPrior, phi=uniformDraw(_.range(0.05,0.95,0.05)), observes speaker1(state,phi,goal,valence,arousal), returns {state, phi, goal, valence, arousal}.
Compute pragmaticListener("dumb as rocks"). Return joint distribution over {state, phi, goal, valence, arousal}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// Level of intelligence2var states = [1,2,3,4]34// What can be said5var utterances = ["", "dumb as rocks", "dumb", "f*cking idiot"]67// Correspondence of utterances to states8var literalSemantics = {9 "" : [.25,.25,.25,.25],10 "dumb as rocks" : [0.45,0.85,0.20,.02],11 "dumb" : [.85,.95,.02,.02],12 "f*cking idiot" : [.95,.55,.02,.02]13}1415// Determine whether the utterance describes the state16// Flip a coin with the literalSemantics weight17// *state - 1 because of 0-indexing*18var meaning = function(utterance, state){19 return flip(literalSemantics[utterance][state - 1]);20}2122// Whether the speaker feels good or bad about the listener's action.23// Speakers’ attitudes toward the true state of the world (e.g., the valence of their affect), which is modeled simply as a binary positive/negative variable (representing whether or not the speaker is repulsed by the listener’s behavior).24var valencePrior = function(state) {25 state === 1 ? flip(0.01) ? "good" : "bad" :26 state === 2 ? flip(0.33) ? "good" : "bad" :27 state === 3 ? flip(0.66) ? "good" : "bad" :28 state === 4 ? flip(0.99) ? "good" : "bad" :29 true}3031// How amplified the speaker feels about the listener being dumb to different degrees.32var arousals = ["low", "high"]3334//How passionate/aroused the listener feels about how intelligent the person is35var arousalPrior = function(state) {36 state === 1 ? categorical([0.1, 0.9], arousals) :37 state === 2 ? categorical([0.7, 0.3], arousals) :38 state === 3 ? categorical([0.7, 0.3], arousals) :39 state === 4 ? categorical([0.1, 0.9], arousals) :40 true41}4243// A list of strings of QUD choices44var goals = ["goalState", "goalValence", "goalArousal"]4546// There are 3 possible goals with a flat prior47var goalPrior = function() {48 categorical([1, 1, 1], goals)49}5051// A speaker's goal is satisfied if the listener infers the correct52// and relevant information.53var goalState = function(goal, state, valence, arousal) {54 goal === "goalState" ? state :55 goal === "goalValence" ? valence :56 goal === "goalArousal" ? arousal :57 true58}5960// literal listener61var literalListener = function(utterance, goal) {62 Infer({model: function(){63 var state = uniformDraw(states)64 var valence = valencePrior(state)65 var arousal = arousalPrior(state)66 var m = meaning(utterance, state);67 condition(m);68 return {QUDanswer: goalState(goal,state,valence,arousal), state: state}69 }})70}7172// value function scales social utility by a parameter lambda73var lambda = -1.2574var valueFunction = function(s){75 return lambda * s76};7778var alpha = 1079var speaker1 = function(state, phi, goal, valence, arousal) {80 Infer({model: function(){81 var utterance = uniformDraw(utterances)82 var QUDanswer = goalState(goal,state,valence,arousal)83 var L0_posterior = literalListener(utterance,goal)84 var L0_stateMarginal = marginalize(L0_posterior,"state")85 var L0_qudAnswerMarginal = marginalize(L0_posterior,"QUDanswer")86 var utility = {87 epistemic: L0_qudAnswerMarginal.score(QUDanswer),88 antisocial: expectation(L0_stateMarginal, valueFunction)89 }90 var speakerUtility = phi * utility.epistemic +91 (1 - phi) * utility.antisocial92 factor(alpha * speakerUtility)93 return utterance94 }})95}9697//pragmatic listener98var pragmaticListener = function(utterance) {99 Infer({model: function(){100 var state = uniformDraw(states)101 var valence = valencePrior(state)102 var arousal = arousalPrior(state)103 var goal = goalPrior()104//0.05 corresponds to the starting point of phi while 0.95 is the endpoint.105//The last 0.05 corresponds to the interval length106 var phi = uniformDraw(_.range(0.05, 0.95, 0.05))107 var S1 = speaker1(state, phi, goal, valence, arousal)108 observe(S1, utterance)109 return { state, phi, goal, valence, arousal }110 }})111}112viz.marginals(pragmaticListener(""))113viz.marginals(pragmaticListener("dumb as rocks"))114viz.marginals(pragmaticListener("dumb"))115viz.marginals(pragmaticListener("f*cking idiot"))116117var ANSWER = (pragmaticListener("dumb as rocks"));118
1var states = [1, 2, 3, 4];2var utterances = ["", "dumb as rocks", "dumb", "f*cking idiot"];34var literalSemantics = {5 "": [0.25, 0.25, 0.25, 0.25],6 "dumb as rocks": [0.45, 0.85, 0.20, 0.02],7 "dumb": [0.85, 0.95, 0.02, 0.02],8 "f*cking idiot": [0.95, 0.55, 0.02, 0.02]9};1011var meaning = function(utterance, state) {12 return flip(literalSemantics[utterance][state - 1]);13};1415var valencePrior = function(state) {16 return state === 1 ? (flip(0.01) ? "good" : "bad") :17 state === 2 ? (flip(0.33) ? "good" : "bad") :18 state === 3 ? (flip(0.66) ? "good" : "bad") :19 (flip(0.99) ? "good" : "bad");20};2122var arousals = ["low", "high"];23var arousalPrior = function(state) {24 return state === 1 ? categorical({vs: arousals, ps: [0.1, 0.9]}) :25 state === 2 ? categorical({vs: arousals, ps: [0.7, 0.3]}) :26 state === 3 ? categorical({vs: arousals, ps: [0.7, 0.3]}) :27 categorical({vs: arousals, ps: [0.1, 0.9]});28};2930var goals = ["goalState", "goalValence", "goalArousal"];31var goalPrior = function() {32 return categorical({vs: goals, ps: [1, 1, 1]});33};3435var goalState = function(goal, state, valence, arousal) {36 return goal === "goalState" ? state :37 goal === "goalValence" ? valence :38 arousal;39};4041var literalListener = mem(function(utterance, goal) {42 return Infer({method: 'enumerate'}, function() {43 var state = uniformDraw(states);44 var valence = valencePrior(state);45 var arousal = arousalPrior(state);46 condition(meaning(utterance, state));47 var qudAnswer = goalState(goal, state, valence, arousal);48 return {QUDanswer: qudAnswer, state: state};49 });50});5152var lambda = -1.25;53var valueFunction = function(s) { return lambda * s; };54var alpha = 10;5556var speaker1 = mem(function(state, phi, goal, valence, arousal) {57 return Infer({method: 'enumerate'}, function() {58 var utterance = uniformDraw(utterances);59 var L0 = literalListener(utterance, goal);60 var qudAnswer = goalState(goal, state, valence, arousal);6162 // Epistemic utility: score of the QUD answer under L063 var epistemicUtility = L0.score({QUDanswer: qudAnswer, state: state});6465 // Antisocial utility: expectation of valueFunction over L0 state marginal66 var L0stateMarginal = Infer({method: 'enumerate'}, function() {67 var sample = sample(L0);68 return sample.state;69 });7071 var antisocialUtility = expectation(L0stateMarginal, valueFunction);7273 var speakerUtility = phi * epistemicUtility + (1 - phi) * antisocialUtility;74 factor(alpha * speakerUtility);75 return utterance;76 });77});7879var pragmaticListener = function(utterance) {80 return Infer({method: 'enumerate'}, function() {81 var state = uniformDraw(states);82 var valence = valencePrior(state);83 var arousal = arousalPrior(state);84 var goal = goalPrior();85 var phiVals = _.range(0.05, 0.95, 0.05);86 var phi = uniformDraw(phiVals);8788 var S1 = speaker1(state, phi, goal, valence, arousal);89 observe(S1, utterance);9091 return {state: state, phi: phi, goal: goal, valence: valence, arousal: arousal};92 });93};9495var ANSWER = pragmaticListener("dumb as rocks");
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◐TV<.5 | 0.446 | 1.1242 | · |
Implement the RSA adjectives model with QUD (question under discussion) for ice cream pricing.
Data:
1var icecream = {2 prices: [1, 4, 6, 10, 14, 18, 22, 30, 34, 38],3 probabilities: [0.01, 0.50, 0.85, 0.63, 0.35, 0.10, 0.04, 0.02, 0.02, 0.01]4};
Helpers: - marginalize(dist, key): Infer over sample(dist)[key] - statePrior(): categorical(icecream.probabilities, icecream.prices) - thetaPrior(): uniformDraw(icecream.prices) - alpha = 1 - utterances = ["expensive", "null", "cheap"] - cost = {"expensive": 1, "cheap": 2, "null": 0} - utterancePrior(): categorical(map(function(u){Math.exp(-cost[u])}, utterances), utterances) - meaning(utterance, price, theta): "expensive" => price >= theta; "cheap" => price <= theta; else true - QUDs = ["expensive?", "less than 15?", "what is the price?"] - QUDPrior(): categorical([0, 0, 1], QUDs) // always returns "what is the price?" - QUDFun(QUD, state): "expensive?" => state >= 15; "less than 15?" => state <= 15; else state - literalListener(utterance, theta, QUD): Infer over statePrior(), condition on meaning, return QUDFun(QUD, price) - speaker(price, theta, QUD): Infer over utterancePrior(), factor alpha * literalListener(utt,theta,QUD).score(QUDFun(QUD,price)), return utt - pragmaticListener(utterance): Infer — sample price, theta, QUD; factor speaker(price,theta,QUD).score(utterance); return {price, theta, qud: QUD}
Bind pragmaticListener to a function of that exact name.
Return the expected price from the pragmaticListener's distribution when hearing 'expensive': expectation(marginalize(pragmaticListener('expensive'), 'price'))
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var marginalize = function(dist, key){2 return Infer( {model: function(){3 return sample(dist)[key];4 }})5};6///78var icecream = {9 "prices": [1, 4, 6, 10, 14, 18, 22, 30, 34, 38],10"probabilities": [0.01, 0.50, 0.85, 0.63, 0.35, 0.10, 0.04, 0.02, 0.02, 0.01]11};1213var statePrior = function() {14 return categorical(icecream.probabilities, icecream.prices);15};1617var thetaPrior = function() {18 return uniformDraw(icecream.prices);19};2021var alpha = 1; // optimality parameter2223var utterances = ["expensive", "null", "cheap"];24var cost = {25 "expensive": 1,26 "cheap": 2,27 "null": 028};29var utterancePrior = function() {30 var uttProbs = map(function(u) {return Math.exp(-cost[u]) }, utterances);31 return categorical(uttProbs, utterances);32};3334var meaning = function(utterance, price, theta) {35 utterance == "expensive" ? price >= theta :36 utterance == "cheap" ? price <= theta :37 true38};39// QUDs40var QUDs = ["expensive?","less than 15?","what is the price?"]41var QUDPrior = function() {42 //uniformDraw(QUDs)43// categorical([1,10,1],QUDs)// this is equivalent to uniformDraw44categorical([0,0,1],QUDs)// this is your baseline version45}46var QUDFun = function(QUD,state) {47 QUD == "expensive?" ? state >= 15 :48 QUD == "less than 15?" ? state <= 15 :49 state;50};5152var literalListener = cache(function(utterance, theta, QUD) {53 return Infer({model:function() {54 var price = statePrior()55 var qPrice = QUDFun(QUD,price)56 condition(meaning(utterance, price, theta))57 return qPrice;58}})59});6061var speaker = cache(function(price, theta,QUD) {62 return Infer( {model: function() {63 var utterance = utterancePrior()64 var qPrice= QUDFun(QUD, price)65 factor( alpha * literalListener(utterance, theta, QUD).score(qPrice) );66 return utterance;67 }});68});6970var pragmaticListener = function(utterance) {71 return Infer({model: function() {72 var price = statePrior()73 var theta = thetaPrior()74 var QUD =QUDPrior()75 factor(speaker(price, theta, QUD).score(utterance));76 return { price: price, theta: theta, qud: QUD};77 }})78}79808182print ('cheap icecream')83var expensiveIcecream = pragmaticListener('expensive');84print(expectation(marginalize(expensiveIcecream, "price")))85viz.hist(marginalize(expensiveIcecream, "price"));86viz.hist(marginalize(expensiveIcecream, "theta"));87viz.hist(marginalize(expensiveIcecream, "qud"));8889var ANSWER = (expectation(marginalize(pragmaticListener('expensive'), 'price')));90
1var icecream = {2 prices: [1, 4, 6, 10, 14, 18, 22, 30, 34, 38],3 probabilities: [0.01, 0.50, 0.85, 0.63, 0.35, 0.10, 0.04, 0.02, 0.02, 0.01]4};56var marginalize = function(dist, key) {7 return Infer({method: 'enumerate'}, function() {8 return sample(dist)[key];9 });10};1112var statePrior = function() {13 return categorical(icecream.probabilities, icecream.prices);14};1516var thetaPrior = function() {17 return uniformDraw(icecream.prices);18};1920var alpha = 1;2122var utterances = ["expensive", "null", "cheap"];23var cost = {"expensive": 1, "cheap": 2, "null": 0};2425var utterancePrior = function() {26 return categorical(map(function(u) { return Math.exp(-cost[u]); }, utterances), utterances);27};2829var meaning = function(utterance, price, theta) {30 return utterance === "expensive" ? price >= theta :31 utterance === "cheap" ? price <= theta :32 true;33};3435var QUDs = ["expensive?", "less than 15?", "what is the price?"];3637var QUDPrior = function() {38 return categorical([0, 0, 1], QUDs);39};4041var QUDFun = function(QUD, state) {42 return QUD === "expensive?" ? state >= 15 :43 QUD === "less than 15?" ? state <= 15 :44 state;45};4647var literalListener = mem(function(utterance, theta, QUD) {48 return Infer({method: 'enumerate'}, function() {49 var price = statePrior();50 condition(meaning(utterance, price, theta));51 return QUDFun(QUD, price);52 });53});5455var speaker = mem(function(price, theta, QUD) {56 return Infer({method: 'enumerate'}, function() {57 var utt = utterancePrior();58 factor(alpha * literalListener(utt, theta, QUD).score(QUDFun(QUD, price)));59 return utt;60 });61});6263var pragmaticListener = function(utterance) {64 return Infer({method: 'enumerate'}, function() {65 var price = statePrior();66 var theta = thetaPrior();67 var QUD = QUDPrior();68 factor(speaker(price, theta, QUD).score(utterance));69 return {price: price, theta: theta, qud: QUD};70 });71};7273var ANSWER = expectation(marginalize(pragmaticListener('expensive'), 'price'));
12.9027
12.9027
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ✓val+ | — | — | · |
A Kao et al. (2014) Rational Speech Act model for metaphor comprehension. Someone says "John is a whale" or "John is a person". The model reasons about what features (large, graceful, majestic) the speaker intends to communicate.
Categories: ["whale", "person"]. categoriesPrior: categorical([0.01, 0.99], categories). Utterances: ["whale", "person"]. utterancePrior: categorical([1,1], utterances). featureSets: all 8 binary combinations of {large, graceful, majestic}. featureSetPrior for "whale": categorical([0.30592786494628, 0.138078454222818, 0.179114768847673, 0.13098781834847, 0.0947267162507846, 0.0531420411185539, 0.0601520520596695, 0.0378702842057509], featureSets). featureSetPrior for "person": categorical([0.11687632453038, 0.105787535267869, 0.11568145784997, 0.130847056136141, 0.15288225956497, 0.128098151176801, 0.114694702836614, 0.135132512637255], featureSets). goals: ["large", "graceful", "majestic"]. goalPrior: categorical([1,1,1], goals). alpha = 3. literalInterpretation(utterance, category): utterance === category. goalState(goal, featureSet): returns featureSet[goal]. literalListener(utterance, goal): Infer over uniformDraw(categories), conditions on literalInterpretation, returns goalState. speaker(large, graceful, majestic, goal): Infer, factors alpha * literalListener(utterance,goal).score(goalState(goal, {large,graceful,majestic})). pragmaticListener(utterance): Infer, samples category from categoriesPrior, featureSet from featureSetPrior(category), extracts large/graceful/majestic, samples goal, observes speaker(large,graceful,majestic,goal), returns {category, large, graceful, majestic}.
Compute pragmaticListener("whale"). Return the full joint distribution over {category, large, graceful, majestic}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// John could either be a whale or a person.2var categories = ["whale", "person"]34// It is extremely unlikely that John is actually a whale.5var categoriesPrior = function() {6 categorical([0.01, 0.99], categories)7}89// The speaker could either say "John is a whale" or "John is a person."10var utterances = ["whale", "person"]1112// The utterances are equally costly.13var utterancePrior = function() {14 categorical([1,1], utterances)15}1617// The features of John being considered are "large", "graceful",18// "majestic." Features are binary.19var featureSets = [20 {large : 1, graceful : 1, majestic : 1},21 {large : 1, graceful : 1, majestic : 0},22 {large : 1, graceful : 0, majestic : 1},23 {large : 1, graceful : 0, majestic : 0},24 {large : 0, graceful : 1, majestic : 1},25 {large : 0, graceful : 1, majestic : 0},26 {large : 0, graceful : 0, majestic : 1},27 {large : 0, graceful : 0, majestic : 0}28]2930// information about feature priors (probabilistic world knowledge)31// obtained by an experimental study (see paper)32var featureSetPrior = function(category) {33 category === "whale" ? categorical([0.30592786494628, 0.138078454222818,34 0.179114768847673, 0.13098781834847,35 0.0947267162507846, 0.0531420411185539,36 0.0601520520596695, 0.0378702842057509],37 featureSets) :38 category === "person" ? categorical([0.11687632453038, 0.105787535267869,39 0.11568145784997, 0.130847056136141,40 0.15288225956497, 0.128098151176801,41 0.114694702836614, 0.135132512637255],42 featureSets) :43 true44}4546// Speaker's possible goals are to communicate feature 1, 2, or 347var goals = ["large", "graceful", "majestic"]4849// Prior probability of speaker's goal is set to uniform but can50// change with context/QUD.51var goalPrior = function() {52 categorical([1,1,1], goals)53}5455// Speaker optimality parameter56var alpha = 35758// Check if interpreted category is identical to utterance59var literalInterpretation = function(utterance, category) {60 utterance === category61}6263// Check if goal is satisfied64var goalState = function(goal, featureSet) {65 goal === "large" ? featureSet.large :66 goal === "graceful" ? featureSet.graceful :67 goal === "majestic" ? featureSet.majestic :68 true69}7071// Define a literal listener72var literalListener = function(utterance, goal) {73 Infer({model: function() {74 var category = uniformDraw(categories)75 var featureSet = featureSetPrior(category)76 condition(literalInterpretation(utterance, category))77 return goalState(goal, featureSet)78 }})79}8081// Speaker model82var speaker = function(large, graceful, majestic, goal) {83 Infer({model: function() {84 var utterance = utterancePrior()85 factor(alpha *86 literalListener(utterance,goal).score(goalState(goal, {large : large, graceful : graceful, majestic : majestic})))87 return utterance88 }})89}9091// Define a pragmatic listener92var pragmaticListener = function(utterance) {93 Infer({model: function() {94 var category = categoriesPrior()95 var featureSet = featureSetPrior(category)96 var large = featureSet.large97 var graceful = featureSet.graceful98 var majestic = featureSet.majestic99 var goal = goalPrior()100 observe(speaker(large, graceful, majestic, goal), utterance)101 return {category, large, graceful, majestic}102 }})103}104105display("The pragmatic listener's interpretation when the speaker says whale")106viz.table(pragmaticListener("whale"))107108display("The pragmatic listener's interpretation when the speaker says person")109viz.table(pragmaticListener("person"))110111var ANSWER = (pragmaticListener("whale"));112
Error: sample() expected a distribution but received "{"large":false,"graceful":false,"majestic":false}".1var categories = ["whale", "person"];2var utterances = ["whale", "person"];3var alpha = 3;45var featureSets = [6 {large: true, graceful: true, majestic: true},7 {large: true, graceful: true, majestic: false},8 {large: true, graceful: false, majestic: true},9 {large: true, graceful: false, majestic: false},10 {large: false, graceful: true, majestic: true},11 {large: false, graceful: true, majestic: false},12 {large: false, graceful: false, majestic: true},13 {large: false, graceful: false, majestic: false}14];1516var whalePriorPs = [0.30592786494628, 0.138078454222818, 0.179114768847673, 0.13098781834847, 0.0947267162507846, 0.0531420411185539, 0.0601520520596695, 0.0378702842057509];17var personPriorPs = [0.11687632453038, 0.105787535267869, 0.11568145784997, 0.130847056136141, 0.15288225956497, 0.128098151176801, 0.114694702836614, 0.135132512637255];1819var goals = ["large", "graceful", "majestic"];2021var literalInterpretation = function(utterance, category) {22 return utterance === category;23};2425var goalState = function(goal, featureSet) {26 return featureSet[goal];27};2829var literalListener = mem(function(utterance, goal) {30 return Infer({method: 'enumerate'}, function() {31 var category = categorical({vs: categories, ps: [0.01, 0.99]});32 condition(literalInterpretation(utterance, category));33 return goalState(goal, sample(categorical({vs: featureSets, ps: category === "whale" ? whalePriorPs : personPriorPs})));34 });35});3637var speaker = mem(function(large, graceful, majestic, goal) {38 return Infer({method: 'enumerate'}, function() {39 var utterance = categorical({vs: utterances, ps: [1, 1]});40 var featureSet = {large: large, graceful: graceful, majestic: majestic};41 factor(alpha * literalListener(utterance, goal).score(goalState(goal, featureSet)));42 return utterance;43 });44});4546var pragmaticListener = function(utterance) {47 return Infer({method: 'enumerate'}, function() {48 var category = categorical({vs: categories, ps: [0.01, 0.99]});49 var featureSet = sample(categorical({vs: featureSets, ps: category === "whale" ? whalePriorPs : personPriorPs}));50 var large = featureSet.large;51 var graceful = featureSet.graceful;52 var majestic = featureSet.majestic;53 var goal = categorical({vs: goals, ps: [1, 1, 1]});54 observe(speaker(large, graceful, majestic, goal), utterance);55 return {category: category, large: large, graceful: graceful, majestic: majestic};56 });57};5859var ANSWER = pragmaticListener("whale");
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ⚠fail | — | — | · |
Implement a simple RSA model for 'Black lives matter' utterance interpretation in WebPPL. The state space is four {black,white} boolean pairs: [{black:true,white:true},{black:true,white:false},{black:false,white:true},{black:false,white:false}] with uniform prior (ps:[.25,.25,.25,.25]). The state prior is a Categorical over these four states. Utterances are uniformDraw(['blm','nblm']). Literal meanings: blm is true when state.black==true; nblm is true when state.black==false. (wlm, alm, nlm are defined but not used in this model — utterancePrior only draws from ['blm','nblm'].) Speaker optimality alpha=1. The literal listener takes utt, samples from statePrior(), conditions on literalMeanings[utt](state), and returns state. The pragmatic speaker takes state, draws utt from utterancePrior, factors by alpha*literalListener(utt).score(state). The pragmatic listener takes utt, samples from statePrior(), observes speaker(state) with utt. Use Infer({method:'enumerate'}) for all. Return the pragmatic listener's distribution when hearing 'blm'.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var alpha = 123var statePrior = function() {4 return Categorical({5 // ps: [.9,.04,.04,.02], // make it a priori likely that all lives matter6 ps: [.25,.25,.25,.25], // uniform prior7 // ps: [.04,.04,.9,.02], // make it a priori likely that only white lives matter8 vs: [{black:true, white:true},{black:true, white:false},{black:false, white:true},{black:false, white:false}]})9};1011// possible utterances12var utterancePrior = function() {13 return uniformDraw(['blm', 'nblm'])14// return uniformDraw(['blm', 'wlm', 'alm', 'nlm'])15};1617// meaning funtion to interpret the utterances18var literalMeanings = {19 blm: function(state) { return state["black"] },20 nblm: function(state) { return !state["black"] },21 wlm: function(state) { return state["white"]},22 alm: function(state) { return state["black"] && state["white"] },23 nlm: function(state) { return !state["black"] && !state["white"] }24};2526// literal listener27var literalListener = cache(function(utt) {28 return Infer({method:"enumerate"},29 function(){30 var state = sample(statePrior())31 var meaning = literalMeanings[utt]32 condition(meaning(state))33 return state34 })35});3637// pragmatic speaker38var speaker = cache(function(state) {39 return Infer({method:"enumerate"},40 function(){41 var utt = utterancePrior()42 factor(alpha * literalListener(utt).score(state))43 return utt44 })45});4647// pragmatic listener48var pragmaticListener = cache(function(utt) {49 return Infer({method:"enumerate"},50 function(){51 var state = sample(statePrior())52 observe(speaker(state),utt)53 return state54 })55});5657pragmaticListener("blm")5859var ANSWER = (pragmaticListener('blm'));60
1var states = [2 {black: true, white: true},3 {black: true, white: false},4 {black: false, white: true},5 {black: false, white: false}6];78var statePrior = function() {9 return sample(Categorical({vs: states, ps: [.25, .25, .25, .25]}));10};1112var utterances = ['blm', 'nblm'];1314var utterancePrior = function() {15 return uniformDraw(utterances);16};1718var literalMeanings = {19 blm: function(state) { return state.black == true; },20 nblm: function(state) { return state.black == false; },21 wlm: function(state) { return state.white == true; },22 alm: function(state) { return state.black == true || state.white == true; },23 nlm: function(state) { return state.black == false && state.white == false; }24};2526var alpha = 1;2728var literalListener = mem(function(utt) {29 return Infer({method: 'enumerate'}, function() {30 var state = statePrior();31 condition(literalMeanings[utt](state));32 return state;33 });34});3536var pragmaticSpeaker = mem(function(state) {37 return Infer({method: 'enumerate'}, function() {38 var utt = utterancePrior();39 factor(alpha * literalListener(utt).score(state));40 return utt;41 });42});4344var pragmaticListener = function(utt) {45 return Infer({method: 'enumerate'}, function() {46 var state = statePrior();47 observe(pragmaticSpeaker(state), utt);48 return state;49 });50};5152var ANSWER = pragmaticListener('blm');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◔TV<1 | 0.500 | 10.4732 | · |
A Kao et al. (2014) RSA metaphor model with a context-biased goal prior. Someone says "John is a whale". The speaker's goal prior is biased toward communicating that John is "large" (weight 5) over "graceful" or "majestic" (weight 1 each).
Categories: ["whale", "person"]. categoriesPrior: categorical([0.01, 0.99], categories). Utterances: ["whale", "person"]. utterancePrior: categorical([1,1], utterances). featureSets: all 8 binary combinations of {large, graceful, majestic}. featureSetPrior for "whale": categorical([0.30592786494628, 0.138078454222818, 0.179114768847673, 0.13098781834847, 0.0947267162507846, 0.0531420411185539, 0.0601520520596695, 0.0378702842057509], featureSets). featureSetPrior for "person": categorical([0.11687632453038, 0.105787535267869, 0.11568145784997, 0.130847056136141, 0.15288225956497, 0.128098151176801, 0.114694702836614, 0.135132512637255], featureSets). goals: ["large", "graceful", "majestic"]. goalPrior: categorical([5,1,1], goals) — biased toward "large". alpha = 3. literalInterpretation(utterance, category): utterance === category. goalState(goal, featureSet): returns featureSet[goal]. literalListener(utterance, goal): Infer over uniformDraw(categories), conditions on literalInterpretation, returns goalState. speaker(large, graceful, majestic, goal): Infer, factors alpha * literalListener(utterance,goal).score(goalState(goal, {large,graceful,majestic})). pragmaticListener(utterance): Infer, samples category from categoriesPrior, featureSet from featureSetPrior(category), extracts large/graceful/majestic, samples goal from goalPrior, observes speaker(large,graceful,majestic,goal), returns {category, large, graceful, majestic}.
Compute pragmaticListener("whale"). Return the full joint distribution over {category, large, graceful, majestic}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// John could either be a whale or a person.2var categories = ["whale", "person"]34// It is extremely unlikely that John is actually a whale.5var categoriesPrior = function() {6 categorical([0.01, 0.99], categories)7}89// The speaker could either say "John is a whale" or "John is a person."10var utterances = ["whale", "person"]1112// The utterances are equally costly.13var utterancePrior = function() {14 categorical([1,1], utterances)15}1617// The features of John being considered are "large", "graceful",18// "majestic." Features are binary.19var featureSets = [20 {large : 1, graceful : 1, majestic : 1},21 {large : 1, graceful : 1, majestic : 0},22 {large : 1, graceful : 0, majestic : 1},23 {large : 1, graceful : 0, majestic : 0},24 {large : 0, graceful : 1, majestic : 1},25 {large : 0, graceful : 1, majestic : 0},26 {large : 0, graceful : 0, majestic : 1},27 {large : 0, graceful : 0, majestic : 0}28]2930// information about feature priors (probabilistic world knowledge)31// obtained by an experimental study (see paper)32var featureSetPrior = function(category) {33 category === "whale" ? categorical([0.30592786494628, 0.138078454222818,34 0.179114768847673, 0.13098781834847,35 0.0947267162507846, 0.0531420411185539,36 0.0601520520596695, 0.0378702842057509],37 featureSets) :38 category === "person" ? categorical([0.11687632453038, 0.105787535267869,39 0.11568145784997, 0.130847056136141,40 0.15288225956497, 0.128098151176801,41 0.114694702836614, 0.135132512637255],42 featureSets) :43 true44}4546// Speaker's possible goals are to communicate feature 1, 2, or 347var goals = ["large", "graceful", "majestic"]4849//// Prior probability of speaker's goal is set to uniform but can50//// change with context/QUD.51var goalPrior = function() {52 categorical([5,1,1], goals)53}54// var goalPrior = function() {55// categorical([1,5,1], goals)56// }57// var goalPrior = function() {58// categorical([1,1,5], goals)59// }6061// Speaker optimality parameter62var alpha = 36364// Check if interpreted category is identical to utterance65var literalInterpretation = function(utterance, category) {66 utterance === category67}6869// Check if goal is satisfied70var goalState = function(goal, featureSet) {71 goal === "large" ? featureSet.large :72 goal === "graceful" ? featureSet.graceful :73 goal === "majestic" ? featureSet.majestic :74 true75}7677// Define a literal listener78var literalListener = function(utterance, goal) {79 Infer({model: function() {80 var category = uniformDraw(categories)81 var featureSet = featureSetPrior(category)82 condition(literalInterpretation(utterance, category))83 return goalState(goal, featureSet)84 }})85}8687// Speaker model88var speaker = function(large, graceful, majestic, goal) {89 Infer({model: function() {90 var utterance = utterancePrior()91 factor(alpha *92 literalListener(utterance,goal).score(goalState(goal, {large : large, graceful : graceful, majestic : majestic})))93 return utterance94 }})95}9697// Define a pragmatic listener98var pragmaticListener = function(utterance) {99 Infer({model: function() {100 var category = categoriesPrior()101 var featureSet = featureSetPrior(category)102 var large = featureSet.large103 var graceful = featureSet.graceful104 var majestic = featureSet.majestic105 var goal = goalPrior()106 observe(speaker(large, graceful, majestic, goal), utterance)107 return {category, large, graceful, majestic}108 }})109}110111viz.table(pragmaticListener("whale"))112113var ANSWER = (pragmaticListener("whale"));114
1var categories = ["whale", "person"];2var utterances = ["whale", "person"];34var featureSets = [5 {large: true, graceful: true, majestic: true},6 {large: true, graceful: true, majestic: false},7 {large: true, graceful: false, majestic: true},8 {large: true, graceful: false, majestic: false},9 {large: false, graceful: true, majestic: true},10 {large: false, graceful: true, majestic: false},11 {large: false, graceful: false, majestic: true},12 {large: false, graceful: false, majestic: false}13];1415var whalePriorPs = [0.30592786494628, 0.138078454222818, 0.179114768847673, 0.13098781834847, 0.0947267162507846, 0.0531420411185539, 0.0601520520596695, 0.0378702842057509];16var personPriorPs = [0.11687632453038, 0.105787535267869, 0.11568145784997, 0.130847056136141, 0.15288225956497, 0.128098151176801, 0.114694702836614, 0.135132512637255];1718var goals = ["large", "graceful", "majestic"];1920var alpha = 3;2122var literalInterpretation = function(utterance, category) {23 return utterance === category;24};2526var goalState = function(goal, featureSet) {27 return featureSet[goal];28};2930var literalListener = mem(function(utterance, goal) {31 return Infer({method: 'enumerate'}, function() {32 var category = categorical({vs: categories, ps: [0.01, 0.99]});33 condition(literalInterpretation(utterance, category));34 return goalState(goal, uniformDraw(featureSets));35 });36});3738var speaker = mem(function(large, graceful, majestic, goal) {39 return Infer({method: 'enumerate'}, function() {40 var utterance = categorical({vs: utterances, ps: [1, 1]});41 var featureSet = {large: large, graceful: graceful, majestic: majestic};42 var gs = goalState(goal, featureSet);43 factor(alpha * literalListener(utterance, goal).score(gs));44 return utterance;45 });46});4748var pragmaticListener = function(utterance) {49 return Infer({method: 'enumerate'}, function() {50 var category = categorical({vs: categories, ps: [0.01, 0.99]});51 var featureSetPs = category === "whale" ? whalePriorPs : personPriorPs;52 var featureSet = categorical({vs: featureSets, ps: featureSetPs});53 var large = featureSet.large;54 var graceful = featureSet.graceful;55 var majestic = featureSet.majestic;56 var goal = categorical({vs: goals, ps: [5, 1, 1]});57 observe(speaker(large, graceful, majestic, goal), utterance);58 return {category: category, large: large, graceful: graceful, majestic: majestic};59 });60};6162var ANSWER = pragmaticListener("whale");
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ○TV=1 | 1.000 | 20.8984 | · |
A Tessler et al. (2017) RSA comparison-class model for gradable adjectives. A pragmatic listener hears "tall" and knows the person is a basketball player. They infer whether "tall" is relative to all people (superordinate) or to basketball players (subordinate).
binParam = 3. superordinate_params = {mu: 0, sigma: 1}. stateVals: _.range(-3, 3, 1/3) (18 values). stateProbs(stateParams): map over stateVals, exp(Gaussian(stateParams).score(s)). generateStatePrior(stateParams): Infer categorical({vs: stateVals, ps: stateProbs(stateParams)}). thresholdBins: {positive: stateVals - 1/6, negative: stateVals + 1/6}. thresholdPrior(form): uniformDraw(thresholdBins[form]). subParams = {gymnasts: {mu:-1, sigma:0.5}, soccerPlayers: {mu:0, sigma:0.5}, basketballPlayers: {mu:1, sigma:0.5}}. utterances = ["tall", "short", "silence"]. meaning(utterance, state, thresholds): tall ? state > thresholds.tall : short ? state < thresholds.short : true. classPrior: uniformDraw(["subordinate", "superordinate"]). alpha = 5. literalListener(utterance, thresholds, comparisonClass): Infer, samples state from generateStatePrior(comparisonClass), conditions on meaning, returns state. cache 10000. speaker1(state, thresholds, comparisonClass): Infer, uniformDraw(utterances), factor(alpha * L0.score(state)), returns utterance. cache 10000. pragmaticListener(utterance, subordinate_params): Infer, samples state from generateStatePrior(subordinate_params), samples thresholds {tall: thresholdPrior("positive"), short: thresholdPrior("negative")}, samples c from classPrior, comparisonClass = c=="subordinate" ? subordinate_params : superordinate_params, observes speaker1, returns {comparisonClass: c, state}. cache 10000.
Compute marginalize(pragmaticListener("tall", subParams["basketballPlayers"]), "comparisonClass"). Return the marginal distribution over comparisonClass.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1///fold:2// helper function3var exp = function(x){return Math.exp(x)}45// for discretization6var binParam = 3;78// information about the superordinate category prior9// e.g., the height distribution for all people10var superordinate_params = {mu: 0, sigma: 1};1112// calculate the range in pre-defined steps;13// these values correspond to possible heights14var stateVals = _.range(superordinate_params.mu - 3 * superordinate_params.sigma,15 superordinate_params.mu + 3 * superordinate_params.sigma,16 superordinate_params.sigma/binParam)1718// for each possible height, calculate its probability of occurrence19var stateProbs = cache(function(stateParams){20 return map(function(s){21 exp(Gaussian(stateParams).score(s))22 }, stateVals)23});2425// generate a statePrior using the possible heights and their probabilities26var generateStatePrior = cache(function(stateParams) {27 return Infer({28 model: function(){29 return categorical({vs: stateVals, ps: stateProbs(stateParams)})30 }31 })32});3334// generate the uniform threshold prior35var thresholdBins ={36 positive: map(function(x){37 return x - (1/(binParam*2));38 }, sort(stateVals)),39 negative: map(function(x){40 return x + (1/(binParam*2));41 }, sort(stateVals))42};4344var thresholdPrior = cache(function(form){45 return Infer({46 model: function() { return uniformDraw(thresholdBins[form]) }47 });48});4950// information about the superordinate category priors51var subParams = {52 gymnasts: {mu: -1, sigma: 0.5}, // gymnast heights53 soccerPlayers: {mu: 0, sigma: 0.5}, // soccer player heights54 basketballPlayers: {mu: 1, sigma: 0.5} // basketball player heights55}5657// possible utterances can be either positive (tall) or negative (short) or a null utterance58var utterances = ["tall", "short", "silence"]5960// meaning function for utterances61var meaning = function(utterance, state, thresholds) {62 utterance == "tall" ? state > thresholds.tall :63 utterance == "short" ? state < thresholds.short :64 true65}6667// assume a uniform prior over comparison classes68var classPrior = Infer({69 model: function(){return uniformDraw(["subordinate", "superordinate"])}70});7172// set speaker optimality73var alpha = 5;7475var literalListener = cache(76 function(utterance, thresholds, comparisonClass) {77 Infer({model: function(){78 var StatePrior = generateStatePrior(comparisonClass)79 var state = sample(StatePrior);80 var m = meaning(utterance, state, thresholds);81 condition(m);82 return state;83 }})84 }, 10000 // limit cache size85)8687var speaker1 = cache(88 function(state, thresholds, comparisonClass) {89 Infer({model: function(){90 var utterance = uniformDraw(utterances);91 var L0 = literalListener(utterance, thresholds, comparisonClass);92 factor( alpha * L0.score(state) );93 return utterance;94 }})95 }, 10000 // limit cache size96)97///9899var pragmaticListener = cache(function(utterance, subordinate_params) {100 Infer({model: function(){101102 var statePrior = generateStatePrior(subordinate_params);103 var state = sample(statePrior);104 // separate thresholds for positive adjective and negative adjective105 var thresholds = {106 tall: sample(thresholdPrior("positive")),107 short: sample(thresholdPrior("negative"))108 }109110 // uncertainty about the comparison class (superordinate vs. subordinate)111 var c = sample(classPrior)112 var comparisonClass = c == "subordinate" ? subordinate_params : superordinate_params113114 var S1 = speaker1(state, thresholds, comparisonClass);115 observe(S1, utterance);116117 return { comparisonClass: c, state : state }118 }})119}, 10000 // limit cache size120 )121122var ANSWER = (marginalize(pragmaticListener("tall", subParams["basketballPlayers"]), "comparisonClass"));123
1var binParam = 3;2var superordinate_params = {mu: 0, sigma: 1};3var stateVals = _.range(-3, 3, 1/3);45var stateProbs = function(stateParams) {6 return map(function(s) {7 return Math.exp(Gaussian({mu: stateParams.mu, sigma: stateParams.sigma}).score(s));8 }, stateVals);9};1011var generateStatePrior = function(stateParams) {12 return Infer({method: 'enumerate'}, function() {13 return sample(Categorical({vs: stateVals, ps: stateProbs(stateParams)}));14 });15};1617var thresholdBins = {18 positive: map(function(s) { return s - 1/6; }, stateVals),19 negative: map(function(s) { return s + 1/6; }, stateVals)20};2122var thresholdPrior = function(form) {23 return uniformDraw(thresholdBins[form]);24};2526var subParams = {27 gymnasts: {mu: -1, sigma: 0.5},28 soccerPlayers: {mu: 0, sigma: 0.5},29 basketballPlayers: {mu: 1, sigma: 0.5}30};3132var utterances = ["tall", "short", "silence"];3334var meaning = function(utterance, state, thresholds) {35 return utterance === "tall" ? state > thresholds.tall :36 utterance === "short" ? state < thresholds.short :37 true;38};3940var alpha = 5;4142var literalListener = mem(function(utterance, thresholds, comparisonClass) {43 return Infer({method: 'enumerate'}, function() {44 var state = sample(generateStatePrior(comparisonClass));45 condition(meaning(utterance, state, thresholds));46 return state;47 });48}, 10000);4950var speaker1 = mem(function(state, thresholds, comparisonClass) {51 return Infer({method: 'enumerate'}, function() {52 var utterance = uniformDraw(utterances);53 var L0 = literalListener(utterance, thresholds, comparisonClass);54 factor(alpha * L0.score(state));55 return utterance;56 });57}, 10000);5859var pragmaticListener = mem(function(utterance, subordinate_params) {60 return Infer({method: 'enumerate'}, function() {61 var state = sample(generateStatePrior(subordinate_params));62 var thresholds = {63 tall: thresholdPrior("positive"),64 short: thresholdPrior("negative")65 };66 var c = uniformDraw(["subordinate", "superordinate"]);67 var comparisonClass = c === "subordinate" ? subordinate_params : superordinate_params;68 var S1 = speaker1(state, thresholds, comparisonClass);69 observe(S1, utterance);70 return {comparisonClass: c, state: state};71 });72}, 10000);7374var result = pragmaticListener("tall", subParams["basketballPlayers"]);75var marginal = marginalize(result, "comparisonClass");7677var ANSWER = marginal;
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◉TV<.05 | 5.6e-16 | -0.0000 | · |
Implement an RSA model for the Codenames word game using GloVe vector similarity as the meaning function.
Helper: the following word vectors (25-dimensional) are given:
1var vectors = {2 eagle: Vector([-0.8186906894583743, -0.8443627918594182, -0.04304780086785447, -0.8257634263841377, -0.7607218950809542, 0.47786735164930183, 0.36942709316422206, 0.18560148725224498, 0.38625176009619944, 0.24384273963053932, 1.0355862286322068, -0.14170089242313555, -0.17017960843359828, 0.27636172471279313, -0.49477465481497807, -1.199206930890509, 1.0531720839078256, -0.5154875303291531, 0.30704269353337016, 1.5382356443196483, -0.13215425501400774, 1.2222507503066664, 1.3819617662995949, -1.1579407453927437, 0.9439311306043343]),3 pig: Vector([-0.9027808771549458, -1.4539105978263833, 0.5743098399154295, 1.3052815987119957, -0.038556210348244704, -0.22144102997326148, 1.222050088622139, -0.027526643946408857, -0.13265827668708097, 1.4799207507145387, 0.02371336629548181, -0.9405402658175948, 0.06556493358788004, -1.6556208133885402, -0.44306373689318584, -0.475710035110901, 1.2435716830499404, -1.0677780309283533, -0.03344465447168945, 0.16184568683816827, 0.8718035460475897, 2.082956682688621, 0.47430271385843953, -0.4479993650378608, 1.5192928553678355]),4 chicken: Vector([-0.9555633717916903, -0.23467550895948608, 0.9081102168032618, 1.7681919864431317, -0.3888166871286516, 1.2292398003323308, 1.0624961440319318, -0.3558803892040966, -0.17024423658814317, 0.7046776782991592, 1.624196256505183, -1.1423231844008523, -0.9490267652945451, -1.8004114037674281, -0.026086280368055388, -2.089757256612839, 3.5660566372328693, -2.4178611093952225, -0.7077960662621875, 0.9418434965990246, 0.438927172322575, 1.0891725023940724, -0.1237861204326181, 0.7602054634506068, 3.0515580696224083]),5 farm: Vector([-1.3191469349030631, -0.34747873883058705, 0.09525267994894762, 0.08014872654330456, 0.1179814806966339, -0.26926061020753783, 0.709033965954239, -0.6521777385143812, 1.0195239553589313, 0.7192612109870958, 1.1711460976059695, -1.0779079866249233, -0.5443503049555966, 0.08523251153754875, 0.1455530206584687, -1.501097375488643, 1.1151234505440395, 0.0581591541412683, -0.1102242123027589, 0.5253857581277014, 0.21780949510893402, 0.026030837039037417, 0.07282095318396448, -0.6093002665622598, 2.0466066458317336]),6 animal: Vector([-0.08132595025854601, -1.8280616716238214, -0.4241550049238374, 1.3405833261217683, 1.3635302219051426, 0.19656106954281044, 1.0553637657141577, 0.8640316722860499, -0.34682275265131135, 0.27196141799987644, 0.9785603157742483, -3.1767493003780873, -0.7566904249011203, -1.1935303767007424, 0.2523177522167622, 0.33414675815038736, 0.21147820292953767, 0.2089073521353749, 0.36413859545070626, -0.3145854077725169, 0.8470589609352164, 0.8914477422324714, 0.06602846837066885, -1.1974184866543685, 1.6807019645814638]),7 bird: Vector([0.9317828273959833, -1.142927450658389, 1.1249556341704339, 0.7533022372085103, 0.039221572709652965, 0.5302815428039684, 1.1525754405638204, 0.5707370610821617, 0.01803607760778035, 0.9527229145321762, 1.0851468114908822, -0.4626041548552341, -0.5371489443168416, -0.8343285842461913, -0.09713481034287788, 0.8070233789520264, 0.21755780815430825, -0.6588132708557186, -0.7963193188039507, 0.12395864485237663, -0.18545774404118467, 1.311026289715281, 0.7764007851264465, -0.5776179488468618, 0.5640559901962993])8};
Meaning function: for a clue and a list of two words, compute the Euclidean distance between each word's vector and the clue vector, pass 1/distance through sigmoid, flip on that probability, return true iff all flips are true.
Priors: - wordsPrior(): uniformDraw over [["chicken","eagle"],["eagle","pig"],["chicken","pig"]] - cluePrior(): uniformDraw over ["farm","animal","bird"] - alpha = 1
RSA chain: - literalListener(clue): Infer over wordsPrior(), condition on meaning(clue, randomSubset), return randomSubset - speaker(subset): Infer over cluePrior(), factor alpha * literalListener(clue).score(subset), return clue - pragmaticListener(clue): Infer over wordsPrior(), observe speaker(randomSubset) given clue, return randomSubset
Bind pragmaticListener to a function of that exact name.
Return the pragmaticListener distribution for the clue "farm": pragmaticListener("farm")
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1///fold: vectors2var vectors = {eagle : Vector([-0.8186906894583743, -0.8443627918594182,3 -0.04304780086785447, -0.8257634263841377,4 -0.7607218950809542, 0.47786735164930183,5 0.36942709316422206, 0.18560148725224498,6 0.38625176009619944, 0.24384273963053932,7 1.0355862286322068, -0.14170089242313555,8 -0.17017960843359828, 0.27636172471279313,9 -0.49477465481497807, -1.199206930890509,10 1.0531720839078256, -0.5154875303291531,11 0.30704269353337016, 1.5382356443196483,12 -0.13215425501400774, 1.2222507503066664,13 1.3819617662995949, -1.1579407453927437,14 0.9439311306043343]),15 pig: Vector([-0.9027808771549458, -1.4539105978263833,16 0.5743098399154295, 1.3052815987119957,17 -0.038556210348244704, -0.22144102997326148,18 1.222050088622139, -0.027526643946408857,19 -0.13265827668708097, 1.4799207507145387,20 0.02371336629548181, -0.9405402658175948,21 0.06556493358788004, -1.6556208133885402,22 -0.44306373689318584, -0.475710035110901,23 1.2435716830499404, -1.0677780309283533,24 -0.03344465447168945, 0.16184568683816827,25 0.8718035460475897, 2.082956682688621,26 0.47430271385843953, -0.4479993650378608,27 1.5192928553678355]),28 chicken: Vector([-0.9555633717916903, -0.23467550895948608,29 0.9081102168032618, 1.7681919864431317,30 -0.3888166871286516, 1.2292398003323308,31 1.0624961440319318, -0.3558803892040966,32 -0.17024423658814317, 0.7046776782991592,33 1.624196256505183, -1.1423231844008523,34 -0.9490267652945451, -1.8004114037674281,35 -0.026086280368055388, -2.089757256612839,36 3.5660566372328693, -2.4178611093952225,37 -0.7077960662621875, 0.9418434965990246,38 0.438927172322575, 1.0891725023940724,39 -0.1237861204326181, 0.7602054634506068,40 3.0515580696224083]),41 farm: Vector([-1.3191469349030631, -0.34747873883058705,42 0.09525267994894762, 0.08014872654330456,43 0.1179814806966339, -0.26926061020753783,44 0.709033965954239, -0.6521777385143812,45 1.0195239553589313, 0.7192612109870958,46 1.1711460976059695, -1.0779079866249233,47 -0.5443503049555966, 0.08523251153754875,48 0.1455530206584687, -1.501097375488643,49 1.1151234505440395, 0.0581591541412683,50 -0.1102242123027589, 0.5253857581277014,51 0.21780949510893402, 0.026030837039037417,52 0.07282095318396448, -0.6093002665622598,53 2.0466066458317336]),54 animal: Vector([-0.08132595025854601, -1.8280616716238214,55 -0.4241550049238374, 1.3405833261217683,56 1.3635302219051426, 0.19656106954281044,57 1.0553637657141577, 0.8640316722860499,58 -0.34682275265131135, 0.27196141799987644,59 0.9785603157742483, -3.1767493003780873,60 -0.7566904249011203, -1.1935303767007424,61 0.2523177522167622, 0.33414675815038736,62 0.21147820292953767, 0.2089073521353749,63 0.36413859545070626, -0.3145854077725169,64 0.8470589609352164, 0.8914477422324714,65 0.06602846837066885, -1.1974184866543685,66 1.6807019645814638]),67 bird: Vector([0.9317828273959833, -1.142927450658389,68 1.1249556341704339, 0.7533022372085103,69 0.039221572709652965, 0.5302815428039684,70 1.1525754405638204, 0.5707370610821617,71 0.01803607760778035, 0.9527229145321762,72 1.0851468114908822, -0.4626041548552341,73 -0.5371489443168416, -0.8343285842461913,74 -0.09713481034287788, 0.8070233789520264,75 0.21755780815430825, -0.6588132708557186,76 -0.7963193188039507, 0.12395864485237663,77 -0.18545774404118467, 1.311026289715281,78 0.7764007851264465, -0.5776179488468618,79 0.5640559901962993])80};81///8283var meaning = function(clue, words) {8485 var distance = function(vector1, vector2)86 {87 var squared = map(function(tuple)88 {89 return (tuple[0] - tuple[1])*(tuple[0] - tuple[1]);90 }91 , zip(ad.tensor.toScalars(vector1), ad.tensor.toScalars(vector2))92 );939495 var answer = Math.sqrt(sum(squared));96 return answer;97 };9899 var sigmoid = function(num)100 {101 return 1/(1 + Math.exp(-1*num));102 };103104 var trueFalse = function(clue, word)105 {106 var dist = distance(vectors[clue], vectors[word]);107 var prob = sigmoid(1/dist);108 return flip(prob);109 };110111 var wordsVectors = map(function(word) {return trueFalse(clue, word);}, words);112113114 return all(function(s) {return s;}, wordsVectors);115};116117var wordsPrior = function()118{119 var pairs = [["chicken", "eagle"], ["eagle", "pig"], ["chicken", "pig"]];120 return uniformDraw(pairs);121};122123var cluePrior = function()124{125 return uniformDraw(["farm", "animal", "bird"]);126};127128129var literalListener = function(clue)130{131 Infer(function()132 {133 var randomSubset = wordsPrior();134 var uttTruthVal = meaning(clue, randomSubset);135 condition(uttTruthVal);136 return randomSubset;137 }138 )139};140141var alpha = 1;142143var speaker = function(subset)144{145 Infer(function()146 {147 var clue = cluePrior();148 factor(alpha*literalListener(clue).score(subset));149 return clue;150 }151 )152};153154var pragmaticListener = function(clue)155{156 Infer(function()157 {158 var randomSubset = wordsPrior();159 var s1 = speaker(randomSubset);160 observe(s1, clue);161 return randomSubset;162 }163 )164};165166viz.table(pragmaticListener("farm"));167168var ANSWER = (pragmaticListener("farm"));169
1var vectors = {2 eagle: Vector([-0.8186906894583743, -0.8443627918594182, -0.04304780086785447, -0.8257634263841377, -0.7607218950809542, 0.47786735164930183, 0.36942709316422206, 0.18560148725224498, 0.38625176009619944, 0.24384273963053932, 1.0355862286322068, -0.14170089242313555, -0.17017960843359828, 0.27636172471279313, -0.49477465481497807, -1.199206930890509, 1.0531720839078256, -0.5154875303291531, 0.30704269353337016, 1.5382356443196483, -0.13215425501400774, 1.2222507503066664, 1.3819617662995949, -1.1579407453927437, 0.9439311306043343]),3 pig: Vector([-0.9027808771549458, -1.4539105978263833, 0.5743098399154295, 1.3052815987119957, -0.038556210348244704, -0.22144102997326148, 1.222050088622139, -0.027526643946408857, -0.13265827668708097, 1.4799207507145387, 0.02371336629548181, -0.9405402658175948, 0.06556493358788004, -1.6556208133885402, -0.44306373689318584, -0.475710035110901, 1.2435716830499404, -1.0677780309283533, -0.03344465447168945, 0.16184568683816827, 0.8718035460475897, 2.082956682688621, 0.47430271385843953, -0.4479993650378608, 1.5192928553678355]),4 chicken: Vector([-0.9555633717916903, -0.23467550895948608, 0.9081102168032618, 1.7681919864431317, -0.3888166871286516, 1.2292398003323308, 1.0624961440319318, -0.3558803892040966, -0.17024423658814317, 0.7046776782991592, 1.624196256505183, -1.1423231844008523, -0.9490267652945451, -1.8004114037674281, -0.026086280368055388, -2.089757256612839, 3.5660566372328693, -2.4178611093952225, -0.7077960662621875, 0.9418434965990246, 0.438927172322575, 1.0891725023940724, -0.1237861204326181, 0.7602054634506068, 3.0515580696224083]),5 farm: Vector([-1.3191469349030631, -0.34747873883058705, 0.09525267994894762, 0.08014872654330456, 0.1179814806966339, -0.26926061020753783, 0.709033965954239, -0.6521777385143812, 1.0195239553589313, 0.7192612109870958, 1.1711460976059695, -1.0779079866249233, -0.5443503049555966, 0.08523251153754875, 0.1455530206584687, -1.501097375488643, 1.1151234505440395, 0.0581591541412683, -0.1102242123027589, 0.5253857581277014, 0.21780949510893402, 0.026030837039037417, 0.07282095318396448, -0.6093002665622598, 2.0466066458317336]),6 animal: Vector([-0.08132595025854601, -1.8280616716238214, -0.4241550049238374, 1.3405833261217683, 1.3635302219051426, 0.19656106954281044, 1.0553637657141577, 0.8640316722860499, -0.34682275265131135, 0.27196141799987644, 0.9785603157742483, -3.1767493003780873, -0.7566904249011203, -1.1935303767007424, 0.2523177522167622, 0.33414675815038736, 0.21147820292953767, 0.2089073521353749, 0.36413859545070626, -0.3145854077725169, 0.8470589609352164, 0.8914477422324714, 0.06602846837066885, -1.1974184866543685, 1.6807019645814638]),7 bird: Vector([0.9317828273959833, -1.142927450658389, 1.1249556341704339, 0.7533022372085103, 0.039221572709652965, 0.5302815428039684, 1.1525754405638204, 0.5707370610821617, 0.01803607760778035, 0.9527229145321762, 1.0851468114908822, -0.4626041548552341, -0.5371489443168416, -0.8343285842461913, -0.09713481034287788, 0.8070233789520264, 0.21755780815430825, -0.6588132708557186, -0.7963193188039507, 0.12395864485237663, -0.18545774404118467, 1.311026289715281, 0.7764007851264465, -0.5776179488468618, 0.5640559901962993])8};910var euclideanDistance = function(v1, v2) {11 var diff = T.sub(v1, v2);12 var sq = T.mul(diff, diff);13 var sumSq = T.sumreduce(sq);14 return Math.sqrt(sumSq);15};1617var sigmoid = function(x) {18 return 1 / (1 + Math.exp(-x));19};2021var meaning = function(clue, words) {22 var clueVec = vectors[clue];23 var allTrue = all(function(word) {24 var wordVec = vectors[word];25 var dist = euclideanDistance(clueVec, wordVec);26 var p = sigmoid(1 / dist);27 return flip(p);28 }, words);29 return allTrue;30};3132var wordSets = [["chicken","eagle"],["eagle","pig"],["chicken","pig"]];33var clues = ["farm","animal","bird"];34var alpha = 1;3536var wordsPrior = function() {37 return uniformDraw(wordSets);38};3940var cluePrior = function() {41 return uniformDraw(clues);42};4344var literalListener = mem(function(clue) {45 return Infer({method: 'enumerate'}, function() {46 var words = wordsPrior();47 condition(meaning(clue, words));48 return words;49 });50});5152var speaker = mem(function(subset) {53 return Infer({method: 'enumerate'}, function() {54 var clue = cluePrior();55 factor(alpha * literalListener(clue).score(subset));56 return clue;57 });58});5960var pragmaticListener = function(clue) {61 return Infer({method: 'enumerate'}, function() {62 var words = wordsPrior();63 observe(speaker(words), clue);64 return words;65 });66};6768var ANSWER = pragmaticListener("farm");
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◉TV<.05 | 1.4e-16 | -0.0000 | · |
Implement an RSA model where the pragmatic listener jointly infers the intended object and the speaker's cost parameter in WebPPL. Objects are: [{color:'blue',shape:'square',string:'blue square'},{color:'blue',shape:'circle',string:'blue circle'},{color:'green',shape:'square',string:'green square'}]. objectPrior() = uniformDraw(objects).string. Utterances are ['blue','green','square','circle','blue square','blue circle','green square']. meaning(utterance,obj) = _.includes(obj, utterance). The cost function takes (utterance, costParameter): count the number of words in the utterance (split by space), multiply by costParameter. costParameterPrior() = uniformDraw(_.range(0.05, 5, 0.5)). Literal listener takes utterance, samples obj from objectPrior(), conditions on meaning(utterance,obj), returns obj. Speaker (alpha=1) takes (obj, costParameter): uniformDraw(utterances), factors by alpha*(literalListener(utterance).score(obj) - cost(utterance,costParameter)). Pragmatic listener takes utterance: samples obj from objectPrior() and costParameter from costParameterPrior(), observes speaker(obj,costParameter). Return the pragmatic listener's joint distribution over {obj,costParameter} when hearing 'blue'.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// set of states2var objects = [{color: "blue", shape: "square", string: "blue square"},3 {color: "blue", shape: "circle", string: "blue circle"},4 {color: "green", shape: "square", string: "green square"}]56// prior over world states7var objectPrior = function() {8 var obj = uniformDraw(objects)9 return obj.string10}1112// set of utterances13var utterances = ["blue", "green", "square", "circle",14 "blue square", "blue circle", "green square"]1516// utterance cost function17var cost = function(utterance, costParameter) {18 var numWords = function(utterance) {19 var split = utterance.split(" ")20 return _.size(split)21 }22 return costParameter*numWords(utterance)23};2425var costParameterPrior = function() {26 return uniformDraw(_.range(0.05, 5, 0.5))27}2829// meaning function to interpret the utterances30var meaning = function(utterance, obj){31 _.includes(obj, utterance)32}3334// literal listener35var literalListener = function(utterance){36 Infer({model: function(){37 var obj = objectPrior();38 condition(meaning(utterance, obj))39 return obj40 }})41}4243// set speaker optimality44var alpha = 14546// pragmatic speaker47var speaker = function(obj,costParameter){48 Infer({model: function(){49 var utterance = uniformDraw(utterances)50 factor(alpha * (literalListener(utterance).score(obj) -51 cost(utterance,costParameter)))52 return utterance53 }})54}5556// pragmatic listener57var pragmaticListener = function(utterance){58 Infer({model: function(){59 var obj = objectPrior()60 var costParameter = costParameterPrior()61 observe(speaker(obj,costParameter),utterance)62 return {obj, costParameter}63 }})64}656667display("cost parameter prior")68viz(Infer(costParameterPrior))6970var listenerPosteriorBlue = pragmaticListener("blue")71display("pragmatic listener hears \"blue\"")72viz.table(marginalize(listenerPosteriorBlue, "obj"))73viz(marginalize(listenerPosteriorBlue, "costParameter"))7475var listenerPosteriorBlueSquare = pragmaticListener("blue square")76display("pragmatic listener hears \"blue square\"")77viz.table(marginalize(listenerPosteriorBlueSquare, "obj"))78viz(marginalize(listenerPosteriorBlueSquare, "costParameter"))7980var ANSWER = (pragmaticListener('blue'));81
1var objects = [2 {color: 'blue', shape: 'square', string: 'blue square'},3 {color: 'blue', shape: 'circle', string: 'blue circle'},4 {color: 'green', shape: 'square', string: 'green square'}5];67var utterances = ['blue', 'green', 'square', 'circle', 'blue square', 'blue circle', 'green square'];89var objectPrior = function() {10 return uniformDraw(objects).string;11};1213var meaning = function(utterance, obj) {14 return _.includes(obj, utterance);15};1617var cost = function(utterance, costParameter) {18 return utterance.split(' ').length * costParameter;19};2021var costParameterPrior = function() {22 return uniformDraw(_.range(0.05, 5, 0.5));23};2425var alpha = 1;2627var literalListener = mem(function(utterance) {28 return Infer({method: 'enumerate'}, function() {29 var obj = objectPrior();30 condition(meaning(utterance, obj));31 return obj;32 });33});3435var speaker = mem(function(obj, costParameter) {36 return Infer({method: 'enumerate'}, function() {37 var utterance = uniformDraw(utterances);38 factor(alpha * (literalListener(utterance).score(obj) - cost(utterance, costParameter)));39 return utterance;40 });41});4243var pragmaticListener = function(utterance) {44 return Infer({method: 'enumerate'}, function() {45 var obj = objectPrior();46 var costParameter = costParameterPrior();47 observe(speaker(obj, costParameter), utterance);48 return {obj: obj, costParameter: costParameter};49 });50};5152var ANSWER = pragmaticListener('blue');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
Implement a prior model for generic language in WebPPL. The prevalence space is a discretized range: bins = _.range(0.01, 1, 0.025) (these are the possible prevalence values). Define a DiscreteBeta(g, d) function that creates a Categorical distribution over bins using the Beta PDF: for each x in bins, weight is x^(g*d-1) * (1-x)^((1-g)*d-1). Use cache for DiscreteBeta. Define priorModel(params) as an Infer over: sample potential, prevalenceWhenPresent (g), and concentrationWhenPresent (d) from params; flip(potential) selects between sample(DiscreteBeta(g,d)) and sample(DiscreteBeta(0.01,100)); return {prevalence: prevalence}. Call priorModel with params {potential:0.3, prevalenceWhenPresent:0.5, concentrationWhenPresent:10} (representing a 'lays eggs' prior). Return the resulting distribution.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1///fold:2// discretized range between 0 - 13var bins = _.range(0.01, 1, 0.025);45// function returns a discretized Beta distribution6var DiscreteBeta = cache(function(g, d){7 var a = g * d, b = (1-g) * d;8 var betaPDF = function(x){9 return Math.pow(x, a-1)*Math.pow((1-x), b-1)10 }11 var probs = map(betaPDF, bins);12 return Categorical({vs: bins, ps: probs})13})14///15var priorModel = function(params){16 Infer({model: function(){1718 var potential = params["potential"]19 var g = params["prevalenceWhenPresent"]20 var d = params["concentrationWhenPresent"]2122 var StableDistribution = DiscreteBeta(g, d)23 var UnstableDistribution = DiscreteBeta(0.01, 100)2425 var prevalence = flip(potential) ?26 sample(StableDistribution) :27 sample(UnstableDistribution)2829 return {prevalence}3031 }})32}3334// e.g. "Lays eggs"35viz(priorModel({36 potential: 0.3,37 prevalenceWhenPresent: 0.5, // how prevalent under the stable cause38 concentrationWhenPresent: 10 // the inverse-variance of the stable cause39}))4041var ANSWER = (priorModel({potential: 0.3, prevalenceWhenPresent: 0.5, concentrationWhenPresent: 10}));42
1var bins = _.range(0.01, 1, 0.025);23var DiscreteBeta = cache(function(g, d) {4 var weights = map(function(x) {5 return Math.pow(x, g * d - 1) * Math.pow(1 - x, (1 - g) * d - 1);6 }, bins);7 var totalWeight = sum(weights);8 var ps = map(function(w) { return w / totalWeight; }, weights);9 return Categorical({vs: bins, ps: ps});10});1112var priorModel = function(params) {13 return Infer({method: 'enumerate'}, function() {14 var potential = sample(Bernoulli({p: params.potential}));15 var g = params.prevalenceWhenPresent;16 var d = params.concentrationWhenPresent;17 var prevalence = potential ?18 sample(DiscreteBeta(g, d)) :19 sample(DiscreteBeta(0.01, 100));20 return {prevalence: prevalence};21 });22};2324var ANSWER = priorModel({25 potential: 0.3,26 prevalenceWhenPresent: 0.5,27 concentrationWhenPresent: 1028});
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◉TV<.05 | 5.4e-17 | 0.0000 | · |
Implement a Bayesian generic language interpretation model in WebPPL. The prevalence space is bins = map(function(x){ _.round(x,2); }, _.range(0.01,1,0.02)). The threshold space is thresholdBins computed as midpoints between consecutive bins. DiscreteBeta(g,d) returns a Categorical over bins using Beta PDF: weight for x is x^(g*d-1)*(1-x)^((1-g)*d-1). Use cache for DiscreteBeta. priorModel(params) draws prevalence from a mixture: flip(potential) selects DiscreteBeta(g,d) vs DiscreteBeta(0.01,100), returning the prevalence (a scalar, not a record). meaning(utterance,prevalence,threshold) is true for 'generic' when prevalence>threshold, always true for other utterances. thresholdPrior() = uniformDraw(thresholdBins). statePrior = priorModel({potential:0.3, prevalenceWhenPresent:0.5, concentrationWhenPresent:10}). The literal listener (named 'listener', using cache) takes an utterance, samples prevalence from statePrior and threshold from thresholdPrior, conditions on meaning(utterance,prevalence,threshold), and returns prevalence. Return the listener's posterior distribution over prevalence when hearing 'generic'.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1///fold:2// discretized range between 0 - 13var bins = map(function(x){4 _.round(x, 2);5}, _.range(0.01, 1, 0.02));67var thresholdBins = map2(function(x,y){8 var d = (y - x)/ 2;9 return x + d10}, bins.slice(0, bins.length - 1), bins.slice(1, bins.length))1112// function returns a discretized Beta distribution13var DiscreteBeta = cache(function(g, d){14 var a = g * d, b = (1-g) * d;15 var betaPDF = function(x){16 return Math.pow(x, a-1)*Math.pow((1-x), b-1)17 }18 var probs = map(betaPDF, bins);19 return Categorical({vs: bins, ps: probs})20})2122var priorModel = function(params){23 Infer({model: function(){2425 var potential = params["potential"]26 var g = params["prevalenceWhenPresent"]27 var d = params["concentrationWhenPresent"]2829 var StableDistribution = DiscreteBeta(g, d)30 var UnstableDistribution = DiscreteBeta(0.01, 100)3132 var prevalence = flip(potential) ?33 sample(StableDistribution) :34 sample(UnstableDistribution)3536 return prevalence3738 }})39}40///41var meaning = function(utterance, prevalence, threshold) {42 return (utterance == 'generic') ? prevalence > threshold : true43}44var thresholdPrior = function() { return uniformDraw(thresholdBins) };4546var statePrior = priorModel({47 potential: 0.3,48 prevalenceWhenPresent: 0.5, // how prevalent under the stable cause49 concentrationWhenPresent: 10 // the inverse-variance of the stable cause50})5152display("prevalence prior")53viz(statePrior)5455var listener = cache(function(utterance) {56 Infer({model: function(){57 var prevalence = sample(statePrior)58 var threshold = thresholdPrior()59 var m = meaning(utterance, prevalence, threshold)60 condition(m)61 return prevalence62 }})63})6465display("listener posterior")66listener("generic")6768var ANSWER = (listener('generic'));69
1var bins = map(function(x){ return _.round(x, 2); }, _.range(0.01, 1, 0.02));23var thresholdBins = map(function(i){4 return _.round((bins[i] + bins[i+1]) / 2, 3);5}, _.range(0, bins.length - 1));67var DiscreteBeta = cache(function(g, d){8 var weights = map(function(x){9 return Math.pow(x, g*d - 1) * Math.pow(1 - x, (1-g)*d - 1);10 }, bins);11 var totalWeight = sum(weights);12 var ps = map(function(w){ return w / totalWeight; }, weights);13 return Categorical({vs: bins, ps: ps});14});1516var priorModel = function(params){17 var potential = params.potential;18 var g = params.prevalenceWhenPresent;19 var d = params.concentrationWhenPresent;20 var isPresent = flip(potential);21 var prevalence = isPresent ?22 sample(DiscreteBeta(g, d)) :23 sample(DiscreteBeta(0.01, 100));24 return prevalence;25};2627var meaning = function(utterance, prevalence, threshold){28 return utterance === 'generic' ? prevalence > threshold : true;29};3031var thresholdPrior = function(){32 return uniformDraw(thresholdBins);33};3435var statePrior = function(){36 return priorModel({potential: 0.3, prevalenceWhenPresent: 0.5, concentrationWhenPresent: 10});37};3839var listener = cache(function(utterance){40 return Infer({method: 'enumerate'}, function(){41 var prevalence = statePrior();42 var threshold = thresholdPrior();43 condition(meaning(utterance, prevalence, threshold));44 return prevalence;45 });46});4748var ANSWER = listener('generic');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◉TV<.05 | 1.6e-16 | 0.0000 | · |
A Tessler et al. (2017) RSA comparison-class model for gradable adjectives. A pragmatic listener hears "tall" and knows the person is a basketball player. They infer whether "tall" is relative to all people (superordinate) or to basketball players (subordinate).
binParam = 3. superordinate_params = {mu: 0, sigma: 1}. stateVals: _.range(-3, 3, 1/3) (18 values). stateProbs(stateParams): map over stateVals, exp(Gaussian(stateParams).score(s)). generateStatePrior(stateParams): Infer categorical({vs: stateVals, ps: stateProbs(stateParams)}). thresholdBins: {positive: stateVals - 1/6, negative: stateVals + 1/6}. thresholdPrior(form): uniformDraw(thresholdBins[form]). subParams = {gymnasts: {mu:-1, sigma:0.5}, soccerPlayers: {mu:0, sigma:0.5}, basketballPlayers: {mu:1, sigma:0.5}}. utterances = ["tall", "short", "silence"]. meaning(utterance, state, thresholds): tall ? state > thresholds.tall : short ? state < thresholds.short : true. classPrior: uniformDraw(["subordinate", "superordinate"]). alpha = 5. literalListener(utterance, thresholds, comparisonClass): Infer, samples state from generateStatePrior(comparisonClass), conditions on meaning, returns state. cache 10000. speaker1(state, thresholds, comparisonClass): Infer, uniformDraw(utterances), factor(alpha * L0.score(state)), returns utterance. cache 10000. pragmaticListener(utterance, subordinate_params): Infer, samples state from generateStatePrior(subordinate_params), samples thresholds {tall: thresholdPrior("positive"), short: thresholdPrior("negative")}, samples c from classPrior, comparisonClass = c=="subordinate" ? subordinate_params : superordinate_params, observes speaker1, returns {comparisonClass: c, state}. cache 10000.
Compute marginalize(pragmaticListener("tall", subParams["basketballPlayers"]), "comparisonClass"). Return the marginal distribution over comparisonClass.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1///fold:2// helper function3var exp = function(x){return Math.exp(x)}45// for discretization6var binParam = 3;78// information about the superordinate category prior9// e.g., the height distribution for all people10var superordinate_params = {mu: 0, sigma: 1};1112// calculate the range in pre-defined steps;13// these values correspond to possible heights14var stateVals = _.range(superordinate_params.mu - 3 * superordinate_params.sigma,15 superordinate_params.mu + 3 * superordinate_params.sigma,16 superordinate_params.sigma/binParam)1718// for each possible height, calculate its probability of occurrence19var stateProbs = cache(function(stateParams){20 return map(function(s){21 exp(Gaussian(stateParams).score(s))22 }, stateVals)23});2425// generate a statePrior using the possible heights and their probabilities26var generateStatePrior = cache(function(stateParams) {27 return Infer({28 model: function(){29 return categorical({vs: stateVals, ps: stateProbs(stateParams)})30 }31 })32});3334// generate the uniform threshold prior35var thresholdBins ={36 positive: map(function(x){37 return x - (1/(binParam*2));38 }, sort(stateVals)),39 negative: map(function(x){40 return x + (1/(binParam*2));41 }, sort(stateVals))42};4344var thresholdPrior = cache(function(form){45 return Infer({46 model: function() { return uniformDraw(thresholdBins[form]) }47 });48});49///5051// information about the superordinate category priors52var subParams = {53 gymnasts: {mu: -1, sigma: 0.5}, // gymnast heights54 soccerPlayers: {mu: 0, sigma: 0.5}, // soccer player heights55 basketballPlayers: {mu: 1, sigma: 0.5} // basketball player heights56}5758// possible utterances can be either positive (tall) or negative (short) or a null utterance59var utterances = ["tall", "short", "silence"]6061// meaning function for utterances62var meaning = function(utterance, state, thresholds) {63 utterance == "tall" ? state > thresholds.tall :64 utterance == "short" ? state < thresholds.short :65 true66}6768// assume a uniform prior over comparison classes69var classPrior = Infer({70 model: function(){return uniformDraw(["subordinate", "superordinate"])}71});7273// set speaker optimality74var alpha = 5;7576var literalListener = cache(77 function(utterance, thresholds, comparisonClass) {78 Infer({model: function(){79 var StatePrior = generateStatePrior(comparisonClass)80 var state = sample(StatePrior);81 var m = meaning(utterance, state, thresholds);82 condition(m);83 return state;84 }})85 }, 10000 // limit cache size86)8788var speaker1 = cache(89 function(state, thresholds, comparisonClass) {90 Infer({model: function(){91 var utterance = uniformDraw(utterances);92 var L0 = literalListener(utterance, thresholds, comparisonClass);93 factor( alpha * L0.score(state) );94 return utterance;95 }})96 }, 10000 // limit cache size97)9899var pragmaticListener = cache(function(utterance, subordinate_params) {100 Infer({model: function(){101102 var statePrior = generateStatePrior(subordinate_params);103 var state = sample(statePrior);104 // separate thresholds for positive adjective and negative adjective105 var thresholds = {106 tall: sample(thresholdPrior("positive")),107 short: sample(thresholdPrior("negative"))108 }109110 // uncertainty about the comparison class (superordinate vs. subordinate)111 var c = sample(classPrior)112 var comparisonClass = c == "subordinate" ? subordinate_params : superordinate_params113114 var S1 = speaker1(state, thresholds, comparisonClass);115 observe(S1, utterance);116117 return { comparisonClass: c, state : state }118 }})119}, 10000 // limit cache size120 )121122// the possible experiment conditions:123// you hear that someone is a member of a subordinate category124// then you are told that they are tall/short;125// the task is to figure out the implicit comparison class126var exptConditions = [127 {utt: "tall", sub: "basketballPlayers"},128 {utt: "short", sub: "basketballPlayers"},129 {utt: "tall", sub: "soccerPlayers"},130 {utt: "short", sub: "soccerPlayers"},131 {utt: "tall", sub: "gymnasts"},132 {utt: "short", sub: "gymnasts"}133];134135// generate structure predictions by mapping through the experiment conditions136var L1predictions = map(function(stim){137 var L1posterior = pragmaticListener(stim.utt, subParams[stim.sub])138 return {139 utterance: stim.utt,140 "P(superordinate comparison class)": exp(marginalize(L1posterior, "comparisonClass").score("superordinate")),141 "subordinate category": stim.sub,142 model: "L1"143 }144}, exptConditions)145146display("the basketball player is tall")147display("--> height = " + expectation(marginalize(pragmaticListener("tall",{mu: 1, sigma: 0.5}), "state")))148display("the basketball player is short")149display("--> height = " + expectation(marginalize(pragmaticListener("short",{mu: 1, sigma: 0.5}), "state")))150151display("probability of superordinate comparison class (i.e., tall for all people)")152viz.bar(L1predictions, {groupBy: "subordinate category"})153154var ANSWER = (marginalize(pragmaticListener("tall", subParams["basketballPlayers"]), "comparisonClass"));155
1var binParam = 3;2var superordinate_params = {mu: 0, sigma: 1};3var stateVals = _.range(-3, 3, 1/3);45var stateProbs = function(stateParams) {6 return map(function(s) {7 return Math.exp(Gaussian({mu: stateParams.mu, sigma: stateParams.sigma}).score(s));8 }, stateVals);9};1011var generateStatePrior = function(stateParams) {12 return Infer({method: 'enumerate'}, function() {13 return sample(Categorical({vs: stateVals, ps: stateProbs(stateParams)}));14 });15};1617var thresholdBins = {18 positive: map(function(s) { return s - 1/6; }, stateVals),19 negative: map(function(s) { return s + 1/6; }, stateVals)20};2122var thresholdPrior = function(form) {23 return uniformDraw(thresholdBins[form]);24};2526var subParams = {27 gymnasts: {mu: -1, sigma: 0.5},28 soccerPlayers: {mu: 0, sigma: 0.5},29 basketballPlayers: {mu: 1, sigma: 0.5}30};3132var utterances = ["tall", "short", "silence"];3334var meaning = function(utterance, state, thresholds) {35 return utterance === "tall" ? state > thresholds.tall :36 utterance === "short" ? state < thresholds.short :37 true;38};3940var alpha = 5;4142var literalListener = mem(function(utterance, thresholds, comparisonClass) {43 return Infer({method: 'enumerate'}, function() {44 var state = sample(generateStatePrior(comparisonClass));45 condition(meaning(utterance, state, thresholds));46 return state;47 });48}, 10000);4950var speaker1 = mem(function(state, thresholds, comparisonClass) {51 return Infer({method: 'enumerate'}, function() {52 var utterance = uniformDraw(utterances);53 var L0 = literalListener(utterance, thresholds, comparisonClass);54 factor(alpha * L0.score(state));55 return utterance;56 });57}, 10000);5859var pragmaticListener = mem(function(utterance, subordinate_params) {60 return Infer({method: 'enumerate'}, function() {61 var state = sample(generateStatePrior(subordinate_params));62 var thresholds = {63 tall: thresholdPrior("positive"),64 short: thresholdPrior("negative")65 };66 var c = uniformDraw(["subordinate", "superordinate"]);67 var comparisonClass = c === "subordinate" ? subordinate_params : superordinate_params;68 var S1 = speaker1(state, thresholds, comparisonClass);69 observe(S1, utterance);70 return {comparisonClass: c, state: state};71 });72}, 10000);7374var result = pragmaticListener("tall", subParams["basketballPlayers"]);75var marginal = marginalize(result, "comparisonClass");7677var ANSWER = marginal;
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◉TV<.05 | 5.6e-16 | -0.0000 | · |
An extension of Kao and Goodman (2015) RSA irony model with 5 weather states and continuous-valued arousal. California weather context. A pragmatic listener hears "terrible" and infers the state, valence, and arousal.
states = ["terrible","bad","ok","good","amazing"]. statePrior: categorical([1,5,40,40,40], states). utterances = states. utterancePrior: uniformDraw(utterances). valencePrior(state): terrible->flip(0.99)?-1:1, bad->flip(0.90)?-1:1, ok->flip(0.5)?-1:1, good->flip(0.09)?-1:1, amazing->flip(0.01)?-1:1. arousals = [0.1, 0.3, 0.5, 0.7, 0.9] (continuous numeric, not "low"/"high"). arousalPrior(state): terrible->categorical([1,10,30,45,50],arousals), bad->categorical([1,5,25,40,45],arousals), ok->categorical([50,45,30,10,1],arousals), good->categorical([1,5,25,40,45],arousals), amazing->categorical([1,10,30,45,50],arousals). goals = ["goalState","goalValence","goalArousal"]. goalPrior: categorical([1,1,1], goals). literalInterpretation(utterance, state): utterance === state. goalState(goal, state, valence, arousal): goalState->state, goalValence->valence, goalArousal->arousal. literalListener(utterance, goal): Infer over uniformDraw(states), samples valence and arousal, conditions on literalInterpretation, returns goalState(goal,state,valence,arousal). speaker(state, valence, arousal, goal): Infer, factor(1 * literalListener(utterance,goal).score(goalState(...))), returns utterance. pragmaticListener(utterance): Infer, samples state/valence/arousal/goal, observes speaker, returns {state, valence, arousal}.
Compute pragmaticListener("terrible"). Return joint distribution over {state, valence, arousal}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// There are three possible states the weather could be in:2// terrible, ok, or amazing3var states = ['terrible','bad','ok','good','amazing']45// Since we are in California, the prior over these states6// are the following. Once could also imagine this being7// the prior in a certain context, e.g. when it's clearly8// sunny and nice out.9var statePrior = function() {10 categorical([1,5,40,40,40], states)11}1213// Valence prior defined in terms of negative valence.14// If the current state is terrible, it's extremely likely15// that the valence associated is negative. If it's ok, then16// the valence could be negative or positive with equal17// probability.18var valencePrior = function(state) {19 state === "terrible" ? flip(0.99) ? -1 : 1 :20 state === "bad" ? flip(0.90) ? -1 : 1 :21 state === "ok" ? flip(0.5) ? -1 : 1 :22 state === "good" ? flip(0.09) ? -1 : 1 :23 state === "amazing" ? flip(0.01) ? -1 : 1 :24 true25}2627// Define binary arousals (could model as continuous).28// var arousals = ["low", "high"]29var arousals = [.1,.3,.5,.7,.9]3031// Define goals and goal priors. Could want to communicate state of the world,32// valence about it, or arousal (intensity of feeling) about it.33var goals = ["goalState", "goalValence", "goalArousal"]3435var goalPrior = function() {36 categorical([1, 1, 1], goals)37}3839// Assume possible utterances are identical to possible states40var utterances = states4142// Assume cost of utterances is uniform.43var utterancePrior = function() {44 uniformDraw(utterances)45}4647// Sample arousal given a state.48var arousalPrior = function(state) {49 state === "terrible" ? categorical([1,10,30,45,50], arousals) :50 state === "bad" ? categorical([1,5,25,40,45], arousals) :51 state === "ok" ? categorical([50,45,30,10,1], arousals) :52 state === "good" ? categorical([1,5,25,40,45], arousals) :53 state === "amazing" ? categorical([1,10,30,45,50], arousals) :54 true55}5657// Literal interpretation is just whether utterance equals state58var literalInterpretation = function(utterance, state) {59 utterance === state60}6162// A speaker's goal is satisfied if the listener infers the correct63// and relevant information.64var goalState = function(goal, state, valence, arousal) {65 goal === "goalState" ? state :66 goal === "goalValence" ? valence :67 goal === "goalArousal" ? arousal :68 true69}7071// Define a literal listener72var literalListener = function(utterance, goal) {73 Infer({model: function(){74 var state = uniformDraw(states)75 var valence = valencePrior(state)76 var arousal = arousalPrior(state)77 condition(literalInterpretation(utterance,state))78 return goalState(goal, state, valence, arousal)79 }})80}8182//The speaker takes in a state, valence, arousal, and a goal and returns an utterance83//based on the probability of the literalListener arriving at the correct84//state given a goalState85var speaker = function(state, valence, arousal, goal) {86 Infer({model: function(){87 var utterance = utterancePrior()88 factor(1 * literalListener(utterance,89 goal).score(goalState(goal,90 state,91 valence,92 arousal)))93 return utterance94 }})95}9697// Define a pragmatic listener98var pragmaticListener = function(utterance) {99 Infer({model: function(){100 var state = statePrior()101 var valence = valencePrior(state)102 var arousal = arousalPrior(state)103 var goal = goalPrior()104 observe(speaker(state, valence, arousal, goal),utterance)105 return {state,valence, arousal}106 }})107}108109viz.table(pragmaticListener('terrible'))110111var ANSWER = (pragmaticListener("terrible"));112
1var states = ["terrible", "bad", "ok", "good", "amazing"];2var utterances = states;3var arousals = [0.1, 0.3, 0.5, 0.7, 0.9];4var goals = ["goalState", "goalValence", "goalArousal"];56var statePrior = function() {7 return categorical({vs: states, ps: [1, 5, 40, 40, 40]});8};910var utterancePrior = function() {11 return uniformDraw(utterances);12};1314var valencePrior = function(state) {15 if (state === "terrible") { return flip(0.99) ? -1 : 1; }16 else if (state === "bad") { return flip(0.90) ? -1 : 1; }17 else if (state === "ok") { return flip(0.5) ? -1 : 1; }18 else if (state === "good") { return flip(0.09) ? -1 : 1; }19 else if (state === "amazing") { return flip(0.01) ? -1 : 1; }20};2122var arousalPrior = function(state) {23 if (state === "terrible") { return categorical({vs: arousals, ps: [1, 10, 30, 45, 50]}); }24 else if (state === "bad") { return categorical({vs: arousals, ps: [1, 5, 25, 40, 45]}); }25 else if (state === "ok") { return categorical({vs: arousals, ps: [50, 45, 30, 10, 1]}); }26 else if (state === "good") { return categorical({vs: arousals, ps: [1, 5, 25, 40, 45]}); }27 else if (state === "amazing") { return categorical({vs: arousals, ps: [1, 10, 30, 45, 50]}); }28};2930var goalPrior = function() {31 return categorical({vs: goals, ps: [1, 1, 1]});32};3334var literalInterpretation = function(utterance, state) {35 return utterance === state;36};3738var goalState = function(goal, state, valence, arousal) {39 if (goal === "goalState") { return state; }40 else if (goal === "goalValence") { return valence; }41 else if (goal === "goalArousal") { return arousal; }42};4344var literalListener = mem(function(utterance, goal) {45 return Infer({method: "enumerate"}, function() {46 var state = uniformDraw(states);47 var valence = valencePrior(state);48 var arousal = arousalPrior(state);49 condition(literalInterpretation(utterance, state));50 return goalState(goal, state, valence, arousal);51 });52});5354var speaker = mem(function(state, valence, arousal, goal) {55 return Infer({method: "enumerate"}, function() {56 var utterance = utterancePrior();57 var gs = goalState(goal, state, valence, arousal);58 factor(1 * literalListener(utterance, goal).score(gs));59 return utterance;60 });61});6263var pragmaticListener = function(utterance) {64 return Infer({method: "enumerate"}, function() {65 var state = statePrior();66 var valence = valencePrior(state);67 var arousal = arousalPrior(state);68 var goal = goalPrior();69 var gs = goalState(goal, state, valence, arousal);70 observe(speaker(state, valence, arousal, goal), utterance);71 return {state: state, valence: valence, arousal: arousal};72 });73};7475var ANSWER = pragmaticListener("terrible");
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
A Tessler et al. (2017) RSA comparison-class model adapted for weight adjectives ("heavy"/"light") and speaker identity (child, adult, bodybuilder). The pragmatic listener hears "heavy" uttered by a child and infers the actual box weight.
binParam = 5. superordinate = {mu: 3, sigma: 1} (scale 0-6 represents lightest to heaviest). stateVals: _.range(0, 6, 0.2) (30 values). stateProbs(stateParams): map over stateVals, exp(Gaussian(stateParams).score(s)). generateStatePrior(stateParams): Infer categorical({vs: stateVals, ps: stateProbs(stateParams)}). speakerParams = {child: {mu:0.5, sigma:1}, adult: {mu:2, sigma:3}, bodybuilder: {mu:5, sigma:3}}. utterances = ["heavy", "light"]. thresholdBins(utterance, stateSupport): heavy ? x-1/10 : x+1/10. thresholdPrior(utterance, stateSupport): uniformDraw(thresholdBins(utterance, stateSupport)). meaning(utterance, state, thresholdHeavy, thresholdLight): heavy ? state>thresholdHeavy ? flip(0.9999) : flip(0.0001) : light ? state<thresholdLight ? flip(0.9999) : flip(0.0001) : true. alpha = 5. literalListener(utterance, thresholdHeavy, thresholdLight, comparisonClass): Infer, samples state from generateStatePrior(speakerParams[comparisonClass]), conditions on meaning. cache 10000. speaker1(state, thresholdHeavy, thresholdLight, comparisonClass): Infer, uniformDraw(utterances), factor(alpha * L0.score(state)). cache 10000. comparisonClasses = ["child","adult","bodybuilder"]. comparisonClassPrior(whoSaidIt): child -> categorical([0.75,0.25,0.15], comparisonClasses), adult -> categorical([0.01,0.7,0.5], comparisonClasses), bodybuilder -> categorical([0.0001,0.2,0.99], comparisonClasses). pragmaticListener(utterance, whoSaidIt): Infer({method:"enumerate"}), samples CC from comparisonClassPrior(whoSaidIt), state from generateStatePrior(speakerParams[CC]), thresholdHeavy and thresholdLight from thresholdPrior, observes speaker1, returns state. cache 10000.
Compute pragmaticListener("heavy", "child"). Return distribution over state (box weight on scale 0-6).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// helper function2var exp = function(x){return Math.exp(x)}34// helper function5var marginalize = function(dist, key){6 return Infer({model: function(){sample(dist)[key]}})7}8// for discretization9var binParam = 5;1011//my model:12var superordinate = {mu: 3, sigma: 1};1314//a list of possible box weights (state values)15var stateVals = _.range(superordinate.mu - 3 * superordinate.sigma,16 superordinate.mu + 3 * superordinate.sigma,17 superordinate.sigma/binParam)181920// for each possible weight, calculate its probability of occurrence21var stateProbs = function(stateParams){22 return map(function(s){23 exp(Gaussian(stateParams).score(s))24 }, stateVals)25};262728// generate a statePrior using the possible weights and their probabilities29var generateStatePrior = cache(function(stateParams) {30 return Infer({31 model: function(){32 return categorical({vs: stateVals, ps: stateProbs(stateParams)})33 }34 })35});3637// information about the category priors38var speakerParams = {39 child: {mu: 0.5, sigma: 1}, // child experience with weights40 adult: {mu: 2, sigma: 3}, // adult experiance with weights41 bodybuilder: {mu:5, sigma:3}, // body builder experience with weights42}434445// generate the uniform threshold prior4647var thresholdBins = cache(function(utterance, stateSupport){48 return map(function(x){49 return utterance == "heavy" ? x - (1/(binParam*2)) : x + (1/(binParam*2));50 }, sort(stateSupport))51},1000052)5354var thresholdPrior = cache(function(utterance, stateSupport){55 return Infer({56 model: function() { return uniformDraw(thresholdBins(utterance, stateSupport)) }57 });58},1000059);60616263var utterances = ["heavy","light"]646566// meaning function for utterances67var meaning = function(utterance, state, thresholdHeavy, thresholdLight) {68 utterance == "heavy" ? state > thresholdHeavy ? flip(0.9999) : flip(0.0001) :69 utterance == "light" ? state < thresholdLight ? flip(0.9999) : flip(0.0001) :70 true71}7273747576// set sepeaker optimality77var alpha = 5;787980var literalListener =cache(81 function(utterance, thresholdHeavy, thresholdLight,comparisonClass){82 Infer({model: function(){83 var state = sample(generateStatePrior(speakerParams[comparisonClass]))84 var m = meaning(utterance, state, thresholdHeavy, thresholdLight);85 condition(m);86 return state;87 }})88 },1000089)909192//literalListener("light", 4, 2, "adult")939495//my model:96var speaker1 = cache(97 function(state, thresholdHeavy, thresholdLight,comparisonClass){98 Infer({model: function(){99 var utterance = uniformDraw(utterances);100 var L0 = literalListener(utterance, thresholdHeavy, thresholdLight, comparisonClass);101 factor( alpha * L0.score(state) );102 return utterance;103 }})104 },10000105)106107108// generateStatePrior(speakerParams["child"]) .support()109110var comparisonClasses = ["child","adult","bodybuilder"]111var comparisonClassPrior = function(whoSaidIt) {112 whoSaidIt == "child" ? categorical([0.75, 0.25,0.15],comparisonClasses):113 whoSaidIt == "adult" ? categorical([0.01,0.7,0.5],comparisonClasses):114 categorical([0.0001,0.2, 0.99],comparisonClasses)115116117}118119var pragmaticListener = cache(function(utterance,whoSaidIt){120 Infer({model: function(){121 var CC = comparisonClassPrior(whoSaidIt);122 var statePrior = generateStatePrior(speakerParams[CC]);123 var state = sample(statePrior);124 var thresholdHeavy = sample(thresholdPrior("heavy", statePrior.support()))125 var thresholdLight = sample(thresholdPrior("light", statePrior.support()))126 var S1 = speaker1(state, thresholdHeavy,thresholdLight,CC);127 observe(S1, utterance);128 return (state)129}, method:"enumerate"})130},10000131)132133134135// pragmaticListener("heavy","adult")136display("Listener's interpretation after hearing a child saying 'this box is heavy'")137viz.density(pragmaticListener("heavy","child"))138display("Listener's interpretation after hearing an adult saying 'this box is heavy'")139viz.density(pragmaticListener("heavy","adult"))140display("Listener's interpretation after hearing a bodybuilder saying 'this box is heavy'")141viz.density(pragmaticListener("heavy","bodybuilder"))142display("Listener's interpretation after hearing a child saying 'this box is light'")143viz.density(pragmaticListener("light","child"))144display("Listener's interpretation after hearing an adult saying 'this box is light'")145viz.density(pragmaticListener("light","adult"))146display("Listener's interpretation after hearing a bodybuilder saying 'this box is light'")147viz.density(pragmaticListener("light","bodybuilder"))148149var ANSWER = (pragmaticListener("heavy", "child"));150
1var binParam = 5;2var superordinate = {mu: 3, sigma: 1};3var stateVals = _.range(0, 6, 0.2);45var stateProbs = function(stateParams) {6 return map(function(s) {7 return Math.exp(Gaussian(stateParams).score(s));8 }, stateVals);9};1011var generateStatePrior = function(stateParams) {12 return Infer({method: "enumerate"}, function() {13 return sample(Categorical({vs: stateVals, ps: stateProbs(stateParams)}));14 });15};1617var speakerParams = {18 child: {mu: 0.5, sigma: 1},19 adult: {mu: 2, sigma: 3},20 bodybuilder: {mu: 5, sigma: 3}21};2223var utterances = ["heavy", "light"];2425var thresholdBins = function(utterance, stateSupport) {26 return map(function(x) {27 return utterance === "heavy" ? x - 1/10 : x + 1/10;28 }, stateSupport);29};3031var thresholdPrior = function(utterance, stateSupport) {32 return uniformDraw(thresholdBins(utterance, stateSupport));33};3435var meaning = function(utterance, state, thresholdHeavy, thresholdLight) {36 return utterance === "heavy" ?37 (state > thresholdHeavy ? flip(0.9999) : flip(0.0001)) :38 utterance === "light" ?39 (state < thresholdLight ? flip(0.9999) : flip(0.0001)) :40 true;41};4243var alpha = 5;4445var literalListener = mem(function(utterance, thresholdHeavy, thresholdLight, comparisonClass) {46 return Infer({method: "enumerate"}, function() {47 var statePrior = generateStatePrior(speakerParams[comparisonClass]);48 var state = sample(statePrior);49 condition(meaning(utterance, state, thresholdHeavy, thresholdLight));50 return state;51 });52}, 10000);5354var speaker1 = mem(function(state, thresholdHeavy, thresholdLight, comparisonClass) {55 return Infer({method: "enumerate"}, function() {56 var utterance = uniformDraw(utterances);57 var L0 = literalListener(utterance, thresholdHeavy, thresholdLight, comparisonClass);58 factor(alpha * L0.score(state));59 return utterance;60 });61}, 10000);6263var comparisonClasses = ["child", "adult", "bodybuilder"];6465var comparisonClassPrior = function(whoSaidIt) {66 return whoSaidIt === "child" ?67 sample(Categorical({vs: comparisonClasses, ps: [0.75, 0.25, 0.15]})) :68 whoSaidIt === "adult" ?69 sample(Categorical({vs: comparisonClasses, ps: [0.01, 0.7, 0.5]})) :70 sample(Categorical({vs: comparisonClasses, ps: [0.0001, 0.2, 0.99]}));71};7273var pragmaticListener = mem(function(utterance, whoSaidIt) {74 return Infer({method: "enumerate"}, function() {75 var cc = comparisonClassPrior(whoSaidIt);76 var statePrior = generateStatePrior(speakerParams[cc]);77 var state = sample(statePrior);78 var thresholdHeavy = thresholdPrior("heavy", stateVals);79 var thresholdLight = thresholdPrior("light", stateVals);80 var S1 = speaker1(state, thresholdHeavy, thresholdLight, cc);81 observe(S1, utterance);82 return state;83 });84}, 10000);8586var ANSWER = pragmaticListener("heavy", "child");
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◉TV<.05 | 3.9e-15 | -0.0000 | · |
Implement the baseline RSA reference game model (Keysar et al.) where the speaker has no uncertainty about the environment.
Setup: - possibleUtterances: all combinations of modifier and type — _.flatten(map(function(modifier){ map(function(type){ return [modifier,type].join(' ').trim(); }, ['apple','fish','cup']); }, ['red','blue','green',''])) - possibleObjects: 9 objects, all combinations of {type: 'apple'|'fish'|'cup', color: 'red'|'blue'|'green'} - exampleContext: { shared: [{type:'apple',color:'red'},{type:'fish',color:'blue'},{type:'cup',color:'green'}], occluded: [{type:'fish',color:'red'}] } - alpha = 3 - uttCost(utt): utt.split(' ').length - uttFitness(utt, object): 0 if the utterance descriptors match the object, -100 otherwise (split by space; if two descriptors, check color and type; if one, check type)
RSA chain (all use method: 'enumerate'): - L0(utt, perceivedContext): uniformDraw over perceivedContext, factor uttFitness, return object - S1(target, knownContext): uniformDraw over possibleUtterances, factor alpha * L0(utt, knownContext).score(target) - uttCost(utt), return utt - L2(utt, perceivedContext): listener only considers shared context; uniform draw over sharedContext = perceivedContext.shared, observe S1(object, sharedContext), return object.color + " " + object.type
Bind these to functions named L0, S1, L2 with the signatures above.
Return the L2 listener distribution after hearing the underinformative utterance 'fish' given exampleContext: L2('fish', exampleContext)
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var possibleUtterances = _.flatten(map(function(modifier) {2 map(function(type) {3 return [modifier, type].join(' ').trim();4 }, ['apple', 'fish', 'cup']);5}, ['red', 'blue', 'green', '']));67var possibleObjects = [8 {type: 'apple', color: 'red'}, {type: 'apple', color: 'blue'}, {type: 'apple', color: 'green'},9 {type: 'fish', color: 'red'}, {type: 'fish', color: 'blue'}, {type: 'fish', color: 'green'},10 {type: 'cup', color: 'red'}, {type: 'cup', color: 'blue'}, {type: 'cup', color: 'green'}]1112var exampleContext = {13 shared: [14 {type: 'apple', color: 'red'},15 {type: 'fish', color: 'blue'},16 {type: 'cup', color: 'green'}17 ],18 occluded: [19 {type: 'fish', color: 'red'}20 ]21};2223var alpha = 3;2425var uttCost = function(utt) {26 return utt.split(' ').length;27}2829var uttFitness = function(utt, object) {30 var descriptors = utt.split(' ');31 if(descriptors.length > 1) {32 return (object.color === descriptors[0] &&33 object.type === descriptors[1]) ? 0 : -100;34 } else {35 return object.type === descriptors[0] ? 0 : -100;36 }37};3839var L0 = cache(function(utt, perceivedContext) {40 return Infer({method: 'enumerate'}, function() {41 var object = uniformDraw(perceivedContext);42 factor(uttFitness(utt, object));43 return object;44 });45});4647var S1 = cache(function(target, knownContext) {48 return Infer({method: 'enumerate'}, function() {49 var utt = uniformDraw(possibleUtterances);50 factor(alpha * L0(utt, knownContext).score(target) - uttCost(utt));51 return utt;52 });53});5455// Listener only considers objects speaker can see (model Keysar is arguing against)56var L2 = cache(function(utt, perceivedContext) {57 var sharedContext = perceivedContext.shared;58 var fullObjSet = sharedContext.concat(perceivedContext.occluded);59 return Infer({method: 'enumerate'}, function() {60 var object = uniformDraw(sharedContext);61 observe(S1(object, sharedContext), utt);62 return object.color + " " + object.type;63 });64});6566console.log("speaker utterance to refer to blue fish");67viz.table(S1({type: 'fish', color: 'blue'}, exampleContext.shared));6869console.log("listener response after hearing (underinformative) 'fish'");70viz.table(L2('fish', exampleContext));7172var ANSWER = (L2('fish', exampleContext));73
1var possibleUtterances = _.flatten(map(function(modifier){2 map(function(type){3 return [modifier, type].join(' ').trim();4 }, ['apple', 'fish', 'cup']);5}, ['red', 'blue', 'green', '']));67var possibleObjects = _.flatten(map(function(color){8 map(function(type){9 return {type: type, color: color};10 }, ['apple', 'fish', 'cup']);11}, ['red', 'blue', 'green']));1213var exampleContext = {14 shared: [{type:'apple',color:'red'},{type:'fish',color:'blue'},{type:'cup',color:'green'}],15 occluded: [{type:'fish',color:'red'}]16};1718var alpha = 3;1920var uttCost = function(utt) {21 return utt.split(' ').length;22};2324var uttFitness = function(utt, object) {25 var descriptors = utt.split(' ');26 if (descriptors.length === 2) {27 var color = descriptors[0];28 var type = descriptors[1];29 return (object.color === color && object.type === type) ? 0 : -100;30 } else {31 var descriptor = descriptors[0];32 return (object.type === descriptor) ? 0 : -100;33 }34};3536var L0 = function(utt, perceivedContext) {37 return Infer({method: 'enumerate'}, function() {38 var object = uniformDraw(perceivedContext);39 factor(uttFitness(utt, object));40 return object;41 });42};4344var S1 = function(target, knownContext) {45 return Infer({method: 'enumerate'}, function() {46 var utt = uniformDraw(possibleUtterances);47 factor(alpha * L0(utt, knownContext).score(target) - uttCost(utt));48 return utt;49 });50};5152var L2 = function(utt, perceivedContext) {53 return Infer({method: 'enumerate'}, function() {54 var sharedContext = perceivedContext.shared;55 var object = uniformDraw(sharedContext);56 observe(S1(object, sharedContext), utt);57 return object.color + ' ' + object.type;58 });59};6061var ANSWER = L2('fish', exampleContext);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | -0.0000 | · |
Implement the context-uncertainty RSA model (Keysar et al.) where the speaker marginalizes over uncertainty about what additional objects the listener might see.
Setup: - possibleUtterances: _.flatten(map(function(modifier){ return map(function(type){ return [modifier,type].join(' ').trim(); }, ['apple','fish','cup']); }, ['red','blue','green',''])) - possibleObjects: 9 objects — all {type,color} pairs for types 'apple'|'fish'|'cup' and colors 'red'|'blue'|'green' - exampleContext: { shared: [{type:'apple',color:'red'},{type:'fish',color:'blue'},{type:'cup',color:'green'}], occluded: [{type:'fish',color:'red'}] } - alpha = 4 - uttCost(utt): utt.split(' ').length / 4 - uttFitness(utt, object): 0 if descriptors match, -100 otherwise (same logic as before)
RSA chain (all use method: 'enumerate'): - L0(utt, perceivedContext): uniformDraw over perceivedContext, factor uttFitness, return object - S1(target, perceivedContext): speaker marginalizes over a hidden additional object — internally runs Infer enumerate over contexts that are perceivedContext.concat(uniformDraw(possibleObjects)), scoring L0 on each, then factors alpha * listener.score(target) - uttCost(utt); return utt - L2(utt, perceivedContext): listener considers both shared and occluded objects as potential referents; uniform draw over fullContext = shared.concat(occluded), observe S1(object, sharedContext), return object.color + " " + object.type
Bind these to functions named L0, S1, L2.
Return the L2 listener distribution after hearing 'fish' given exampleContext: L2('fish', exampleContext)
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var possibleUtterances = _.flatten(map(function(modifier) {2 return map(function(type) {3 return [modifier, type].join(' ').trim();4 }, ['apple', 'fish', 'cup']);5}, ['red', 'blue', 'green', '']));67var possibleObjects = [8 {type: 'apple', color: 'red'}, {type: 'apple', color: 'blue'}, {type: 'apple', color: 'green'},9 {type: 'fish', color: 'red'}, {type: 'fish', color: 'blue'}, {type: 'fish', color: 'green'},10 {type: 'cup', color: 'red'}, {type: 'cup', color: 'blue'}, {type: 'cup', color: 'green'}]1112var exampleContext = {13 shared: [14 {type: 'apple', color: 'red'},15 {type: 'fish', color: 'blue'},16 {type: 'cup', color: 'green'}17 ],18 occluded: [19 {type: 'fish', color: 'red'}20 ]21};2223var alpha = 4;2425var uttCost = function(utt) {26 return utt.split(' ').length/4;27}2829var uttFitness = function(utt, object) {30 var descriptors = utt.split(' ');31 if(descriptors.length > 1) {32 return (object.color === descriptors[0] &&33 object.type === descriptors[1]) ? 0 : -100;34 } else {35 return object.type === descriptors[0] ? 0 : -100;36 }37};3839var L0 = cache(function(utt, perceivedContext) {40 return Infer({method: 'enumerate'}, function() {41 var object = uniformDraw(perceivedContext);42 factor(uttFitness(utt, object))43 return object;44 })45})4647// speaker has uncertainty over what's behind occluded square48// marginalizes over all possibilities49var S1 = cache(function(target, perceivedContext) {50 return Infer({method: 'enumerate'}, function() {51 var utt = uniformDraw(possibleUtterances);52 var listener = Infer({method: 'enumerate', model: function() {53 var context = perceivedContext.concat(uniformDraw(possibleObjects));54 return sample(L0(utt, context));55 }})56 factor(alpha * listener.score(target) - uttCost(utt))57 return utt;58 });59});6061// Listener reasons about S1; assumes they only see what's in shared context62// but could be trying to refer to any of the objects63var L2 = cache(function(utt, perceivedContext) {64 var sharedContext = perceivedContext.shared;65 var fullContext = sharedContext.concat(perceivedContext.occluded);66 return Infer({method: 'enumerate', model: function() {67 var object = uniformDraw(fullContext);68 observe(S1(object, sharedContext), utt);69 return object.color + " " + object.type;70 }});71})7273console.log("speaker utterance to refer to blue fish")74viz.table(S1({type: 'fish', color: 'blue'}, exampleContext.shared))7576console.log("listener response after hearing (underinformative) 'fish'")77viz.table(L2('fish', exampleContext));7879var ANSWER = (L2('fish', exampleContext));80
1var possibleUtterances = _.flatten(map(function(modifier){2 return map(function(type){3 return [modifier, type].join(' ').trim();4 }, ['apple', 'fish', 'cup']);5}, ['red', 'blue', 'green', '']));67var possibleObjects = _.flatten(map(function(type){8 return map(function(color){9 return {type: type, color: color};10 }, ['red', 'blue', 'green']);11}, ['apple', 'fish', 'cup']));1213var exampleContext = {14 shared: [{type:'apple',color:'red'},{type:'fish',color:'blue'},{type:'cup',color:'green'}],15 occluded: [{type:'fish',color:'red'}]16};1718var alpha = 4;1920var uttCost = function(utt) {21 return utt.split(' ').length / 4;22};2324var uttFitness = function(utt, object) {25 var descriptors = utt.split(' ');26 var allMatch = all(function(d){27 return d === object.type || d === object.color;28 }, descriptors);29 return allMatch ? 0 : -100;30};3132var L0 = function(utt, perceivedContext) {33 return Infer({method: 'enumerate'}, function(){34 var object = uniformDraw(perceivedContext);35 factor(uttFitness(utt, object));36 return object;37 });38};3940var S1 = function(target, perceivedContext) {41 return Infer({method: 'enumerate'}, function(){42 var utt = uniformDraw(possibleUtterances);43 // marginalize over uncertainty about hidden additional object44 var listenerDist = Infer({method: 'enumerate'}, function(){45 var hiddenObj = uniformDraw(possibleObjects);46 var expandedContext = perceivedContext.concat([hiddenObj]);47 var l0 = L0(utt, expandedContext);48 var obj = sample(l0);49 return obj;50 });51 factor(alpha * listenerDist.score(target) - uttCost(utt));52 return utt;53 });54};5556var L2 = function(utt, context) {57 var sharedContext = context.shared;58 var fullContext = context.shared.concat(context.occluded);59 return Infer({method: 'enumerate'}, function(){60 var object = uniformDraw(fullContext);61 var s1 = S1(object, sharedContext);62 observe(s1, utt);63 return object.color + ' ' + object.type;64 });65};6667var ANSWER = L2('fish', exampleContext);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | -0.0000 | · |
Implement a scope ambiguity RSA model for 'every-not' utterances in WebPPL. There are numHorses=2 horses. The world state is the count of true (red) horses (0, 1, or 2). statePrior() samples via: a recursive stateMaker function that flips a baserate=0.5 coin for each of numHorses horses and tallies up the number of true values. Utterances are ['null','every-not'] with uniform prior. Cost is uniform: cost(utterance)=1. Scopes are ['surface','inverse'] with categorical([.5,.5]). meaning(utterance,state,scope): for 'every-not', surface scope means state==0 (not any horse is red), inverse scope means state<numHorses (not all horses are red); 'null' is always true. QUDs are ['how many?','all red?','none red?'] with uniform prior. QUDFun(QUD,state): 'all red?' returns state==numHorses, 'none red?' returns state==0, else state. Literal listener (L0, cached) takes (utterance,scope,QUD): uniformDraw(states=[0,1,2]), apply QUDFun, condition on meaning, return qState. Speaker (S1, cached, alpha=1) takes (scope,state,QUD): utterancePrior(), factor by alpha*(L0.score(QUDFun(QUD,state)) - cost(utterance)). Pragmatic listener (L1, cached) takes utterance: sample statePrior, scopePrior, QUDPrior, observe S1(scope,state,QUD), return state. Pragmatic speaker (S2, cached) takes state: utterancePrior(), factor by L1(utterance).score(state). Return the pragmatic speaker's distribution over utterances for state=1 (one horse is red — the 'not all' world state).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// helper function to tally up the state2var numTrue = function(state) {3 var fun = function(x) {4 x ? 1 : 05 }6 return sum(map(fun,state))7}89// possible utterances10var utterances = ["null","every-not"];11var utterancePrior = function() {12 uniformDraw(utterances)13}14// uniform utterance cost15var cost = function(utterance) {16 return 117}1819// possible world states20var numHorses = 221var states = [0,1,2]22var baserate = 0.5 // change to manipulate prior on world states23var stateMaker = function(numHorses,stateSoFar) {24 if (numHorses == 0) {25 return stateSoFar26 } else {27 var newHorse = flip(baserate)28 var newState = stateSoFar.concat([newHorse])29 return stateMaker(numHorses - 1, newState)30 }31}32var statePrior = function() {33 return numTrue(stateMaker(numHorses,[]))34}3536// possible scope interpretations37var scopes = ["surface", "inverse"]38var scopePrior = function(){39 return categorical([.5,.5],scopes) // change to manipulate prior on scope interpretations40}414243// meaning function44var meaning = function(utterance, state, scope) {45 return utterance == "every-not" ?46 scope == "surface" ? state == 0 :47 state < numHorses :48 true;49};5051// possible QUDs52var QUDs = ["how many?","all red?","none red?"];53var QUDPrior = function() {54 uniformDraw(QUDs);55 // categorical([.05,.05,.9],QUDs) // change to manipulate prior on QUDs56}57var QUDFun = function(QUD,state) {58 QUD == "all red?" ? state == numHorses :59 QUD == "none red?" ? state == 0 :60 state;61};6263// Literal listener (L0)64var literalListener = cache(function(utterance,scope,QUD) {65 Infer({model: function(){66 var state = uniformDraw(states);67 var qState = QUDFun(QUD,state)68 condition(meaning(utterance,state,scope));69 return qState;70 }});71});7273var alpha = 17475// Speaker (S1)76var speaker = cache(function(scope, state, QUD) {77 return Infer({model: function(){78 var utterance = utterancePrior()79 var qState = QUDFun(QUD, state)80 factor(alpha*(literalListener(utterance,scope,QUD).score(qState)81 - cost(utterance)))82 return utterance83 }})84})8586// Pragmatic listener (L1)87var pragmaticListener = cache(function(utterance) {88 Infer({model: function(){89 var state = statePrior();90 var scope = scopePrior();91 var QUD = QUDPrior();92 observe(speaker(scope,state,QUD),utterance);93 return state94 }});95});9697// Pragmatic speaker (S2)98var pragmaticSpeaker = cache(function(state) {99 Infer({model: function(){100 var utterance = utterancePrior();101 factor(pragmaticListener(utterance).score(state))102 return utterance103 }})104})105106// A speaker decides whether to endorse the ambiguous utterance as a107// description of the not-all world state108display(pragmaticSpeaker(1))109110var ANSWER = (pragmaticSpeaker(1));111
1var numHorses = 2;2var baserate = 0.5;3var alpha = 1;45var states = [0, 1, 2];6var utterances = ['null', 'every-not'];7var scopes = ['surface', 'inverse'];8var quds = ['how many?', 'all red?', 'none red?'];910var stateMaker = function(n, acc) {11 if (n === 0) return acc;12 var coin = flip(baserate);13 return stateMaker(n - 1, acc + (coin ? 1 : 0));14};1516var statePrior = function() {17 return stateMaker(numHorses, 0);18};1920var utterancePrior = function() {21 return uniformDraw(utterances);22};2324var scopePrior = function() {25 return categorical({vs: scopes, ps: [0.5, 0.5]});26};2728var qudPrior = function() {29 return uniformDraw(quds);30};3132var cost = function(utterance) {33 return 1;34};3536var meaning = function(utterance, state, scope) {37 if (utterance === 'every-not') {38 if (scope === 'surface') {39 return state === 0;40 } else {41 return state < numHorses;42 }43 } else {44 return true;45 }46};4748var QUDFun = function(qud, state) {49 if (qud === 'all red?') return state === numHorses;50 if (qud === 'none red?') return state === 0;51 return state;52};5354var L0 = mem(function(utterance, scope, qud) {55 return Infer({method: 'enumerate'}, function() {56 var state = uniformDraw(states);57 var qState = QUDFun(qud, state);58 condition(meaning(utterance, state, scope));59 return qState;60 });61});6263var S1 = mem(function(scope, state, qud) {64 return Infer({method: 'enumerate'}, function() {65 var utterance = utterancePrior();66 var qState = QUDFun(qud, state);67 factor(alpha * (L0(utterance, scope, qud).score(qState) - cost(utterance)));68 return utterance;69 });70});7172var L1 = mem(function(utterance) {73 return Infer({method: 'enumerate'}, function() {74 var state = statePrior();75 var scope = scopePrior();76 var qud = qudPrior();77 observe(S1(scope, state, qud), utterance);78 return state;79 });80});8182var S2 = mem(function(state) {83 return Infer({method: 'enumerate'}, function() {84 var utterance = utterancePrior();85 factor(L1(utterance).score(state));86 return utterance;87 });88});8990var ANSWER = S2(1);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
An RSA scope ambiguity model for "two-not" utterances in a 1-of-2 context (2 horses, 0/1/2 fed). A pragmatic speaker at world state 1 (only 1 of 2 horses fed) decides whether to endorse "two-not".
utterances = ["null", "two-not"]. utterancePrior: uniformDraw(utterances). cost(utterance): 1. numHorses = 2. states = [0,1,2]. baserate = 0.5. stateMaker(numHorses, stateSoFar): recursive, flip(baserate) for each horse, concatenates boolean array. statePrior: numTrue(stateMaker(numHorses,[])) — returns count 0,1,2. scopePrior: categorical([0.5,0.5], ["surface","inverse"]). meaning(utterance, state, scope): two-not ? surface ? state==0 : state<numHorses : true. QUDs = ["many?","all?","none?"]. QUDPrior: uniformDraw(QUDs). QUDFun(QUD,state): all?->state==numHorses, none?->state==0, else state. literalListener(utterance, scope, QUD): Infer over uniformDraw(states), conditions on meaning, returns QUDFun(QUD,state). alpha = 1. speaker(scope, state, QUD): Infer, factor(alpha*(literalListener(utterance,scope,QUD).score(QUDFun(QUD,state))-cost(utterance))). pragmaticListener(utterance): Infer, samples state/scope/QUD, observes speaker(scope,state,QUD). pragmaticSpeaker(state): Infer, utterancePrior, factor(pragmaticListener(utterance).score(state)), returns utterance.
Compute pragmaticSpeaker(1). Return distribution over utterances ["null", "two-not"] when 1 of 2 horses was fed.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// tally up the state2var numTrue = function(state) {3 var fun = function(x) {4 x ? 1 : 05 }6 return sum(map(fun,state))7}89// possible utterances10var utterances = ["null","two-not"];11var utterancePrior = function() {12 uniformDraw(utterances)13}14// uniform utterance cost15var cost = function(utterance) {16 return 117}1819// possible world states20var numHorses = 221var states = [0,1,2]22var baserate = 0.5 // change to manipulate prior on world states23var stateMaker = function(numHorses,stateSoFar) {24 if (numHorses == 0) {25 return stateSoFar26 } else {27 var newHorse = flip(baserate)28 var newState = stateSoFar.concat([newHorse])29 return stateMaker(numHorses - 1, newState)30 }31}32var statePrior = function() {33 return numTrue(stateMaker(numHorses,[]))34}3536// possible scope interpretations37var scopes = ["surface", "inverse"]38var scopePrior = function(){39 return categorical([.5,.5],scopes) // change to manipulate prior on scope interpretations40}4142// meaning function43var meaning = function(utterance, state, scope) {44 if (utterance == "two-not") {45 if (scope == "surface") {46 return state == 047 } else {48 return state < numHorses49 }50 } else {51 return true;52 }53};5455// possible QUDs56var QUDs = ["many?","all?","none?"];57var QUDPrior = function() {58 uniformDraw(QUDs);59 // categorical([.05,.9,.05],QUDs) // change to manipulate prior on QUDs60}61var QUDFun = function(QUD,state) {62 QUD == "all?" ? state == numHorses :63 QUD == "none?" ? state == 0 :64 state;65};6667// Literal listener (L0)68var literalListener = cache(function(utterance,scope,QUD) {69 Infer({model: function(){70 var state = uniformDraw(states);71 var qState = QUDFun(QUD,state)72 condition(meaning(utterance,state,scope));73 return qState;74 }});75});7677var alpha = 17879// Speaker (S1)80var speaker = cache(function(scope, state, QUD) {81 return Infer({model: function(){82 var utterance = utterancePrior()83 var qState = QUDFun(QUD, state)84 factor(alpha*(literalListener(utterance,scope,QUD).score(qState)85 - cost(utterance)))86 return utterance87 }})88})8990// Pragmatic listener (L1)91var pragmaticListener = cache(function(utterance) {92 Infer({model: function(){93 var state = statePrior();94 var scope = scopePrior();95 var QUD = QUDPrior();96 observe(speaker(scope,state,QUD),utterance);97 return state98 }});99});100101// Pragmatic speaker (S2)102var pragmaticSpeaker = cache(function(state) {103 Infer({model: function(){104 var utterance = utterancePrior();105 factor(pragmaticListener(utterance).score(state))106 return utterance107 }})108})109110// A speaker decides whether to endorse the ambiguous utterance as a111// description of the not-all world state112display(pragmaticSpeaker(1))113114var ANSWER = (pragmaticSpeaker(1));115
1var numHorses = 2;2var states = [0, 1, 2];3var baserate = 0.5;4var utterances = ["null", "two-not"];5var QUDs = ["many?", "all?", "none?"];6var alpha = 1;78var cost = function(utterance) { return 1; };910var utterancePrior = function() { return uniformDraw(utterances); };1112var stateMaker = function(numHorses, stateSoFar) {13 if (numHorses === 0) {14 return stateSoFar;15 } else {16 var horseFed = flip(baserate);17 return stateMaker(numHorses - 1, stateSoFar.concat([horseFed]));18 }19};2021var numTrue = function(arr) {22 return reduce(function(x, acc) { return x ? acc + 1 : acc; }, 0, arr);23};2425var statePrior = function() {26 return numTrue(stateMaker(numHorses, []));27};2829var scopePrior = function() {30 return categorical([0.5, 0.5], ["surface", "inverse"]);31};3233var QUDPrior = function() {34 return uniformDraw(QUDs);35};3637var QUDFun = function(QUD, state) {38 if (QUD === "all?") {39 return state === numHorses;40 } else if (QUD === "none?") {41 return state === 0;42 } else {43 return state;44 }45};4647var meaning = function(utterance, state, scope) {48 if (utterance === "two-not") {49 if (scope === "surface") {50 return state === 0;51 } else {52 return state < numHorses;53 }54 } else {55 return true;56 }57};5859var literalListener = mem(function(utterance, scope, QUD) {60 return Infer({method: 'enumerate'}, function() {61 var state = uniformDraw(states);62 condition(meaning(utterance, state, scope));63 return QUDFun(QUD, state);64 });65});6667var speaker = mem(function(scope, state, QUD) {68 return Infer({method: 'enumerate'}, function() {69 var utterance = utterancePrior();70 var qudVal = QUDFun(QUD, state);71 var score = literalListener(utterance, scope, QUD).score(qudVal);72 factor(alpha * (score - cost(utterance)));73 return utterance;74 });75});7677var pragmaticListener = mem(function(utterance) {78 return Infer({method: 'enumerate'}, function() {79 var state = statePrior();80 var scope = scopePrior();81 var QUD = QUDPrior();82 observe(speaker(scope, state, QUD), utterance);83 return state;84 });85});8687var pragmaticSpeaker = function(state) {88 return Infer({method: 'enumerate'}, function() {89 var utterance = utterancePrior();90 observe(pragmaticListener(utterance), state);91 return utterance;92 });93};9495var ANSWER = pragmaticSpeaker(1);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
Implement the Kao & Goodman (2015) irony RSA model for weather state inference.
Context: The model is set in California where good weather is likely. Speakers may be ironic — saying 'terrible' when the weather is actually amazing.
Setup: - states = ['terrible', 'ok', 'amazing'] - statePrior(): categorical([1, 50, 50], states) // California prior: ok and amazing are likely - valencePrior(state): if 'terrible' flip(0.99)?-1:1; if 'ok' flip(0.5)?-1:1; if 'amazing' flip(0.01)?-1:1 - arousals = ['low', 'high'] - arousalPrior(state): if 'terrible' categorical([0.1,0.9],arousals); if 'ok' categorical([0.9,0.1],arousals); if 'amazing' categorical([0.1,0.9],arousals) - goals = ['goalState', 'goalValence', 'goalArousal'] - goalPrior(): categorical([1,1,1], goals) - utterances = states; utterancePrior(): uniformDraw(utterances) - literalInterpretation(utterance, state): utterance === state - goalState(goal, state, valence, arousal): returns state / valence / arousal based on goal - literalListener(utterance, goal): Infer over uniformDraw(states), valence=valencePrior(state), arousal=arousalPrior(state); condition on literalInterpretation; return goalState(goal,state,valence,arousal) - speaker(state, valence, arousal, goal): Infer over utterancePrior(); factor 1 * literalListener(utt,goal).score(goalState(goal,state,valence,arousal)); return utt - pragmaticListener(utterance): Infer over statePrior(), valencePrior, arousalPrior, goalPrior(); observe speaker(state,valence,arousal,goal) given utterance; return {state, valence, arousal, goal}
Bind pragmaticListener to a function of that exact name.
Return the pragmaticListener distribution for the utterance 'terrible': pragmaticListener('terrible')
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// There are three possible states the weather could be in:2// terrible, ok, or amazing3var states = ['terrible', 'ok', 'amazing']45// Since we are in California, the prior over these states6// are the following. Once could also imagine this being7// the prior in a certain context, e.g. when it's clearly8// sunny and nice out.9var statePrior = function() {10 categorical([1, 50, 50], states)11}1213// Valence prior defined in terms of negative valence.14// If the current state is terrible, it's extremely likely15// that the valence associated is negative. If it's ok, then16// the valence could be negative or positive with equal17// probability.18var valencePrior = function(state) {19 state === "terrible" ? flip(0.99) ? -1 : 1 :20 state === "ok" ? flip(0.5) ? -1 : 1 :21 state === "amazing" ? flip(0.01) ? -1 : 1 :22 true23}2425// Define binary arousals (could model as continuous).26var arousals = ["low", "high"]2728// Define goals and goal priors. Could want to communicate state of the world,29// valence about it, or arousal (intensity of feeling) about it.30var goals = ["goalState", "goalValence", "goalArousal"]3132var goalPrior = function() {33 categorical([1, 1, 1], goals)34}3536// Assume possible utterances are identical to possible states37var utterances = states3839// Assume cost of utterances is uniform.40var utterancePrior = function() {41 uniformDraw(utterances)42}4344// Sample arousal given a state.45var arousalPrior = function(state) {46 state === "terrible" ? categorical([0.1, 0.9], arousals) :47 state === "ok" ? categorical([0.9, 0.1], arousals) :48 state === "amazing" ? categorical([0.1, 0.9], arousals) :49 true50}5152// Literal interpretation is just whether utterance equals state53var literalInterpretation = function(utterance, state) {54 utterance === state55}5657// A speaker's goal is satisfied if the listener infers the correct58// and relevant information.59var goalState = function(goal, state, valence, arousal) {60 goal === "goalState" ? state :61 goal === "goalValence" ? valence :62 goal === "goalArousal" ? arousal :63 true64}6566// Define a literal listener67var literalListener = function(utterance, goal) {68 Infer({model: function(){69 var state = uniformDraw(states)70 var valence = valencePrior(state)71 var arousal = arousalPrior(state)72 condition(literalInterpretation(utterance,state))73 return goalState(goal, state, valence, arousal)74 }})75}7677// Define a speaker78var speaker = function(state, valence, arousal, goal) {79 Infer({model: function(){80 var utterance = utterancePrior()81 factor(1 * literalListener(utterance,82 goal).score(goalState(goal,83 state,84 valence,85 arousal)))86 return utterance87 }})88}8990// Define a pragmatic listener91var pragmaticListener = function(utterance) {92 Infer({model: function(){93 var state = statePrior()94 var valence = valencePrior(state)95 var arousal = arousalPrior(state)96 var goal = goalPrior()97 observe(speaker(state, valence, arousal, goal),utterance)98 return {state, valence, arousal, goal}99 }})100}101102viz.table(literalListener("terrible", "goalState"))103viz.table(speaker("terrible", -1, "high", "goalValence"))104viz.table(pragmaticListener("terrible"))105106var ANSWER = (pragmaticListener('terrible'));107
1var states = ['terrible', 'ok', 'amazing'];23var statePrior = function() {4 return categorical([1, 50, 50], states);5};67var valencePrior = function(state) {8 if (state === 'terrible') {9 return flip(0.99) ? -1 : 1;10 } else if (state === 'ok') {11 return flip(0.5) ? -1 : 1;12 } else if (state === 'amazing') {13 return flip(0.01) ? -1 : 1;14 }15};1617var arousals = ['low', 'high'];1819var arousalPrior = function(state) {20 if (state === 'terrible') {21 return categorical([0.1, 0.9], arousals);22 } else if (state === 'ok') {23 return categorical([0.9, 0.1], arousals);24 } else if (state === 'amazing') {25 return categorical([0.1, 0.9], arousals);26 }27};2829var goals = ['goalState', 'goalValence', 'goalArousal'];3031var goalPrior = function() {32 return categorical([1, 1, 1], goals);33};3435var utterances = states;3637var utterancePrior = function() {38 return uniformDraw(utterances);39};4041var literalInterpretation = function(utterance, state) {42 return utterance === state;43};4445var goalState = function(goal, state, valence, arousal) {46 if (goal === 'goalState') {47 return state;48 } else if (goal === 'goalValence') {49 return valence;50 } else if (goal === 'goalArousal') {51 return arousal;52 }53};5455var literalListener = mem(function(utterance, goal) {56 return Infer({method: 'enumerate'}, function() {57 var state = uniformDraw(states);58 var valence = valencePrior(state);59 var arousal = arousalPrior(state);60 condition(literalInterpretation(utterance, state));61 return goalState(goal, state, valence, arousal);62 });63});6465var speaker = mem(function(state, valence, arousal, goal) {66 return Infer({method: 'enumerate'}, function() {67 var utt = utterancePrior();68 factor(1 * literalListener(utt, goal).score(goalState(goal, state, valence, arousal)));69 return utt;70 });71});7273var pragmaticListener = function(utterance) {74 return Infer({method: 'enumerate'}, function() {75 var state = statePrior();76 var valence = valencePrior(state);77 var arousal = arousalPrior(state);78 var goal = goalPrior();79 observe(speaker(state, valence, arousal, goal), utterance);80 return {state: state, valence: valence, arousal: arousal, goal: goal};81 });82};8384var ANSWER = pragmaticListener('terrible');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
Implement an RSA model of quantifier scope ambiguity for English (Experiment 1: surface scope context) in WebPPL. States are [0,1,2] with categorical({vs:[0,1,2],ps:[1,1,1]}). Utterances are ['null','not-two'] with categorical({vs:['null','not-two'],ps:[1,1]}). cost(utterance)=1 for all utterances. Scopes are ['surface','inverse'] with categorical({vs:['surface','inverse'],ps:[1,1]}). meaning(utterance,state,scope): for 'not-two', surface scope means state==0 || state==1; inverse scope means state==0; 'null' is always true. QUDs are ['how many?','all red?','none red?'] with uniform prior. QUDFun(QUD,state): 'all red?' returns state==2, 'none red?' returns state==0, else state. Literal listener (L0, cached) takes (utterance,scope,QUD): uniformDraw(states), apply QUDFun, condition on meaning, return qState. Speaker (S, cached, alpha=1) takes (scope,state,QUD): utterancePrior(), factor by alpha*(L0(utterance,scope,QUD).score(QUDFun(QUD,state))-cost(utterance)). Pragmatic listener (L1, cached) takes utterance: statePrior, scopePrior, QUDPrior, observe S(scope,state,QUD). Pragmatic speaker (S2, cached) takes state: utterancePrior, factor by L1(utterance).score(state). Return the pragmatic speaker's distribution over utterances for state=1 (one rabbit was fed — the surface-scope-true, inverse-scope-false state).
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// Here is the code for English Expt 1 (surface scope)2//different qud prior, different state prior, access to alternative utterances34// Here is the code for the quantifier scope model56// possible utterances7var utterances = ["null","not-two"];89var utterancePrior = function() {10 categorical({vs:["null","not-two"],ps:[1,1]})11}1213var cost = function(utterance) {14 return 115}1617// possible world states18var states = [0,1,2];19var statePrior = function() {20 categorical({vs:[0,1,2],ps:[1,1,1]})21}2223// possible scopes24var scopePrior = function(){25 return categorical({vs:["surface", "inverse"],ps:[1,1]})26}2728var meaning = function(utterance, state, scope) {29 //if utterance == none:30 //return state==031 //else:32 //elif utternace == nottwo:33 //if scope == surface:34 //return state == 0 / state==135 //else:36 //return state == 037 //else:38 //return true3940 return utterance == "not-two" ?41 scope == "surface" ? (state == 0 | state ==1):42 state == 0 :43 true;44};454647// QUDs48var QUDs = ["how many?","all red?","none red?"];49var QUDPrior = function() {50 uniformDraw(QUDs);51}52var QUDFun = function(QUD,state) {53 QUD == "all red?" ? state == 2 :54 QUD == "none red?" ? state == 0 :55 state;56};5758// Literal listener (L0)59var literalListener = cache(function(utterance,scope,QUD) {60 Infer({model: function(){61 var state = uniformDraw(states);62 var qState = QUDFun(QUD,state)63 condition(meaning(utterance,state,scope));64 return qState;65 }});66});6768var alpha = 16970// Speaker (S)71var speaker = cache(function(scope, state, QUD) {72 return Infer({model: function(){73 var utterance = utterancePrior()74 var qState = QUDFun(QUD, state)75 factor(alpha*(literalListener(utterance,scope,QUD).score(qState)76 - cost(utterance)))77 return utterance78 }})79})8081// Pragmatic listener (L1)82var pragmaticListener = cache(function(utterance) {83 Infer({model: function(){84 var state = statePrior();85 var scope = scopePrior();86 var QUD = QUDPrior();87 observe(speaker(scope,state,QUD),utterance);88 return state89 }});90});9192// Pragmatic speaker (S2)93var pragmaticSpeaker = cache(function(state) {94 Infer({model: function(){95 var utterance = utterancePrior();96 factor(pragmaticListener(utterance).score(state))97 return utterance98 }})99})100101102103// A speaker decides whether to endorse the ambiguous utterance as a104// description of the not-all world state105viz.table(pragmaticSpeaker(1))106viz(pragmaticSpeaker(1))107//literalListener("surface", 2, "all red?")108109var ANSWER = (pragmaticSpeaker(1));110
1var states = [0, 1, 2];2var utterances = ['null', 'not-two'];3var scopes = ['surface', 'inverse'];4var quds = ['how many?', 'all red?', 'none red?'];56var statePrior = function() {7 return categorical({vs: states, ps: [1, 1, 1]});8};910var utterancePrior = function() {11 return categorical({vs: utterances, ps: [1, 1]});12};1314var scopePrior = function() {15 return categorical({vs: scopes, ps: [1, 1]});16};1718var qudPrior = function() {19 return categorical({vs: quds, ps: [1, 1, 1]});20};2122var cost = function(utterance) {23 return 1;24};2526var meaning = function(utterance, state, scope) {27 if (utterance === 'not-two') {28 if (scope === 'surface') {29 return state === 0 || state === 1;30 } else {31 return state === 0;32 }33 } else {34 return true;35 }36};3738var QUDFun = function(qud, state) {39 if (qud === 'all red?') {40 return state === 2;41 } else if (qud === 'none red?') {42 return state === 0;43 } else {44 return state;45 }46};4748var alpha = 1;4950var L0 = mem(function(utterance, scope, qud) {51 return Infer({method: 'enumerate'}, function() {52 var state = statePrior();53 var qState = QUDFun(qud, state);54 condition(meaning(utterance, state, scope));55 return qState;56 });57});5859var S = mem(function(scope, state, qud) {60 return Infer({method: 'enumerate'}, function() {61 var utterance = utterancePrior();62 var qState = QUDFun(qud, state);63 factor(alpha * (L0(utterance, scope, qud).score(qState) - cost(utterance)));64 return utterance;65 });66});6768var L1 = mem(function(utterance) {69 return Infer({method: 'enumerate'}, function() {70 var state = statePrior();71 var scope = scopePrior();72 var qud = qudPrior();73 observe(S(scope, state, qud), utterance);74 return state;75 });76});7778var S2 = mem(function(state) {79 return Infer({method: 'enumerate'}, function() {80 var utterance = utterancePrior();81 factor(L1(utterance).score(state));82 return utterance;83 });84});8586var ANSWER = S2(1);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
An RSA scope ambiguity model for English "not-two" utterances, Experiment 2 (4-object inverse scope context). 4 objects, states 0-4. A pragmatic speaker at state 2 (2 of 4 rabbits fed) decides whether to endorse "not-two".
utterances = ["null", "not-two"]. utterancePrior: categorical({vs:["null","not-two"], ps:[1,10]}) — strong prior toward "not-two". cost(utterance): both->1. states = [0,1,2,3,4]. statePrior: categorical({vs:[0,1,2,3,4], ps:[1,1,1,1,1]}). scopePrior: categorical({vs:["surface","inverse"], ps:[1,1]}). meaning(utterance, state, scope): not-two ? surface ? state<2 : state<3 : true. QUDs = ["how many?","all red?","none red?"]. QUDPrior: categorical({vs:[...], ps:[1,1,1]}). QUDFun(QUD,state): all red?->state==4, none red?->state==0, else state. literalListener(utterance, scope, QUD): Infer over uniformDraw(states), conditions on meaning, returns QUDFun(QUD,state). alpha = 1. speaker(scope, state, QUD): Infer, factor(alpha*(L0.score(QUDFun(QUD,state))-cost(utterance))). pragmaticListener(utterance): Infer, samples state/scope/QUD, observes speaker. pragmaticSpeaker(state): Infer, factor(pragmaticListener(utterance).score(state)), returns utterance.
Compute pragmaticSpeaker(2). Return distribution over utterances ["null", "not-two"] when 2 of 4 rabbits were fed.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// Here is the code for English Expt 2 (inverse scope)2//different qud prior, different state prior, access to alternative utterances34// Here is the code for the quantifier scope model56// possible utterances7var utterances = ["null","not-two"];89var utterancePrior = function() {10 categorical({vs:["null","not-two"],ps:[1,10]})11}1213var cost = function(utterance) {14 return utterance == "not-two" ? 1 :15 116}1718// possible world states19var states = [0,1,2,3,4];20var statePrior = function() {21 categorical({vs:[0,1,2,3,4],ps:[1,1,1,1,1]})22}2324// possible scopes25var scopePrior = function(){26 return categorical({vs:["surface", "inverse"],ps:[1,1]})27}2829// meaning function30var meaning = function(utterance, state, scope) {31 return utterance == "not-two" ?32 scope == "surface" ? (state < 2):33 (state < 3) :34 true;35};3637// QUDs38var QUDs = ["how many?","all red?","none red?"];39var QUDPrior = function() {40 categorical({vs:["how many?","all red?","none red?"],ps:[1,1,1]})41 //uniformDraw(QUDs);42}4344var QUDFun = function(QUD,state) {45 QUD == "all red?" ? state == 4 :46 QUD == "none red?" ? state == 0 :47 state;48};4950// Literal listener (L0)51var literalListener = cache(function(utterance,scope,QUD) {52 Infer({model: function(){53 var state = uniformDraw(states);54 var qState = QUDFun(QUD,state)55 condition(meaning(utterance,state,scope));56 return qState;57 }});58});5960var alpha = 16162// Speaker (S)63var speaker = cache(function(scope, state, QUD) {64 return Infer({model: function(){65 var utterance = utterancePrior()66 var qState = QUDFun(QUD, state)67 factor(alpha*(literalListener(utterance,scope,QUD).score(qState)68 - cost(utterance)))69 return utterance70 }})71})7273// Pragmatic listener (L1)74var pragmaticListener = cache(function(utterance) {75 Infer({model: function(){76 var state = statePrior();77 var scope = scopePrior();78 var QUD = QUDPrior();79 observe(speaker(scope,state,QUD),utterance);80 return state81 }});82});8384// Pragmatic speaker (S2)85var pragmaticSpeaker = cache(function(state) {86 Infer({model: function(){87 var utterance = utterancePrior();88 factor(pragmaticListener(utterance).score(state))89 return utterance90 }})91})9293// A speaker decides whether to endorse the ambiguous utterance as a94// description of the not-all world state95//viz.table(pragmaticSpeaker(0))96viz.table(pragmaticSpeaker(2))97//viz.table(pragmaticSpeaker(2))98//literalListener("surface", 2, "all red?")99100var ANSWER = (pragmaticSpeaker(2));101
1var states = [0, 1, 2, 3, 4];2var utterances = ["null", "not-two"];3var scopes = ["surface", "inverse"];4var QUDs = ["how many?", "all red?", "none red?"];56var statePrior = function() {7 return categorical({vs: states, ps: [1, 1, 1, 1, 1]});8};910var utterancePrior = function() {11 return categorical({vs: utterances, ps: [1, 10]});12};1314var scopePrior = function() {15 return categorical({vs: scopes, ps: [1, 1]});16};1718var QUDPrior = function() {19 return categorical({vs: QUDs, ps: [1, 1, 1]});20};2122var cost = function(utterance) {23 return 1;24};2526var meaning = function(utterance, state, scope) {27 if (utterance === "not-two") {28 if (scope === "surface") {29 return state !== 2;30 } else {31 return state < 3;32 }33 } else {34 return true;35 }36};3738var QUDFun = function(QUD, state) {39 if (QUD === "all red?") {40 return state === 4;41 } else if (QUD === "none red?") {42 return state === 0;43 } else {44 return state;45 }46};4748var alpha = 1;4950var literalListener = mem(function(utterance, scope, QUD) {51 return Infer({method: 'enumerate'}, function() {52 var state = uniformDraw(states);53 condition(meaning(utterance, state, scope));54 return QUDFun(QUD, state);55 });56});5758var speaker = mem(function(scope, state, QUD) {59 return Infer({method: 'enumerate'}, function() {60 var utterance = utterancePrior();61 var L0 = literalListener(utterance, scope, QUD);62 factor(alpha * (L0.score(QUDFun(QUD, state)) - cost(utterance)));63 return utterance;64 });65});6667var pragmaticListener = mem(function(utterance) {68 return Infer({method: 'enumerate'}, function() {69 var state = statePrior();70 var scope = scopePrior();71 var QUD = QUDPrior();72 var S1 = speaker(scope, state, QUD);73 observe(S1, utterance);74 return state;75 });76});7778var pragmaticSpeaker = function(state) {79 return Infer({method: 'enumerate'}, function() {80 var utterance = utterancePrior();81 var PL = pragmaticListener(utterance);82 factor(PL.score(state));83 return utterance;84 });85};8687var ANSWER = pragmaticSpeaker(2);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◉TV<.05 | 0.038 | 0.0095 | · |
An RSA scope ambiguity model for Mandarin "not-two" utterances, Experiment 1 (2-object surface-scope biased context). The model includes a third utterance "none" and a strongly surface-scope biased scopePrior. A pragmatic speaker at state 1 (1 of 2 rabbits fed) decides whether to endorse "not-two".
utterances = ["null", "not-two", "none"]. utterancePrior: categorical({vs:["null","not-two","none"], ps:[1,1,1]}). cost(utterance): all->1. states = [0,1,2]. statePrior: categorical({vs:[0,1,2], ps:[1,1,1]}). scopePrior: categorical({vs:["surface","inverse"], ps:[100,1]}) — strongly biased toward surface scope. meaning(utterance, state, scope): none->state==0, not-two ? surface ? state<2 : state==0 : true. QUDs = ["how many?","all red?","none red?"]. QUDPrior: categorical({vs:[...], ps:[1,1,1]}). QUDFun(QUD,state): all red?->state==2, none red?->state==0, else state. literalListener(utterance, scope, QUD): Infer over uniformDraw(states), conditions on meaning, returns QUDFun(QUD,state). alpha = 1. speaker(scope, state, QUD): Infer, factor(alpha*(L0.score(QUDFun(QUD,state))-cost(utterance))). pragmaticListener(utterance): Infer, samples state/scope/QUD, observes speaker. pragmaticSpeaker(state): Infer, factor(pragmaticListener(utterance).score(state)), returns utterance.
Compute pragmaticSpeaker(1). Return distribution over utterances ["null", "not-two", "none"] when 1 of 2 rabbits was fed.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// Here is the code of Chinese model for Expt 1.2//different qud prior, different state prior, access to alternative utterances34// Here is the code for the quantifier scope model56// possible utterances7var utterances = ["null","not-two","none"];89var utterancePrior = function() {10 categorical({vs:["null","not-two","none"],ps:[1,1,1]})11}1213var cost = function(utterance) {14 return utterance == "not-two" ? 1 :15 utterance == 'none'? 1 :16 117}1819// possible world states20var states = [0,1,2];21var statePrior = function() {22 categorical({vs:[0,1,2],ps:[1,1,1]})23}2425// possible scopes26var scopePrior = function(){27 return categorical({vs:["surface", "inverse"],ps:[100,1]})28}2930// meaning function31var meaning = function(utterance, state, scope) {32 return utterance == "none"? state == 0:33 utterance == "not-two" ?34 scope == "surface" ? (state < 2):35 (state == 0) :36 true;37};3839// QUDs40var QUDs = ["how many?","all red?","none red?"];41var QUDPrior = function() {42 categorical({vs:["how many?","all red?","none red?"],ps:[1,1,1]})43 //uniformDraw(QUDs);44}45var QUDFun = function(QUD,state) {46 QUD == "all red?" ? state == 2 :47 QUD == "none red?" ? state == 0 :48 state;49};5051// Literal listener (L0)52var literalListener = cache(function(utterance,scope,QUD) {53 Infer({model: function(){54 var state = uniformDraw(states);55 var qState = QUDFun(QUD,state)56 condition(meaning(utterance,state,scope));57 return qState;58 }});59});6061var alpha = 16263// Speaker (S)64var speaker = cache(function(scope, state, QUD) {65 return Infer({model: function(){66 var utterance = utterancePrior()67 var qState = QUDFun(QUD, state)68 factor(alpha*(literalListener(utterance,scope,QUD).score(qState)69 - cost(utterance)))70 return utterance71 }})72})7374// Pragmatic listener (L1)75var pragmaticListener = cache(function(utterance) {76 Infer({model: function(){77 var state = statePrior();78 var scope = scopePrior();79 var QUD = QUDPrior();80 observe(speaker(scope,state,QUD),utterance);81 return state82 }});83});8485// Pragmatic speaker (S2)86var pragmaticSpeaker = cache(function(state) {87 Infer({model: function(){88 var utterance = utterancePrior();89 factor(pragmaticListener(utterance).score(state))90 return utterance91 }})92})9394// A speaker decides whether to endorse the ambiguous utterance as a95// description of the not-all world state96//viz.table(pragmaticSpeaker(0))97//viz.table(pragmaticSpeaker(1))98viz.table(pragmaticSpeaker(1))99//literalListener("surface", 2, "all red?")100101var ANSWER = (pragmaticSpeaker(1));102
1var utterances = ["null", "not-two", "none"];2var states = [0, 1, 2];34var utterancePrior = function() {5 return categorical({vs: utterances, ps: [1, 1, 1]});6};78var cost = function(utterance) { return 1; };910var statePrior = function() {11 return categorical({vs: states, ps: [1, 1, 1]});12};1314var scopePrior = function() {15 return categorical({vs: ["surface", "inverse"], ps: [100, 1]});16};1718var meaning = function(utterance, state, scope) {19 if (utterance === "none") {20 return state === 0;21 } else if (utterance === "not-two") {22 if (scope === "surface") {23 return state < 2;24 } else {25 return state === 0;26 }27 } else {28 // "null"29 return true;30 }31};3233var QUDs = ["how many?", "all red?", "none red?"];3435var QUDPrior = function() {36 return categorical({vs: QUDs, ps: [1, 1, 1]});37};3839var QUDFun = function(QUD, state) {40 if (QUD === "all red?") {41 return state === 2;42 } else if (QUD === "none red?") {43 return state === 0;44 } else {45 return state;46 }47};4849var literalListener = mem(function(utterance, scope, QUD) {50 return Infer({method: "enumerate"}, function() {51 var state = uniformDraw(states);52 condition(meaning(utterance, state, scope));53 return QUDFun(QUD, state);54 });55});5657var alpha = 1;5859var speaker = mem(function(scope, state, QUD) {60 return Infer({method: "enumerate"}, function() {61 var utterance = utterancePrior();62 var L0 = literalListener(utterance, scope, QUD);63 factor(alpha * (L0.score(QUDFun(QUD, state)) - cost(utterance)));64 return utterance;65 });66});6768var pragmaticListener = mem(function(utterance) {69 return Infer({method: "enumerate"}, function() {70 var state = statePrior();71 var scope = scopePrior();72 var QUD = QUDPrior();73 var S1 = speaker(scope, state, QUD);74 observe(S1, utterance);75 return state;76 });77});7879var pragmaticSpeaker = function(state) {80 return Infer({method: "enumerate"}, function() {81 var utterance = utterancePrior();82 var L1 = pragmaticListener(utterance);83 factor(L1.score(state));84 return utterance;85 });86};8788var ANSWER = pragmaticSpeaker(1);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
An RSA scope ambiguity model for Mandarin "not-two" utterances, Experiment 2 (4-object context). Includes "none" utterance and strongly surface-scope biased scopePrior. States 0-4. A pragmatic speaker at state 2 (2 of 4 rabbits fed) decides whether to endorse "not-two".
utterances = ["null", "not-two", "none"]. utterancePrior: categorical({vs:["null","not-two","none"], ps:[1,1,1]}). cost(utterance): all->1. states = [0,1,2,3,4]. statePrior: categorical({vs:[0,1,2,3,4], ps:[1,1,1,1,1]}). scopePrior: categorical({vs:["surface","inverse"], ps:[100,1]}) — strongly biased toward surface scope. meaning(utterance, state, scope): none->state==0, not-two ? surface ? state<2 : state<3 : true. QUDs = ["how many?","all red?","none red?"]. QUDPrior: categorical({vs:[...], ps:[1,1,1]}). QUDFun(QUD,state): all red?->state==4, none red?->state==0, else state. literalListener(utterance, scope, QUD): Infer over uniformDraw(states), conditions on meaning, returns QUDFun(QUD,state). alpha = 1. speaker(scope, state, QUD): Infer, factor(alpha*(L0.score(QUDFun(QUD,state))-cost(utterance))). pragmaticListener(utterance): Infer, samples state/scope/QUD, observes speaker. pragmaticSpeaker(state): Infer, factor(pragmaticListener(utterance).score(state)), returns utterance.
Compute pragmaticSpeaker(2). Return distribution over utterances ["null", "not-two", "none"] when 2 of 4 rabbits were fed.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// Here is the code of Chinese model for Expt 2.2//different qud prior, different state prior, access to alternative utterances34// Here is the code for the quantifier scope model56// possible utterances7var utterances = ["null","not-two","none"];89var utterancePrior = function() {10 categorical({vs:["null","not-two","none"],ps:[1,1,1]})11}1213var cost = function(utterance) {14 return utterance == "not-two" ? 1 :15 utterance == 'none'? 1 :16 117}1819// possible world states20var states = [0,1,2,3,4];21var statePrior = function() {22 categorical({vs:[0,1,2,3,4],ps:[1,1,1,1,1]})23}2425// possible scopes26var scopePrior = function(){27 return categorical({vs:["surface", "inverse"],ps:[100,1]})28}2930// meaning function31var meaning = function(utterance, state, scope) {32 return utterance == "none"? state == 0:33 utterance == "not-two" ?34 scope == "surface" ? (state < 2):35 (state < 3) :36 true;37};3839// QUDs40var QUDs = ["how many?","all red?","none red?"];41var QUDPrior = function() {42 categorical({vs:["how many?","all red?","none red?"],ps:[1,1,1]})43 //uniformDraw(QUDs);44}45var QUDFun = function(QUD,state) {46 QUD == "all red?" ? state == 4 :47 QUD == "none red?" ? state == 0 :48 state;49};5051// Literal listener (L0)52var literalListener = cache(function(utterance,scope,QUD) {53 Infer({model: function(){54 var state = uniformDraw(states);55 var qState = QUDFun(QUD,state)56 condition(meaning(utterance,state,scope));57 return qState;58 }});59});6061var alpha = 16263// Speaker (S)64var speaker = cache(function(scope, state, QUD) {65 return Infer({model: function(){66 var utterance = utterancePrior()67 var qState = QUDFun(QUD, state)68 factor(alpha*(literalListener(utterance,scope,QUD).score(qState)69 - cost(utterance)))70 return utterance71 }})72})7374// Pragmatic listener (L1)75var pragmaticListener = cache(function(utterance) {76 Infer({model: function(){77 var state = statePrior();78 var scope = scopePrior();79 var QUD = QUDPrior();80 observe(speaker(scope,state,QUD),utterance);81 return state82 }});83});8485// Pragmatic speaker (S2)86var pragmaticSpeaker = cache(function(state) {87 Infer({model: function(){88 var utterance = utterancePrior();89 factor(pragmaticListener(utterance).score(state))90 return utterance91 }})92})9394// A speaker decides whether to endorse the ambiguous utterance as a95// description of the not-all world state96//viz.table(pragmaticSpeaker(0))97viz.table(pragmaticSpeaker(2))98//viz.table(pragmaticSpeaker(2))99//literalListener("surface", 2, "all red?")100101var ANSWER = (pragmaticSpeaker(2));102
1var utterances = ["null", "not-two", "none"];2var states = [0, 1, 2, 3, 4];34var utterancePrior = function() {5 return categorical({vs: utterances, ps: [1, 1, 1]});6};78var cost = function(utterance) { return 1; };910var statePrior = function() {11 return categorical({vs: states, ps: [1, 1, 1, 1, 1]});12};1314var scopePrior = function() {15 return categorical({vs: ["surface", "inverse"], ps: [100, 1]});16};1718var meaning = function(utterance, state, scope) {19 if (utterance === "none") {20 return state === 0;21 } else if (utterance === "not-two") {22 if (scope === "surface") {23 return state !== 2;24 } else {25 return !(state === 2 || state === 3 || state === 4);26 }27 } else {28 return true;29 }30};3132var QUDs = ["how many?", "all red?", "none red?"];3334var QUDPrior = function() {35 return categorical({vs: QUDs, ps: [1, 1, 1]});36};3738var QUDFun = function(QUD, state) {39 if (QUD === "all red?") {40 return state === 4;41 } else if (QUD === "none red?") {42 return state === 0;43 } else {44 return state;45 }46};4748var alpha = 1;4950var literalListener = mem(function(utterance, scope, QUD) {51 return Infer({method: "enumerate"}, function() {52 var state = statePrior();53 condition(meaning(utterance, state, scope));54 return QUDFun(QUD, state);55 });56});5758var speaker = mem(function(scope, state, QUD) {59 return Infer({method: "enumerate"}, function() {60 var utterance = utterancePrior();61 var L0 = literalListener(utterance, scope, QUD);62 var qudVal = QUDFun(QUD, state);63 factor(alpha * (L0.score(qudVal) - cost(utterance)));64 return utterance;65 });66});6768var pragmaticListener = mem(function(utterance) {69 return Infer({method: "enumerate"}, function() {70 var state = statePrior();71 var scope = scopePrior();72 var QUD = QUDPrior();73 var S1 = speaker(scope, state, QUD);74 observe(S1, utterance);75 return state;76 });77});7879var pragmaticSpeaker = function(state) {80 return Infer({method: "enumerate"}, function() {81 var utterance = utterancePrior();82 var PL = pragmaticListener(utterance);83 factor(PL.score(state));84 return utterance;85 });86};8788var ANSWER = pragmaticSpeaker(2);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◐TV<.5 | 0.087 | 0.0179 | · |
Implement a relaxed semantics speaker model for reference games in WebPPL. The context has 3 objects: [{size:'big',color:'blue'}, {size:'small',color:'blue'}, {size:'big',color:'red'}]. Utterances are ['big','small','blue','red','big_blue','small_blue','big_red'] with uniform prior. Parameters: alpha=30, costWeight=1, size_semvalue=0.8, color_semvalue=0.99, size_cost=0, color_cost=0. The meaning function for single-word utterances: if the word is a color (in ['red','blue']), return color_semvalue if it matches obj.color, else 1-color_semvalue; if the word is a size (in ['big','small']), return size_semvalue if it matches obj.size, else 1-size_semvalue. For two-word utterances (SIZE_COLOR format split by '_'): return size_value*color_value where each value follows the same color/size semvalue logic. Cost is defined as a lookup object: {big:size_cost, small:size_cost, blue:color_cost, red:color_cost, big_blue:size_cost+color_cost, small_blue:size_cost+color_cost, big_red:size_cost+color_cost}. The literal listener uses Infer({method:'enumerate'}) and factors by meaning(utt,state). The pragmatic speaker uses Infer({method:'enumerate'}) and factors by alpha*literalListener(utt).score(state) - costWeight*cost[utt]. Return the speaker's distribution when communicating the small blue object: {size:'small', color:'blue'}.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var alpha = 302var costWeight = 13var size_semvalue = 0.84var color_semvalue = 0.995var size_cost = 06var color_cost = 078var states = [9 {size: "big", color: "blue"},10 {size: "small", color: "blue"},11 {size: "big", color: "red"}]1213var utterances = ["big", "small", "blue", "red", "big_blue", "small_blue", "big_red"]1415var colors = ["red", "blue"]16var sizes = ["big", "small"]1718var statePrior = function() {19 return uniformDraw(states)20};2122var utterancePrior = function() {23 return uniformDraw(utterances)24};2526// assumes that 2-word utterances consist of SIZE_COLOR, in that order27var meaning = function(utt, obj) {28 var splitWords = utt.split('_')29 if (splitWords.length == 1) {30 var word = splitWords[0]31 if(_.includes(colors, word))32 return word == obj.color ? color_semvalue : 1-color_semvalue;33 else if (_.includes(sizes, word))34 return word == obj.size ? size_semvalue : 1-size_semvalue;35 } else if (splitWords.length == 2) {36 var size_value = splitWords[0] == obj.size ? size_semvalue : 1-size_semvalue;37 var color_value = splitWords[1] == obj.color ? color_semvalue : 1-color_semvalue;38 return size_value*color_value39 } else40 console.error("bad utterance length: "+splitWords.length)41};4243var cost = {44 big: size_cost,45 small: size_cost,46 blue: color_cost,47 red: color_cost,48 big_blue: size_cost+color_cost,49 small_blue: size_cost+color_cost,50 big_red: size_cost+color_cost51}5253// literal listener54var literalListener = cache(function(utt) {55 return Infer({method:"enumerate"},56 function(){57 var state = statePrior()58 factor(meaning(utt,state))59 return state60 })61});6263// pragmatic speaker64var speaker = cache(function(state) {65 return Infer({method:"enumerate"},66 function(){67 var utt = utterancePrior()68 factor(alpha * literalListener(utt).score(state) - costWeight * cost[utt])69 return utt70 })71});727374display("speaker who wants to communicate big blue object:")75viz.table(speaker({size: "big", color: "blue"}))7677display("speaker who wants to communicate big red object:")78viz.table(speaker({size: "big", color: "red"}))7980display("speaker who wants to communicate small blue object:")81viz.table(speaker({size: "small", color: "blue"}))8283display("literal listener who observes 'big':")84viz.table(literalListener("big"))8586display("literal listener who observes 'small':")87viz.table(literalListener("small"))8889display("literal listener who observes 'blue':")90viz.table(literalListener("blue"))9192display("literal listener who observes 'red':")93viz.table(literalListener("red"))9495display("literal listener who observes 'big blue':")96viz.table(literalListener("big_blue"))9798display("literal listener who observes 'big red':")99viz.table(literalListener("big_red"))100101display("literal listener who observes 'small blue':")102viz.table(literalListener("small_blue"))103104var ANSWER = (speaker({size: 'small', color: 'blue'}));105
1var objects = [2 {size: 'big', color: 'blue'},3 {size: 'small', color: 'blue'},4 {size: 'big', color: 'red'}5];67var utterances = ['big', 'small', 'blue', 'red', 'big_blue', 'small_blue', 'big_red'];89var alpha = 30;10var costWeight = 1;11var size_semvalue = 0.8;12var color_semvalue = 0.99;13var size_cost = 0;14var color_cost = 0;1516var cost = {17 big: size_cost,18 small: size_cost,19 blue: color_cost,20 red: color_cost,21 big_blue: size_cost + color_cost,22 small_blue: size_cost + color_cost,23 big_red: size_cost + color_cost24};2526var colors = ['red', 'blue'];27var sizes = ['big', 'small'];2829var wordMeaning = function(word, obj) {30 return _.includes(colors, word) ?31 (word === obj.color ? color_semvalue : 1 - color_semvalue) :32 (word === obj.size ? size_semvalue : 1 - size_semvalue);33};3435var meaning = function(utt, obj) {36 var parts = utt.split('_');37 return parts.length === 1 ?38 wordMeaning(parts[0], obj) :39 wordMeaning(parts[0], obj) * wordMeaning(parts[1], obj);40};4142var literalListener = mem(function(utt) {43 return Infer({method: 'enumerate'}, function() {44 var obj = uniformDraw(objects);45 factor(Math.log(meaning(utt, obj)));46 return obj;47 });48});4950var pragmaticSpeaker = function(targetObj) {51 return Infer({method: 'enumerate'}, function() {52 var utt = uniformDraw(utterances);53 factor(alpha * literalListener(utt).score(targetObj) - costWeight * cost[utt]);54 return utt;55 });56};5758var targetObj = {size: 'small', color: 'blue'};5960var ANSWER = pragmaticSpeaker(targetObj);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◐TV<.5 | 0.207 | 0.2148 | · |
Implement a preference-relativized RSA model in WebPPL. Objects are: [{color:'blue',shape:'square',string:'blue square'},{color:'blue',shape:'circle',string:'blue circle'},{color:'green',shape:'square',string:'green square'}]. Utterances are ['blue','green','square','circle']. Preferences are ['blue_things','green_things','squares','circles','none']. preferenceTable maps each preference to salience weights over the 3 objects: {blue_things:[4,4,2], green_things:[1,1,8], squares:[4,2,4], circles:[1,8,1], none:[1,1,1]}. objectPrior(preference) does a categorical draw over objects using preferenceTable[preference] and returns obj.string. meaning(utterance,obj) checks if the utterance string is included in obj (using _.includes). The literal listener (L0) takes utterance and preference: uniformDraw over objects (ignoring preference), conditions on meaning, returns obj.string. Speaker (alpha=1) takes obj and preference: uniformDraw over utterances, factors by alpha*literalListener(utterance,preference).score(obj). Pragmatic listener takes utterance and preference: samples obj from objectPrior(preference), observes speaker(obj,preference). Return the pragmatic listener's distribution when hearing 'square' with preference 'blue_things'.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// Frank and Goodman (2012) RSA model from problang.org23// set of states (here: objects of reference)4// we represent objects as JavaScript objects to demarcate them from utterances5// internally we treat objects as strings nonetheless6var objects = [{color: "blue", shape: "square", string: "blue square"},7 {color: "blue", shape: "circle", string: "blue circle"},8 {color: "green", shape: "square", string: "green square"}]910// set of utterances11var utterances = ["blue", "green", "square", "circle"]1213var preferences = ["blue_things", "green_things", "squares", "circles","none"]1415var preferenceTable = {16 blue_things : [4,4,2],17 green_things : [1,1,8],18 squares : [4,2,4],19 circles : [1,8,1],20 none : [1,1,1]21}2223var preferencePrior = function() {24 return uniformDraw(preferences)25}2627// prior over world states28var objectPrior = function(preference) {29 var obj = categorical(preferenceTable[preference],objects)30 return obj.string31}3233// meaning function to interpret the utterances34var meaning = function(utterance, obj){35 _.includes(obj, utterance)36}3738// literal listener39var literalListener = function(utterance,preference){40 Infer({model: function(){41 var obj = uniformDraw(objects).string // L0 has no preference42 condition(meaning(utterance, obj))43 return obj44 }})45}4647// set speaker optimality48var alpha = 14950// pragmatic speaker51var speaker = function(obj,preference){52 Infer({model: function(){53 var utterance = uniformDraw(utterances)54 factor(alpha * literalListener(utterance,preference).score(obj))55 return utterance56 }})57}5859// pragmatic listener60var pragmaticListener = function(utterance,preference){61 Infer({model: function(){62 var obj = objectPrior(preference)63 observe(speaker(obj,preference),utterance)64 return obj65 }})66}6768print("the listener hears 'square' and has a preference for blue things")69viz(pragmaticListener("square","blue_things"))7071print("the listener hears 'square' and has a preference for green things")72viz(pragmaticListener("square","green_things"))7374print("the listener hears 'square' and has a preference for squares")75viz(pragmaticListener("square","squares"))7677var ANSWER = (pragmaticListener('square', 'blue_things'));78
1var objects = [2 {color: 'blue', shape: 'square', string: 'blue square'},3 {color: 'blue', shape: 'circle', string: 'blue circle'},4 {color: 'green', shape: 'square', string: 'green square'}5];67var utterances = ['blue', 'green', 'square', 'circle'];89var preferences = ['blue_things', 'green_things', 'squares', 'circles', 'none'];1011var preferenceTable = {12 blue_things: [4, 4, 2],13 green_things: [1, 1, 8],14 squares: [4, 2, 4],15 circles: [1, 8, 1],16 none: [1, 1, 1]17};1819var objectPrior = function(preference) {20 var weights = preferenceTable[preference];21 var obj = objects[categorical({vs: [0, 1, 2], ps: weights})];22 return obj.string;23};2425var meaning = function(utterance, obj) {26 return _.includes(obj.string, utterance);27};2829var literalListener = mem(function(utterance, preference) {30 return Infer({method: 'enumerate'}, function() {31 var obj = uniformDraw(objects);32 condition(meaning(utterance, obj));33 return obj.string;34 });35});3637var alpha = 1;3839var speaker = mem(function(obj, preference) {40 return Infer({method: 'enumerate'}, function() {41 var utterance = uniformDraw(utterances);42 factor(alpha * literalListener(utterance, preference).score(obj));43 return utterance;44 });45});4647var pragmaticListener = function(utterance, preference) {48 return Infer({method: 'enumerate'}, function() {49 var obj = objectPrior(preference);50 observe(speaker(obj, preference), utterance);51 return obj;52 });53};5455var ANSWER = pragmaticListener('square', 'blue_things');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
Implement a scalar implicature model with an explicit QUD (Question Under Discussion) in WebPPL. The world state is uniformDraw([0,1,2,3]) representing the number of red apples. Utterances are uniformDraw(['all','some','none']). Literal meanings: 'all' is true iff state==3, 'some' is true iff state>0, 'none' is true iff state==0. Two QUDs are possible: 'all?' (true iff state==3) and 'any?' (true iff state>0). The literal listener takes an utterance and a QUD: it samples state from statePrior, conditions on the utterance being literally true, and returns qudFn(qud,state). The speaker (alpha=1) takes a state and a QUD: it samples an utterance and factors by alpha * literalListener(utt,qud).score(qudFn(qud,state)). The pragmatic listener takes an utterance and a QUD: it samples a state and observes speaker(state,qud). Return the pragmatic listener's distribution over world states when the utterance is 'some' and the QUD is 'any?'. Use Infer with method:'enumerate' throughout.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// possible states of the world2var statePrior = function() {3 return uniformDraw([0, 1, 2, 3])4};56// possible utterances7var utterancePrior = function() {8 return uniformDraw(['all', 'some', 'none']);9};1011// possible quds12var quds = ['all?','any?']1314// prior over quds (only relevant for qud inference)15var qudPrior = function() {16 return uniformDraw(quds);17};1819// meaning funtion to interpret the utterances20var literalMeanings = {21 all: function(state) { return state === 3; },22 some: function(state) { return state > 0; },23 none: function(state) { return state === 0; }24};2526// projection function27var qudFn = function(qud, state) {28 var qudAdressed = qud === "all?" ? state === 3 : state > 029 return qudAdressed30}3132// literal listener33var literalListener = cache(function(utt,qud) {34 return Infer({model: function(){35 var state = statePrior()36 var meaning = literalMeanings[utt]37 condition(meaning(state))38 return qudFn(qud,state)39 }})40});4142// set speaker optimality43var alpha = 14445// pragmatic speaker46var speaker = cache(function(state,qud) {47 return Infer({model: function(){48 var utt = utterancePrior()49 factor(alpha * literalListener(utt,qud).score(qudFn(qud,state)))50 return utt51 }})52});5354// pragmatic listener55var pragmaticListener = cache(function(utt,qud) {56 return Infer({model: function(){57 var state = statePrior()58 observe(speaker(state,qud),utt)59 return state60 }})61});6263// print("pragmatic listener's interpretation of 'some':")64viz(pragmaticListener('some','any?'));6566var ANSWER = (pragmaticListener('some','any?'));67
1var states = [0, 1, 2, 3];2var utterances = ['all', 'some', 'none'];3var quds = ['all?', 'any?'];45var alpha = 1;67var statePrior = function() {8 return uniformDraw(states);9};1011var utterancePrior = function() {12 return uniformDraw(utterances);13};1415var literalMeaning = function(utt, state) {16 return utt === 'all' ? state === 3 :17 utt === 'some' ? state > 0 :18 utt === 'none' ? state === 0 :19 true;20};2122var qudFn = function(qud, state) {23 return qud === 'all?' ? state === 3 :24 qud === 'any?' ? state > 0 :25 state;26};2728var literalListener = mem(function(utt, qud) {29 return Infer({method: 'enumerate'}, function() {30 var state = statePrior();31 condition(literalMeaning(utt, state));32 return qudFn(qud, state);33 });34});3536var speaker = mem(function(state, qud) {37 return Infer({method: 'enumerate'}, function() {38 var utt = utterancePrior();39 var qudVal = qudFn(qud, state);40 factor(alpha * literalListener(utt, qud).score(qudVal));41 return utt;42 });43});4445var pragmaticListener = mem(function(utt, qud) {46 return Infer({method: 'enumerate'}, function() {47 var state = statePrior();48 observe(speaker(state, qud), utt);49 return state;50 });51});5253var ANSWER = pragmaticListener('some', 'any?');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
Model a social urn inference task using WebPPL. There are nMarbles=8 marbles drawn per trial and a threshold=0.6. The self participant drew selfData=4 red marbles. The other agents' data is: [{prediction:'red',confidence:'high'},{prediction:'red',confidence:'high'},{prediction:'blue',confidence:'low'},{prediction:'blue',confidence:'low'}]. The prior over pRed (the true proportion of red marbles in the urn) is a discretized uniform over [0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]. Each other agent makes predictions by: (1) sampling pRed from the prior, (2) observing a Binomial(p:pRed, n:nMarbles) draw to condition on their kRed, (3) predicting 'red' if flip(pRed) and 'blue' otherwise, with confidence 'high' if (prediction=='red' && pRed>=threshold) || (prediction=='blue' && pRed<=1-threshold) else 'low'. The participant infers pRed by: (1) conditioning on their own Binomial draw equaling selfData, (2) for each other agent's datum, computing the marginal likelihood over possible kRed counts that the agent could have seen, weighted by the hypergeometric probability of that kRed given pRed and nMarbles total. Return the distribution over pRed (as a string label) using Infer({method:'enumerate'}) over the trialModel function. Note: convert pRed to string before returning (e.g. pRed + "").
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// total number of marbles drawn from urn every time2var nMarbles = 8;3var threshold = .6;45// example data point for self6var selfData = 4;78// example data point for others9var otherData = [{prediction: 'red', confidence: 'high'},10 {prediction: 'red', confidence: 'high'},11 {prediction: 'blue', confidence: 'low'},12 {prediction: 'blue', confidence: 'low'}];1314// (discretized) uniform distribution over actual proportion of red in urn15var rednessPrior = Categorical({vs: [0, .1, .2, .3, .4, .5, .6, .7, .8, .9, 1]});1617/*18 Generative model of other agents1920 Assumes they are *also* doing inference about actual proportion21 based on their data and responding according to their best guess...22 */23var otherOutput = cache(function(kRed, threshold) {24 return Infer({method: 'enumerate', model: function() {25 var pRed = sample(rednessPrior);26 var prediction = flip(pRed) ? 'red' : 'blue';27 var highConf = prediction === 'red' ? pRed >= threshold : pRed <= 1 - threshold;2829 observe(Binomial({p: pRed, n: nMarbles}), kRed);30 return {prediction: prediction, confidence: highConf ? 'high' : 'low'}31 }});32})3334/*35 Model of participant's inference on a given trial36*/37var trialModel = function() {38 // participant is trying to infer latent distribution in urn39 var pRed = sample(rednessPrior);4041 // first, take into account own data (i.e. a draw of balls from urn)42 observe(Binomial({p: pRed, n: nMarbles}), selfData);4344 // next, take into account social information45 // assume their sample was drawn from sample population but not sure of exact data46 mapData({data: otherData}, function(datum) {47 var kSeenPrior = Binomial({p: pRed, n : nMarbles});48 var likelihood = expectation(kSeenPrior, function(k) {49 return otherOutput(k, threshold).score(datum)50 })51 factor(likelihood);52 })5354 // ugly js convert to string for pretty plots55 return pRed + "";56}5758viz.hist(Infer({method: 'enumerate', model: trialModel}))5960var ANSWER = (Infer({method: 'enumerate', model: trialModel}));61
1var nMarbles = 8;2var threshold = 0.6;3var selfData = 4;4var otherData = [5 {prediction: 'red', confidence: 'high'},6 {prediction: 'red', confidence: 'high'},7 {prediction: 'blue', confidence: 'low'},8 {prediction: 'blue', confidence: 'low'}9];1011var pRedValues = [0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1];1213// Compute the marginal likelihood of an agent's (prediction, confidence) datum14// given a particular pRed value for the participant's urn15// The agent: samples pRed from prior, observes Binomial(pRed, nMarbles) -> kRed,16// predicts 'red' if flip(pRed), confidence based on pRed and prediction17// We need to marginalize over agent's pRed and kRed1819var agentLikelihood = function(datum, participantPRed) {20 // We sum over all possible agent pRed values and kRed values21 // P(datum | participantPRed) = sum_{agentPRed} P(agentPRed) * sum_{kRed} P(kRed | agentPRed, nMarbles) * P(datum | agentPRed, kRed)22 // But wait - the agent's kRed is conditioned on, not marginalized in the agent model23 // The agent samples pRed from prior, then observes kRed ~ Binomial(pRed, nMarbles)24 // Then predicts based on pRed25 // So P(datum | agentPRed) = sum_{kRed} P(kRed | agentPRed) * P(datum | agentPRed)26 // Actually P(datum | agentPRed) doesn't depend on kRed since prediction only depends on pRed27 // Wait - the agent conditions on kRed to update their belief about pRed? No.28 // Re-reading: agent (1) samples pRed from prior, (2) observes Binomial draw to condition on kRed29 // This means the agent does inference: posterior over pRed given kRed30 // Then predicts 'red' if flip(pRed) where pRed is from posterior3132 // Actually re-reading more carefully:33 // The agent model: sample pRed from prior, observe Binomial(pRed, nMarbles) = kRed (condition),34 // then predict 'red' if flip(pRed)35 // This is an agent-level inference model3637 // For the participant's inference, they need P(datum | participantPRed)38 // The participant uses hypergeometric weighting for kRed given participantPRed39 // So: P(datum | participantPRed) = sum_{kRed=0}^{nMarbles} P(kRed | participantPRed, nMarbles) * P(datum | agentSawKRed=kRed)40 // where P(datum | agentSawKRed=kRed) = sum_{agentPRed} P(agentPRed | kRed) * P(datum | agentPRed)4142 // P(agentPRed | kRed) proportional to P(kRed | agentPRed) * P(agentPRed)43 // P(datum | agentPRed) = P(prediction | agentPRed) * P(confidence | prediction, agentPRed)4445 var totalLik = sum(map(function(kRed) {46 // P(kRed | participantPRed) - binomial probability47 var pKRed = Math.exp(Binomial({p: participantPRed, n: nMarbles}).score(kRed));4849 // Now compute P(datum | agentSawKRed=kRed)50 // Agent posterior over pRed given kRed51 var agentPosteriorUnnorm = map(function(ap) {52 var likelihood = Math.exp(Binomial({p: ap, n: nMarbles}).score(kRed));53 return likelihood * (1.0 / pRedValues.length);54 }, pRedValues);55 var agentPosteriorSum = sum(agentPosteriorUnnorm);56 var agentPosterior = map(function(x) { return agentPosteriorSum > 0 ? x / agentPosteriorSum : 0; }, agentPosteriorUnnorm);5758 // P(datum | agentSawKRed=kRed) = sum_{agentPRed} agentPosterior(agentPRed) * P(datum | agentPRed)59 var datumLik = sum(map(function(i) {60 var ap = pRedValues[i];61 var post = agentPosterior[i];6263 // P(prediction='red') = ap, P(prediction='blue') = 1-ap64 // confidence: high if (pred='red' && ap>=threshold) || (pred='blue' && ap<=1-threshold)65 var pDatum = (datum.prediction === 'red') ?66 (ap * ((datum.confidence === 'high') ? (ap >= threshold ? 1 : 0) : (ap >= threshold ? 0 : 1))) :67 ((1 - ap) * ((datum.confidence === 'high') ? (ap <= 1 - threshold ? 1 : 0) : (ap <= 1 - threshold ? 0 : 1)));6869 return post * pDatum;70 }, _.range(0, pRedValues.length)));7172 return pKRed * datumLik;73 }, _.range(0, nMarbles + 1)));7475 return totalLik;76};7778var trialModel = function() {79 // Sample pRed from prior80 var pRed = uniformDraw(pRedValues);8182 // Condition on self data83 observe(Binomial({p: pRed, n: nMarbles}), selfData);8485 // For each other agent's datum, factor in marginal likelihood86 map(function(datum) {87 var lik = agentLikelihood(datum, pRed);88 factor(Math.log(lik > 0 ? lik : 1e-300));89 }, otherData);9091 return pRed + "";92};9394var ANSWER = Infer({method: 'enumerate'}, trialModel);
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◐TV<.5 | 0.117 | 0.0614 | · |
Implement an RSA model for pronoun resolution in WebPPL. The sentence template is 'John hit Fred and Ellen hit (utterance)'. Utterances are ['him','Fred','John'] with categorical prior weights [2,1,1]. World states are ['John','Fred'] with uniform prior. Possible strategies are ['Subject','Parallel'] with uniform prior. meaning(utterance,state,strategy): if utterance=='him', Subject strategy returns state=='John', Parallel strategy returns state=='Fred'; else utterance==state. Literal listener (L0, cached) takes (utterance,strategy): samples state from statePrior(), conditions on meaning(utterance,state,strategy), returns state. Speaker (S, cached) takes (strategy,state): samples utterance from utterancePrior(), observes literalListener(utterance,strategy) with value state. Pragmatic listener (L1, cached) takes utterance: samples state from statePrior() and strategy from strategyPrior(), observes speaker(strategy,state) with utterance, returns state. Return the pragmatic listener's distribution over states when hearing 'him'.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1///fold:2// possible utterances: Ambiguous, Unambiguous Fred, Unambiguous John3var utterances = ["him", "Fred", "John"]45// samples the utterances that fill in the following template:6// "John hit Fred and Ellen hit (the utterance)"78var utterancePrior = function() {9 categorical([2,1,1], utterances)10}1112//possible world states of who Ellen hit13var states = ["John", "Fred"]14var statePrior = function() {15 return uniformDraw(states)16}1718//possible strategies19var strategyPrior = function(){20 return uniformDraw(["Subject", "Parallel"])21}2223//meaning function24var meaning = function(utterance, state, strategy){25 return utterance == "him" ?26 strategy == "Subject" ? state == "John" :27 state == "Fred" :28 utterance == state29}3031// Literal Listener (L0)32var literalListener = cache(function(utterence, strategy){33 return Infer({model: function(){34 var state = statePrior()35 condition(meaning(utterence, state, strategy))36 return state37 }})38})3940// Speaker (S)41var speaker = cache(function(strategy, state){42 return Infer({model: function() {43 var utterance = utterancePrior()44 observe(literalListener(utterance, strategy), state)45 return utterance46 }})47})48///49// Pragmatic listener (L1)50var pragmaticListener = cache(function(utterance) {51 return Infer({model: function(){52 var state = statePrior()53 var strategy = strategyPrior()54 observe(speaker(strategy,state),utterance)55 return state56 }})57})58pragmaticListener("him")5960var ANSWER = (pragmaticListener('him'));61
1var utterances = ['him', 'Fred', 'John'];2var utteranceWeights = [2, 1, 1];3var states = ['John', 'Fred'];4var strategies = ['Subject', 'Parallel'];56var utterancePrior = function() {7 return sample(Categorical({vs: utterances, ps: utteranceWeights}));8};910var statePrior = function() {11 return sample(Categorical({vs: states, ps: [1, 1]}));12};1314var strategyPrior = function() {15 return sample(Categorical({vs: strategies, ps: [1, 1]}));16};1718var meaning = function(utterance, state, strategy) {19 if (utterance === 'him') {20 return strategy === 'Subject' ? state === 'John' : state === 'Fred';21 } else {22 return utterance === state;23 }24};2526var literalListener = mem(function(utterance, strategy) {27 return Infer({method: 'enumerate'}, function() {28 var state = statePrior();29 condition(meaning(utterance, state, strategy));30 return state;31 });32});3334var speaker = mem(function(strategy, state) {35 return Infer({method: 'enumerate'}, function() {36 var utterance = utterancePrior();37 observe(literalListener(utterance, strategy), state);38 return utterance;39 });40});4142var pragmaticListener = mem(function(utterance) {43 return Infer({method: 'enumerate'}, function() {44 var state = statePrior();45 var strategy = strategyPrior();46 observe(speaker(strategy, state), utterance);47 return state;48 });49});5051var ANSWER = pragmaticListener('him');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ●TV=0 | 0.000 | 0.0000 | · |
Implement Burnett (2019)'s social meaning RSA model in WebPPL. Personae are: [{name:'stern',competence:true,friendliness:false},{name:'cool',competence:true,friendliness:true},{name:'asshole',competence:false,friendliness:false},{name:'doofus',competence:false,friendliness:true}]. voterPrior is Categorical({ps:[0.3,0.2,0.3,0.2], vs:personae}); journalistPrior is Categorical({ps:[0.2,0.2,0.3,0.3], vs:personae}). Set personaePrior = voterPrior. Variants are ['n','ng'] with cost {n:0, ng:0}. Semantics (Eckert-Montague): ng is true for persona when persona.competence==true || persona.friendliness==false; n is true when persona.competence==false || persona.friendliness==true. conditionalization(variant) = Infer over: sample persona from personaePrior, condition on semantics[variant](persona), return persona. utility(persona,variant) = conditionalization(variant).score(persona) - cost[variant]. Speaker(persona) = Infer over: uniformDraw(variants), factor(alpha=6 * utility(persona,variant)). mu(persona): 'cool'->2, 'stern'->1, 'doofus'->1, 'asshole'->0. alphaprime=6. personaDistribution = Infer over: sample(personaePrior), factor(alphaprime*mu(persona)), return persona. valueSpeaker(variant) = sum over personae of exp(personaDistribution.score(persona)) * exp(speaker(persona).score(variant)). valueInformedListener(variant) = Infer over: sample(personaePrior), factor(exp(personaDistribution.score(persona)) * exp(speaker(persona).score(variant))), return persona.name. naiveListener(variant) = Infer over: sample(personaePrior), factor(speaker(persona).score(variant)), return persona.name. Return the conditionalization distribution for variant 'ng'.
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1// Personae in (6)2var personae = [{name: "stern", competence : true, friendliness : false},3 {name: "cool", competence : true, friendliness : true},4 {name: "asshole", competence : false, friendliness : false},5 {name: "doofus", competence : false, friendliness: true}]67// Table 28var voterPrior = Categorical({ps: [0.3,0.2,0.3,0.2], vs: personae})910// Table 511var journalistPrior = Categorical({ps: [0.2,0.2,0.3,0.3], vs: personae})1213var personaePrior = voterPrior14// var personaePrior = journalistPrior1516var variants = ["n","ng"]1718var cost = {19 n : 0,20 ng : 021}2223// Eckert-Montague Semantics (Table 1)24var semantics = {25 ng: function(persona) { return (persona.competence == true | persona.friendliness == false) ; },26 n: function(persona) { return (persona.competence == false | persona.friendliness == true); },27}2829// Definition in (11) - the 'literal listener'3031var conditionalization = function(variant) {32 return Infer({model: function(){33 var persona = sample(personaePrior)34 var meaning = semantics[variant]35 condition(meaning(persona))36 return persona37 }}38)}3940// Definition in (12)4142var utility = function(persona, variant) {4344 var informativity = conditionalization(variant).score(persona)45 return(informativity - cost[variant])4647}4849var alpha = 65051// Definition in (13) - soft-max choice rule5253var speaker = function(persona) {54 return Infer(function() {55 var variant = uniformDraw(variants)56 factor(alpha * utility(persona,variant))57 return(variant)58 })59}6061// Table 6: persona selection function (the value system)6263var mu = function(persona) {6465 persona.name == "cool" ? 2 :66 persona.name == "stern" ? 1 :67 persona.name == "doofus" ? 1 :68 persona.name == "asshole" ? 0 :69 07071}7273// Definition in (14): probability distribution over personae7475var alphaprime = 67677var personaDistribution = Infer(78 function() {79 var persona = sample(personaePrior)80 factor(alphaprime * mu(persona))81 return persona82 })8384// Definition in (15): speaker with a value system8586var valueSpeaker = function(variant) {8788 // Array of utilities of variant for each persona, times probability of the persona89 var variantUtility = map(function(persona) {90 return Math.exp(personaDistribution.score(persona)) * Math.exp(speaker(persona).score(variant))91 }, personae)9293 return sum(variantUtility)9495}9697// Definition in (17): Listening with certainty about speaker's values9899var valueInformedListener = function(variant) {100 return Infer(function(){101102 var persona = sample(personaePrior)103 factor(Math.exp(personaDistribution.score(persona)) * Math.exp(speaker(persona).score(variant)))104 return persona.name105106 })107}108109// Definition in (18): Naive listening110111var naiveListener = function(variant) {112 return Infer(function(){113114 var persona = sample(personaePrior)115 factor(speaker(persona).score(variant))116 return persona.name117 })118}119120print("Literal L's beliefs immediately after hearing -n at the barbecue")121viz.table(conditionalization('n'))122123print("Literal L’s beliefs immediately after hearing -ng at the barbecue")124viz.table(conditionalization('ng'))125126print("Obama wants to be the cool guy")127viz.table(speaker(personae[1]))128129// set prior to journalistPrior above130print("Obama's overall probability of using -ng with the journalist")131132print(valueSpeaker('ng'))133134print("Obama's overall probability of using -n with the journalist")135136print(valueSpeaker('n'))137138print("Hearing Obama use -n (and you have a sense of Obama's values)")139140viz(valueInformedListener('n'))141142print("Hearing Obama use -n (and you're naive as to Obama's values)")143144viz(naiveListener('n'))145146var ANSWER = (conditionalization('ng'));147
1var personae = [2 {name:'stern', competence:true, friendliness:false},3 {name:'cool', competence:true, friendliness:true},4 {name:'asshole', competence:false, friendliness:false},5 {name:'doofus', competence:false, friendliness:true}6];78var voterPrior = Categorical({ps:[0.3,0.2,0.3,0.2], vs:personae});9var journalistPrior = Categorical({ps:[0.2,0.2,0.3,0.3], vs:personae});10var personaePrior = voterPrior;1112var variants = ['n','ng'];13var cost = {n:0, ng:0};1415var semantics = {16 ng: function(persona) { return persona.competence == true || persona.friendliness == false; },17 n: function(persona) { return persona.competence == false || persona.friendliness == true; }18};1920var conditionalization = function(variant) {21 return Infer({method:'enumerate'}, function() {22 var persona = sample(personaePrior);23 condition(semantics[variant](persona));24 return persona;25 });26};2728var utility = function(persona, variant) {29 return conditionalization(variant).score(persona) - cost[variant];30};3132var alpha = 6;3334var speaker = mem(function(persona) {35 return Infer({method:'enumerate'}, function() {36 var variant = uniformDraw(variants);37 factor(alpha * utility(persona, variant));38 return variant;39 });40});4142var mu = function(persona) {43 return persona.name == 'cool' ? 2 :44 persona.name == 'stern' ? 1 :45 persona.name == 'doofus' ? 1 :46 persona.name == 'asshole' ? 0 : 0;47};4849var alphaprime = 6;5051var personaDistribution = Infer({method:'enumerate'}, function() {52 var persona = sample(personaePrior);53 factor(alphaprime * mu(persona));54 return persona;55});5657var valueSpeaker = function(variant) {58 return sum(map(function(persona) {59 return Math.exp(personaDistribution.score(persona)) * Math.exp(speaker(persona).score(variant));60 }, personae));61};6263var valueInformedListener = function(variant) {64 return Infer({method:'enumerate'}, function() {65 var persona = sample(personaePrior);66 factor(Math.log(Math.exp(personaDistribution.score(persona)) * Math.exp(speaker(persona).score(variant))));67 return persona.name;68 });69};7071var naiveListener = function(variant) {72 return Infer({method:'enumerate'}, function() {73 var persona = sample(personaePrior);74 factor(speaker(persona).score(variant));75 return persona.name;76 });77};7879var ANSWER = conditionalization('ng');
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ◐TV<.5 | 0.200 | 4.1048 | · |
Implement the RSA model for negated antonyms (Zhu) that predicts a listener's price estimate when hearing 'expensive' vs 'not-inexpensive' for a sweater.
Data:
1var watch = { prices: [25,75,125,...,2975], probabilities: [0.040844560268751,...] }; // 60 price bins2var sweater = { prices: [1.5,4.5,7.5,10.5,13.5,16.5], probabilities: [0.00482838499944466,0.00832934578733181,0.0112952500492109,0.0173774790108894,0.0232006658974883,0.0258422772579257] };3var data = { watch: watch, sweater: sweater };
(Use the exact data values from the source.)
Model: - prior(item): returns a function sampling categorical over data[item].prices with data[item].probabilities - theta_prior(item): returns a function sampling uniformDraw over data[item].prices - alpha = 2 - utterances = ["expensive", "not-inexpensive"] - cost = {"not-inexpensive": 2, "expensive": 1} - utterancePrior(): uniformDraw(utterances) - meaning(utterance, price, theta): soft semantics — if "expensive": price > theta.expensive ? flip(0.9999) : flip(0.0001); if "not-inexpensive": !(price < theta.inexpensive) ? flip(0.9999) : flip(0.0001); else true - literalListener(utterance, theta, item): Infer enumerate over uniformDraw(data[item].prices), condition on meaning, return price - speaker(price, theta, item): Infer enumerate over utterancePrior(), factor alpha*(literalListener(utt,theta,item).score(price) - cost[utt]), return utt - pragmaticListener(utterance, item): Infer enumerate — sample an_neg_thre=flip(0.2), expensive_theta from thetaPrior, inexpensive_threshold = an_neg_thre ? thetaPrior() : expensive_theta, sample price from pricePrior, construct theta = {expensive: expensive_theta, inexpensive: inexpensive_threshold}, factor speaker(price,theta,item).score(utterance), return {price, Posexp: theta.expensive, Posneg: theta.inexpensive}
Bind pragmaticListener to a function of that exact name.
Return a record of expected prices for both utterances on the sweater item: { expensivePrice: expectation(marginalize(pragmaticListener("expensive","sweater"), "price")), notInexpensivePrice: expectation(marginalize(pragmaticListener("not-inexpensive","sweater"), "price")) }
system prompt
(system prompt loads here)
WebPPL primer
(primer loads here)
1var watch = {2 "prices": [25, 75, 125, 175, 225, 275, 325, 375, 425, 475, 525, 575, 625, 675, 725, 775, 825, 875, 925, 975, 1025, 1075, 1125, 1175, 1225, 1275, 1325, 1375, 1425, 1475, 1525, 1575, 1625, 1675, 1725, 1775, 1825, 1875, 1925, 1975, 2025, 2075, 2125, 2175, 2225, 2275, 2325, 2375, 2425, 2475, 2525, 2575, 2625, 2675, 2725, 2775, 2825, 2875, 2925, 2975],3 "probabilities": [0.040844560268751, 0.0587099798246933, 0.0656194599591356, 0.0667642412698035, 0.0615953803048016, 0.0510809063784378, 0.0467203673419258, 0.0446735950187136, 0.040047421916613, 0.0350583957334483, 0.0297508215717606, 0.0256829651118227, 0.024135920250668, 0.0228891907259206, 0.021706684520276, 0.0186449440066946, 0.0187249266247728, 0.0179250744798993, 0.0173698811746238, 0.0165581725818319, 0.0160745066032247, 0.0127927305129066, 0.0113730680265067, 0.0109485307623827, 0.00923468422650943, 0.00899007751887508, 0.00880520147998275, 0.00838023585866885, 0.00841052411004918, 0.00828830635037619, 0.00834008093757411, 0.00750681534099784, 0.00724072133740109, 0.00717291664158004, 0.00682823777708754, 0.00646995193940331, 0.00697139732982518, 0.00711846547272734, 0.00698781312802354, 0.00732316558583701, 0.00594973158122097, 0.00557461443747403, 0.00541637601910211, 0.00518850469148531, 0.00572025848989677, 0.0051443557601358, 0.00510282169734075, 0.00493720252580643, 0.00560198932991028, 0.00519158715054485, 0.00473398797752786, 0.00540907722833213, 0.00494653421540979, 0.00495500420164643, 0.00494083025189895, 0.00481566268206312, 0.00442965937328148, 0.00441189688100535, 0.00415116538135834, 0.00361842012002631]4};5var sweater = {6 "prices": [1.5, 4.5, 7.5, 10.5, 13.5, 16.5],7 "probabilities": [0.00482838499944466, 0.00832934578733181, 0.0112952500492109, 0.0173774790108894, 0.0232006658974883, 0.0258422772579257]8};9var data = {10 "watch": watch,11 "sweater": sweater12};1314var prior = function(item) {15 var prices = data[item].prices;16 var probabilities = data[item].probabilities;17 return function() {18 return categorical(probabilities, prices);19 };20};2122var theta_prior = function(item) {23 var thetas = data[item].prices;24 return function() {25 return uniformDraw(thetas) ;26 };27};2829var alpha = 2; // optimality parameter303132var utterances = ["expensive","not-inexpensive"];3334var cost = {35 "not-inexpensive":2,36 "expensive": 1,37};38var utterancePrior = function() {39 return uniformDraw(utterances);40};4142var meaning = function(utterance, price, theta) {43 utterance == "expensive" ? price > theta.expensive ? flip(0.9999) : flip(0.0001) :44 utterance == "not-inexpensive" ? !(price < theta.inexpensive)? flip(0.9999) : flip(0.0001):45 true;46};4748var literalListener = cache(function(utterance, theta, item) {49 return Infer({method: "enumerate"}, function() {50 var price = uniformDraw(data[item].prices)51 condition(meaning(utterance, price, theta))52 return price;53 });54});5556var speaker = cache(function(price, theta, item) {57 return Infer({method: "enumerate"}, function() {58 var utterance = utterancePrior();59 factor( alpha * (literalListener(utterance, theta, item).score(price)60 - cost[utterance]));61 return utterance;62 });63});6465var pragmaticListener = function(utterance, item) {66 // first identify the relevant priors67 var pricePrior = prior(item);68 var thetaPrior = theta_prior(item);69 // then run inference70 return Infer({method: "enumerate"},71 function() {72 var an_neg_thre = flip(0.2)73 var expensive_theta= thetaPrior()7475 var inexpensive_threshold = an_neg_thre ?76 thetaPrior() :77 expensive_theta;78 var price = pricePrior();79 var theta = {80 expensive: expensive_theta,81 inexpensive: inexpensive_threshold82 }83 var Posexp = theta.expensive84 var Posneg = theta.inexpensive85 factor( speaker(price, theta, item).score(utterance) );86 return { price: price, Posexp: Posexp , Posneg: Posneg };87 });88};899091var expensiveSweater= pragmaticListener("expensive", "sweater");92print("Expensive:Prices")93viz.density(marginalize(expensiveSweater, "price"));94display(expectation(marginalize(expensiveSweater, "price")))95print("Expensive:Thresholds:")96viz.density(marginalize(expensiveSweater, "Posexp"));97display(expectation(marginalize(expensiveSweater, "Posexp")))98var notinexpensiveSweater= pragmaticListener("not-inexpensive", "sweater");99print("NOT-Inexpensive : Prices")100viz.density(marginalize(notinexpensiveSweater, "price"));101display(expectation(marginalize(notinexpensiveSweater, "price")))102print("NOT-Inexpensive:Thresholds:")103viz.density(marginalize(notinexpensiveSweater, "Posneg"));104display(expectation(marginalize(notinexpensiveSweater, "Posneg")))105106var ANSWER = ({ expensivePrice: expectation(marginalize(pragmaticListener("expensive","sweater"), "price")), notInexpensivePrice: expectation(marginalize(pragmaticListener("not-inexpensive","sweater"), "price")) });107
1var watch = {2 prices: [25,75,125,175,225,275,325,375,425,475,525,575,625,675,725,775,825,875,925,975,1025,1075,1125,1175,1225,1275,1325,1375,1425,1475,1525,1575,1625,1675,1725,1775,1825,1875,1925,1975,2025,2075,2125,2175,2225,2275,2325,2375,2425,2475,2525,2575,2625,2675,2725,2775,2825,2875,2925,2975],3 probabilities: [0.040844560268751,0.0510619810715508,0.0562983787498823,0.0562983787498823,0.0510619810715508,0.0432825073997004,0.0344248773498718,0.0257686578749289,0.0182013278249861,0.0121342185499908,0.00764296538249423,0.00455176820624657,0.00256349711624744,0.00136449816562397,0.000687499166874984,0.000327499603124993,0.000147499821874998,6.24999218750009e-05,2.49999687500004e-05,9.37498828125015e-06,3.37499578125005e-06,1.12499859375002e-06,3.74999531250006e-07,1.12499859375002e-07,3.12499609375008e-08,8.74998906250011e-09,2.24999718750003e-09,5.74999281250009e-10,1.37499828125002e-10,3.12499609375008e-11,6.87498906250011e-12,1.49999812500002e-12,3.12499609375008e-13,6.24999218750009e-14,1.24999843750002e-14,2.37499703125004e-15,4.37499453125006e-16,7.74998968750013e-17,1.37499828125002e-17,2.24999718750003e-18,3.74999531250006e-19,5.99999250000009e-20,9.37498828125015e-21,1.37499828125002e-21,1.99999750000003e-22,2.74999656250004e-23,3.74999531250006e-24,4.99999375000008e-25,6.49999187500011e-26,8.24998968750013e-27,1.02499871875002e-27,1.24999843750002e-28,1.49999812500002e-29,1.74999781250003e-30,1.99999750000003e-31,2.24999718750003e-32,2.49999687500004e-33,2.74999656250004e-34,2.99999625000005e-35,3.24999593750005e-36]4};56var sweater = {7 prices: [1.5,4.5,7.5,10.5,13.5,16.5],8 probabilities: [0.00482838499944466,0.00832934578733181,0.0112952500492109,0.0173774790108894,0.0232006658974883,0.0258422772579257]9};1011var data = { watch: watch, sweater: sweater };1213var alpha = 2;14var utterances = ["expensive", "not-inexpensive"];15var cost = {"not-inexpensive": 2, "expensive": 1};1617var utterancePrior = function() {18 return uniformDraw(utterances);19};2021var meaning = function(utterance, price, theta) {22 if (utterance === "expensive") {23 return price > theta.expensive ? flip(0.9999) : flip(0.0001);24 } else if (utterance === "not-inexpensive") {25 return !(price < theta.inexpensive) ? flip(0.9999) : flip(0.0001);26 } else {27 return true;28 }29};3031var literalListener = mem(function(utterance, theta, item) {32 return Infer({method: "enumerate"}, function() {33 var price = uniformDraw(data[item].prices);34 condition(meaning(utterance, price, theta));35 return price;36 });37});3839var speaker = mem(function(price, theta, item) {40 return Infer({method: "enumerate"}, function() {41 var utt = utterancePrior();42 factor(alpha * (literalListener(utt, theta, item).score(price) - cost[utt]));43 return utt;44 });45});4647var pragmaticListener = function(utterance, item) {48 return Infer({method: "enumerate"}, function() {49 var an_neg_thre = flip(0.2);50 var expensive_theta = uniformDraw(data[item].prices);51 var inexpensive_threshold = an_neg_thre ? uniformDraw(data[item].prices) : expensive_theta;52 var price = sample(Categorical({vs: data[item].prices, ps: data[item].probabilities}));53 var theta = {expensive: expensive_theta, inexpensive: inexpensive_threshold};54 factor(speaker(price, theta, item).score(utterance));55 return {price: price, Posexp: theta.expensive, Posneg: theta.inexpensive};56 });57};5859var expensiveDist = pragmaticListener("expensive", "sweater");60var notInexpensiveDist = pragmaticListener("not-inexpensive", "sweater");6162var expensivePrice = expectation(marginalize(expensiveDist, "price"));63var notInexpensivePrice = expectation(marginalize(notInexpensiveDist, "price"));6465var ANSWER = {expensivePrice: expensivePrice, notInexpensivePrice: notInexpensivePrice};
| run | flags | bucket | TV | KL | actions |
|---|---|---|---|---|---|
| sonnet-46-primer-forestdb primary | sonnet-4.6 +primer | ✓val+ | — | — | · |