diff --git a/README.md b/README.md index 4af68f2..210929e 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,6 @@ We start from the most specific grammar the data permits: every sample contribut From there, we generalise with two operators, merging and chunking. Merging takes a pair of nonterminals and folds them into a single nonterminal containing the union of their productions; chunking replaces a contiguous sequence of symbols with a fresh nonterminal. Chunking doesn't itself change the language the grammar generates, but it changes the internal structure in a way that can expose useful merges which weren't previously available. -We rank candidate grammars by the posterior `P(M | X) ∝ P(M) P(X | M)`, where the prior is a description length and the likelihood integrates over the production probabilities under symmetric Dirichlet priors. The scoring therefore accounts for uncertainty in the production probabilities, rather than relying on a single fitted parameterisation. +We rank candidate grammars with a structural description length and an approximate evidence objective. The default scorer fits production probabilities with expectation maximisation, computes fractional expected production counts, and applies a symmetric Dirichlet correction. Maximum likelihood and a variational evidence lower bound are also available. We explore the resulting grammar space using either beam or best first search, then fit the parameters by expectation maximisation once the grammar structure is fixed. Parsing is a generalised CYK or inside computation over spans, which lets the grammar be scored directly without an intermediate conversion into Chomsky normal form. diff --git a/bin/main.ml b/bin/main.ml index 7e952e5..a60397c 100644 --- a/bin/main.ml +++ b/bin/main.ml @@ -46,10 +46,11 @@ let config_of_opts opts = em_iters = int_opt opts "em-iters" Scoring.default_config.em_iters; max_chunk = int_opt opts "max-chunk" Scoring.default_config.max_chunk; mode = - (match opt opts "mode" "marginal" with + (match opt opts "mode" "approximate" with | "ml" -> Scoring.Maximum_likelihood | "variational" -> Scoring.Variational - | _ -> Scoring.Marginal); + | "approximate" | "marginal" -> Scoring.Approximate_evidence + | mode -> invalid_arg ("unknown scoring mode: " ^ mode)); chunk_occurrence = (if opt opts "chunk-occurrence" "all" = "first" then Scoring.First_occurrence @@ -63,7 +64,7 @@ let usage () = Commands:\n\ \ induce --corpus FILE [--out DIR] [--beam N] [--max-steps N]\n\ \ [--patience N] [--alpha A] [--prior W] [--max-chunk N]\n\ - \ [--em-iters N] [--mode marginal|ml|variational]\n\ + \ [--em-iters N] [--mode approximate|ml|variational]\n\ \ [--chunk-occurrence all|first] [--render]\n\ \ [--search beam|bestfirst] [--frontier N] [--max-expansions N]\n\ \ score --grammar FILE --corpus FILE [--alpha A] [--prior W]\n\ @@ -159,8 +160,8 @@ let cmd_induce opts = result.Search.iterations (List.length result.Search.steps); Printf.printf "posterior: %.4f -> %.4f\n" result.Search.initial_score result.Search.best_score; - Printf.printf "structure prior %.4f, marginal loglik %.4f, dl %.1f bits\n" - details.Scoring.prior details.Scoring.marginal details.Scoring.dl_bits; + Printf.printf "structure prior %.4f, objective %.4f, dl %.1f bits\n" + details.Scoring.prior details.Scoring.objective details.Scoring.dl_bits; Printf.printf "final grammar: %d nonterminals, %d productions\n" details.Scoring.num_nonterminals details.Scoring.num_productions; print_endline "\nfinal grammar:"; @@ -178,7 +179,9 @@ let cmd_score opts = let d = Scoring.details ~config g corpus in Printf.printf "posterior %.6f\n" d.Scoring.posterior; Printf.printf "structural prior %.6f\n" d.Scoring.prior; - Printf.printf "marginal loglik %.6f\n" d.Scoring.marginal; + Printf.printf "objective %.6f\n" d.Scoring.objective; + Printf.printf "approximate evidence %.6f\n" + d.Scoring.approximate_evidence; Printf.printf "ML corpus loglik %.6f\n" d.Scoring.ml_loglik; Printf.printf "description length %.2f bits\n" d.Scoring.dl_bits; Printf.printf "nonterminals %d\n" d.Scoring.num_nonterminals; diff --git a/lib/scoring.ml b/lib/scoring.ml index 0189945..05a6152 100644 --- a/lib/scoring.ml +++ b/lib/scoring.ml @@ -1,4 +1,4 @@ -type score_mode = Marginal | Maximum_likelihood | Variational +type score_mode = Approximate_evidence | Maximum_likelihood | Variational type chunk_occurrence = All_occurrences | First_occurrence @@ -19,7 +19,7 @@ let default_config = em_iters = 8; em_tol = 1e-7; max_chunk = 2; - mode = Marginal; + mode = Approximate_evidence; chunk_occurrence = All_occurrences; } @@ -95,7 +95,8 @@ let description_length_bits (g : Grammar.t) = let structural_logprior ?(config = default_config) (g : Grammar.t) = -.config.prior_weight *. description_length_bits g *. log 2.0 -let marginal_loglik ?(config = default_config) (g : Grammar.t) (corpus : Corpus.t) = +let approximate_log_evidence ?(config = default_config) (g : Grammar.t) + (corpus : Corpus.t) = let uprep = Parse.prepare_uniform g in if @@ -228,15 +229,16 @@ let posterior ?(config = default_config) g corpus = match config.mode with | Variational -> prior +. variational_loglik ~config g corpus | _ -> - let marg, ll, _, _ = marginal_loglik ~config g corpus in + let evidence, ll, _, _ = approximate_log_evidence ~config g corpus in (match config.mode with - | Marginal -> prior +. marg + | Approximate_evidence -> prior +. evidence | Maximum_likelihood -> prior +. ll | Variational -> assert false) type details = { prior : float; - marginal : float; + objective : float; + approximate_evidence : float; ml_loglik : float; posterior : float; dl_bits : float; @@ -245,18 +247,19 @@ type details = { } let details ?(config = default_config) (g : Grammar.t) (corpus : Corpus.t) = - let marg, ll, _, _ = marginal_loglik ~config g corpus in + let evidence, ll, _, _ = approximate_log_evidence ~config g corpus in let prior = structural_logprior ~config g in let dl = description_length_bits g in let likelihood = match config.mode with - | Marginal -> marg + | Approximate_evidence -> evidence | Maximum_likelihood -> ll | Variational -> variational_loglik ~config g corpus in { prior; - marginal = likelihood; + objective = likelihood; + approximate_evidence = evidence; ml_loglik = ll; posterior = prior +. likelihood; dl_bits = dl; diff --git a/test/test_scfg.ml b/test/test_scfg.ml index 386c39f..e6f7cad 100644 --- a/test/test_scfg.ml +++ b/test/test_scfg.ml @@ -148,14 +148,14 @@ let () = let cfg = { Scoring.default_config with alpha = 1.0; em_iters = 5 } in let g1 = g "S" [ prod "S" [ tm "a" ] ] in let m1, _, _, _ = - Scoring.marginal_loglik ~config:cfg g1 [ sample [ "a" ] 1.0 ] + Scoring.approximate_log_evidence ~config:cfg g1 [ sample [ "a" ] 1.0 ] in - check "marginal(S->a, {a}, alpha=1) = 0" (approx m1 0.0); + check "evidence(S->a, {a}, alpha=1) = 0" (approx m1 0.0); let g2 = g "S" [ prod "S" [ tm "a" ]; prod "S" [ tm "b" ] ] in let m2, _, _, _ = - Scoring.marginal_loglik ~config:cfg g2 [ sample [ "a" ] 1.0 ] + Scoring.approximate_log_evidence ~config:cfg g2 [ sample [ "a" ] 1.0 ] in - check "marginal(S->a|b, {a}, alpha=1) = log 1/2" + check "evidence(S->a|b, {a}, alpha=1) = log 1/2" (approx m2 (log 0.5)) let paper_initial () = @@ -308,9 +308,11 @@ let () = check "variational ELBO (single production) = 0" (approx d1.Scoring.posterior 0.0 ~tol:1e-6); let g2 = g "S" [ prod "S" [ tm "a" ]; prod "S" [ tm "b" ] ] in - let dm = Scoring.details ~config:(mk Scoring.Marginal) g2 corpus in + let dm = + Scoring.details ~config:(mk Scoring.Approximate_evidence) g2 corpus + in let dv = Scoring.details ~config:(mk Scoring.Variational) g2 corpus in - check "variational matches marginal (unambiguous)" + check "variational matches approximate evidence (unambiguous)" (approx dv.Scoring.posterior dm.Scoring.posterior ~tol:1e-6) let () =