cerno_core/
labels.rs

1//! The answer alphabet.
2
3use cerno_types::MAX_OPTIONS;
4
5/// Single capital letters, in order.
6///
7/// Every entry is one token in every tokeniser we have measured, which is the whole point:
8/// multi-token answers ("Nein" arrives as `Ne` + `in`) cannot be read off a single-token
9/// distribution at all, and near-variants ("Ja", "JA", " Ja", "ja") split the mass across
10/// entries. A capital letter has neither problem.
11///
12/// The list stops at 20 because that is Ollama's `top_logprobs` ceiling — a 21st label could
13/// never be observed.
14const ALPHABET: [&str; MAX_OPTIONS] = [
15    "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S",
16    "T",
17];
18
19/// The label for a 0-based option index, or `None` past the end of the alphabet.
20pub fn label(index: usize) -> Option<&'static str> {
21    ALPHABET.get(index).copied()
22}
23
24/// The first `count` labels.
25pub fn labels(count: usize) -> Vec<&'static str> {
26    ALPHABET.iter().take(count).copied().collect()
27}
28
29/// Whether `token` is a spelling of `label`.
30///
31/// Matching ignores surrounding whitespace and letter case, because a tokeniser may hand back
32/// `" A"` where another hands back `"A"`, and a model may answer in lower case. Callers fold
33/// every matching token together rather than picking one.
34pub fn matches(token: &str, label: &str) -> bool {
35    let token = token.trim();
36    token.len() == label.len() && token.eq_ignore_ascii_case(label)
37}
38
39#[cfg(test)]
40mod tests {
41    use super::*;
42
43    #[test]
44    fn labels_start_at_a_and_run_in_order() {
45        assert_eq!(labels(3), vec!["A", "B", "C"]);
46        assert_eq!(label(0), Some("A"));
47        assert_eq!(label(19), Some("T"));
48    }
49
50    /// The alphabet must not outrun what the host can report.
51    #[test]
52    fn the_alphabet_stops_at_the_host_ceiling() {
53        assert_eq!(ALPHABET.len(), MAX_OPTIONS);
54        assert_eq!(label(MAX_OPTIONS), None);
55        assert_eq!(labels(99).len(), MAX_OPTIONS);
56    }
57
58    #[test]
59    fn matching_tolerates_leading_space_and_lower_case() {
60        assert!(matches("A", "A"));
61        assert!(matches(" A", "A"));
62        assert!(matches("a", "A"));
63        assert!(matches("  a  ", "A"));
64    }
65
66    /// `"AB"` is a different answer, not a sloppy `"A"`.
67    #[test]
68    fn matching_rejects_longer_tokens() {
69        assert!(!matches("AB", "A"));
70        assert!(!matches("A)", "A"));
71        assert!(!matches("B", "A"));
72        assert!(!matches("", "A"));
73    }
74}
cerno_core/labels.rs

cerno_core/
labels.rs
