| 1 | //! Minimal 8051 assembly tokenizer for syntax highlighting. |
| 2 | |
| 3 | use std::ops::Range; |
| 4 | |
| 5 | use crate::{isa, sfr}; |
| 6 | |
| 7 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 8 | pub enum Tok { |
| 9 | /// Whitespace, punctuation and identifiers without a special meaning. |
| 10 | Plain, |
| 11 | Comment, |
| 12 | Str, |
| 13 | /// A label definition (`name:` at the start of a line). |
| 14 | Label, |
| 15 | Mnemonic, |
| 16 | /// Directives and expression operators (`ORG`, `DB`, `HIGH`, `MOD`, …). |
| 17 | Directive, |
| 18 | Register, |
| 19 | /// Predefined SFR and bit names. |
| 20 | Sfr, |
| 21 | Number, |
| 22 | } |
| 23 | |
| 24 | fn is_directive(w: &str) -> bool { |
| 25 | matches!( |
| 26 | w, |
| 27 | "ORG" |
| 28 | | "DB" |
| 29 | | "DW" |
| 30 | | "DS" |
| 31 | | "END" |
| 32 | | "EQU" |
| 33 | | "SET" |
| 34 | | "BIT" |
| 35 | | "DATA" |
| 36 | | "IDATA" |
| 37 | | "XDATA" |
| 38 | | "CODE" |
| 39 | | "CSEG" |
| 40 | | "XSEG" |
| 41 | | "DSEG" |
| 42 | | "AT" |
| 43 | | "USING" |
| 44 | | "NAME" |
| 45 | | "BYTE" |
| 46 | | "WORD" |
| 47 | ) |
| 48 | } |
| 49 | |
| 50 | fn word_kind(w: &str, first_word: bool) -> Tok { |
| 51 | let u = w.to_ascii_uppercase(); |
| 52 | let reg = matches!(u.as_str(), "A" | "AB" | "C" | "DPTR" | "PC") |
| 53 | || (u.len() == 2 |
| 54 | && u.starts_with('R') |
| 55 | && u.as_bytes()[1].is_ascii_digit() |
| 56 | && u.as_bytes()[1] < b'8'); |
| 57 | if reg { |
| 58 | Tok::Register |
| 59 | } else if is_directive(&u) { |
| 60 | Tok::Directive |
| 61 | } else if first_word && isa::is_mnemonic(&u) { |
| 62 | Tok::Mnemonic |
| 63 | } else if sfr::SFRS.iter().chain(sfr::BITS).any(|(n, _)| *n == u) { |
| 64 | Tok::Sfr |
| 65 | } else if matches!( |
| 66 | u.as_str(), |
| 67 | "HIGH" | "LOW" | "MOD" | "AND" | "OR" | "XOR" | "NOT" | "SHL" | "SHR" |
| 68 | ) { |
| 69 | Tok::Directive |
| 70 | } else { |
| 71 | Tok::Plain |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | /// Splits a source line into classified byte ranges that together cover |
| 76 | /// the whole line. |
| 77 | pub fn tokens(line: &str) -> Vec<(Range<usize>, Tok)> { |
| 78 | let b = line.as_bytes(); |
| 79 | let n = b.len(); |
| 80 | let mut out = Vec::new(); |
| 81 | let mut i = 0; |
| 82 | let mut seen_word = false; |
| 83 | while i < n { |
| 84 | let c = b[i]; |
| 85 | let start = i; |
| 86 | let tok = if c == b';' { |
| 87 | i = n; |
| 88 | Tok::Comment |
| 89 | } else if c == b'\'' || c == b'"' { |
| 90 | i += 1; |
| 91 | while i < n && b[i] != c { |
| 92 | i += 1; |
| 93 | } |
| 94 | i = (i + 1).min(n); |
| 95 | Tok::Str |
| 96 | } else if c.is_ascii_alphabetic() || c == b'_' || c == b'?' { |
| 97 | while i < n && (b[i].is_ascii_alphanumeric() || b[i] == b'_' || b[i] == b'?') { |
| 98 | i += 1; |
| 99 | } |
| 100 | if !seen_word && b.get(i) == Some(&b':') { |
| 101 | Tok::Label |
| 102 | } else { |
| 103 | let t = word_kind(&line[start..i], !seen_word); |
| 104 | seen_word = true; |
| 105 | t |
| 106 | } |
| 107 | } else if c.is_ascii_digit() || c == b'#' || c == b'$' { |
| 108 | i += 1; |
| 109 | while i < n && b[i].is_ascii_alphanumeric() { |
| 110 | i += 1; |
| 111 | } |
| 112 | Tok::Number |
| 113 | } else { |
| 114 | // Non-ASCII bytes land here too, so ranges stay on char boundaries. |
| 115 | while i < n && !(b[i].is_ascii_alphanumeric() || b"_?;'\"#$".contains(&b[i])) { |
| 116 | i += 1; |
| 117 | } |
| 118 | Tok::Plain |
| 119 | }; |
| 120 | out.push((start..i, tok)); |
| 121 | } |
| 122 | out |
| 123 | } |
| 124 | |
| 125 | #[cfg(test)] |
| 126 | mod tests { |
| 127 | use super::{Tok, tokens}; |
| 128 | |
| 129 | fn kinds(line: &str) -> Vec<(&str, Tok)> { |
| 130 | tokens(line) |
| 131 | .into_iter() |
| 132 | .filter(|(_, t)| *t != Tok::Plain) |
| 133 | .map(|(r, t)| (&line[r], t)) |
| 134 | .collect() |
| 135 | } |
| 136 | |
| 137 | #[test] |
| 138 | fn classifies_an_instruction_line() { |
| 139 | assert_eq!( |
| 140 | kinds("main: MOV TMOD,#20h ; timer 1"), |
| 141 | vec![ |
| 142 | ("main", Tok::Label), |
| 143 | ("MOV", Tok::Mnemonic), |
| 144 | ("TMOD", Tok::Sfr), |
| 145 | ("#20h", Tok::Number), |
| 146 | ("; timer 1", Tok::Comment), |
| 147 | ] |
| 148 | ); |
| 149 | assert_eq!( |
| 150 | kinds(" mov a,r7"), |
| 151 | vec![ |
| 152 | ("mov", Tok::Mnemonic), |
| 153 | ("a", Tok::Register), |
| 154 | ("r7", Tok::Register) |
| 155 | ] |
| 156 | ); |
| 157 | } |
| 158 | |
| 159 | #[test] |
| 160 | fn directives_strings_and_operators() { |
| 161 | assert_eq!( |
| 162 | kinds("msg: DB \"Hi; there\", HIGH(x) ; c"), |
| 163 | vec![ |
| 164 | ("msg", Tok::Label), |
| 165 | ("DB", Tok::Directive), |
| 166 | ("\"Hi; there\"", Tok::Str), |
| 167 | ("HIGH", Tok::Directive), |
| 168 | ("; c", Tok::Comment), |
| 169 | ] |
| 170 | ); |
| 171 | } |
| 172 | |
| 173 | #[test] |
| 174 | fn ranges_cover_the_line_even_with_unicode() { |
| 175 | for line in [ |
| 176 | "", |
| 177 | " ", |
| 178 | "x DB 'é', 1 ; µs → ok", |
| 179 | "\"unterminated", |
| 180 | "€€ MOV", |
| 181 | ] { |
| 182 | let t = tokens(line); |
| 183 | let mut pos = 0; |
| 184 | for (r, _) in &t { |
| 185 | assert_eq!(r.start, pos, "{line:?}"); |
| 186 | assert!(line.is_char_boundary(r.end)); |
| 187 | pos = r.end; |
| 188 | } |
| 189 | assert_eq!(pos, line.len()); |
| 190 | } |
| 191 | } |
| 192 | } |