Krolyxon
Commit 815d036

Branches

Tags

Refs →

bench51cratesbench51-coresrcasmlexer.rs

175 lines · 5.2 KB · Rust

Raw Blame History Download
1#[derive(Debug, Clone, PartialEq)]
2pub enum Tok {
3 Ident(String),
4 Num(i64),
5 Str(Vec<u8>),
6 Punct(char),
7 /// `<<` and `>>`
8 Shl,
9 Shr,
10}
11
12#[derive(Debug, Clone)]
13pub struct Token {
14 pub tok: Tok,
15 pub col: usize,
16}
17
18impl Token {
19 pub fn ident(&self) -> Option<&str> {
20 match &self.tok {
21 Tok::Ident(s) => Some(s),
22 _ => None,
23 }
24 }
25
26 pub fn is_punct(&self, c: char) -> bool {
27 self.tok == Tok::Punct(c)
28 }
29}
30
31/// Splits one source line into tokens, stopping at a `;` comment.
32/// Identifiers are upper-cased (the assembler is case-insensitive).
33pub fn lex(line: &str) -> Result<Vec<Token>, (usize, String)> {
34 let chars: Vec<char> = line.chars().collect();
35 let mut out = Vec::new();
36 let mut i = 0;
37 while i < chars.len() {
38 let c = chars[i];
39 let col = i;
40 if c.is_whitespace() {
41 i += 1;
42 } else if c == ';' {
43 break;
44 } else if c.is_ascii_alphabetic() || c == '_' || c == '?' {
45 let start = i;
46 while i < chars.len()
47 && (chars[i].is_ascii_alphanumeric() || chars[i] == '_' || chars[i] == '?')
48 {
49 i += 1;
50 }
51 let s: String = chars[start..i].iter().collect();
52 out.push(Token {
53 tok: Tok::Ident(s.to_ascii_uppercase()),
54 col,
55 });
56 } else if c.is_ascii_digit() {
57 let start = i;
58 while i < chars.len() && chars[i].is_ascii_alphanumeric() {
59 i += 1;
60 }
61 let s: String = chars[start..i].iter().collect();
62 let n = parse_number(&s).ok_or((col, format!("invalid number '{s}'")))?;
63 out.push(Token {
64 tok: Tok::Num(n),
65 col,
66 });
67 } else if c == '\'' || c == '"' {
68 i += 1;
69 let mut bytes = Vec::new();
70 loop {
71 match chars.get(i) {
72 None => return Err((col, "unterminated string".into())),
73 // A doubled quote inside a string stands for the quote itself.
74 Some(&q) if q == c && chars.get(i + 1) == Some(&c) => {
75 bytes.push(c as u8);
76 i += 2;
77 }
78 Some(&q) if q == c => {
79 i += 1;
80 break;
81 }
82 Some(&ch) => {
83 let mut buf = [0u8; 4];
84 bytes.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes());
85 i += 1;
86 }
87 }
88 }
89 out.push(Token {
90 tok: Tok::Str(bytes),
91 col,
92 });
93 } else if c == '<' && chars.get(i + 1) == Some(&'<') {
94 out.push(Token { tok: Tok::Shl, col });
95 i += 2;
96 } else if c == '>' && chars.get(i + 1) == Some(&'>') {
97 out.push(Token { tok: Tok::Shr, col });
98 i += 2;
99 } else if "+-*/%()#@.,:$~&|^".contains(c) {
100 out.push(Token {
101 tok: Tok::Punct(c),
102 col,
103 });
104 i += 1;
105 } else {
106 return Err((col, format!("unexpected character '{c}'")));
107 }
108 }
109 Ok(out)
110}
111
112/// Number literals: `123`, `123d`, `0x1F`, `1Fh`, `0b101`, `101b`, `17o`, `17q`.
113pub fn parse_number(s: &str) -> Option<i64> {
114 let l = s.to_ascii_lowercase();
115 let (digits, radix) = if let Some(r) = l.strip_prefix("0x") {
116 (r, 16)
117 } else if let Some(r) = l.strip_suffix('h') {
118 (r, 16)
119 } else if let Some(r) = l
120 .strip_prefix("0b")
121 .filter(|r| !r.is_empty() && r.chars().all(|c| c == '0' || c == '1'))
122 {
123 (r, 2)
124 } else if let Some(r) = l
125 .strip_suffix('b')
126 .filter(|r| r.chars().all(|c| c == '0' || c == '1'))
127 {
128 (r, 2)
129 } else if let Some(r) = l.strip_suffix('o').or_else(|| l.strip_suffix('q')) {
130 (r, 8)
131 } else if let Some(r) = l.strip_suffix('d') {
132 (r, 10)
133 } else {
134 (l.as_str(), 10)
135 };
136 if digits.is_empty() {
137 return None;
138 }
139 i64::from_str_radix(digits, radix).ok()
140}
141
142#[cfg(test)]
143mod tests {
144 use super::*;
145
146 #[test]
147 fn numbers() {
148 assert_eq!(parse_number("0FFh"), Some(255));
149 assert_eq!(parse_number("0bh"), Some(11));
150 assert_eq!(parse_number("0b101"), Some(5));
151 assert_eq!(parse_number("1010b"), Some(10));
152 assert_eq!(parse_number("0x1f"), Some(31));
153 assert_eq!(parse_number("17o"), Some(15));
154 assert_eq!(parse_number("42"), Some(42));
155 assert_eq!(parse_number("12x"), None);
156 }
157
158 #[test]
159 fn lexes_line() {
160 let t = lex("loop: mov a, #'x' ; comment").unwrap();
161 let toks: Vec<_> = t.into_iter().map(|t| t.tok).collect();
162 assert_eq!(
163 toks,
164 vec![
165 Tok::Ident("LOOP".into()),
166 Tok::Punct(':'),
167 Tok::Ident("MOV".into()),
168 Tok::Ident("A".into()),
169 Tok::Punct(','),
170 Tok::Punct('#'),
171 Tok::Str(vec![b'x']),
172 ]
173 );
174 }
175}