sneeker
4d26b548ba
Words, strings, braces and comments, each carrying its source range, so an error points at the offending token rather than the whole line.
99 lines
2.8 KiB
OCaml
99 lines
2.8 KiB
OCaml
type kind =
|
|
| Word of string
|
|
| Str of string
|
|
| Lbrace
|
|
| Rbrace
|
|
| Lparen
|
|
| Rparen
|
|
| Newline
|
|
| Eof
|
|
|
|
type t = { kind : kind; span : Span.t }
|
|
|
|
let describe = function
|
|
| Word w -> Printf.sprintf "%S" w
|
|
| Str _ -> "a string"
|
|
| Lbrace -> "'{'"
|
|
| Rbrace -> "'}'"
|
|
| Lparen -> "'('"
|
|
| Rparen -> "')'"
|
|
| Newline -> "end of line"
|
|
| Eof -> "end of file"
|
|
|
|
let is_space c = c = ' ' || c = '\t' || c = '\r'
|
|
let is_delim c = is_space c || c = '\n' || c = '{' || c = '}' || c = '(' || c = ')' || c = '#'
|
|
|
|
let escape = function
|
|
| 'n' -> '\n'
|
|
| 't' -> '\t'
|
|
| 'r' -> '\r'
|
|
| '0' -> '\000'
|
|
| c -> c
|
|
|
|
let tokens ~file src =
|
|
let n = String.length src in
|
|
let out = ref [] in
|
|
let line = ref 1 and bol = ref 0 in
|
|
let i = ref 0 in
|
|
let col () = !i - !bol + 1 in
|
|
let emit kind start_col len = out := { kind; span = Span.make file !line start_col len } :: !out in
|
|
while !i < n do
|
|
let c = src.[!i] in
|
|
let c0 = col () in
|
|
if is_space c then incr i
|
|
else if c = '#' then while !i < n && src.[!i] <> '\n' do incr i done
|
|
else if c = '\\' && !i + 1 < n && src.[!i + 1] = '\n' then begin
|
|
i := !i + 2;
|
|
incr line;
|
|
bol := !i
|
|
end
|
|
else if c = '\n' then begin
|
|
emit Newline c0 1;
|
|
incr i;
|
|
incr line;
|
|
bol := !i
|
|
end
|
|
else if c = '{' then (emit Lbrace c0 1; incr i)
|
|
else if c = '}' then (emit Rbrace c0 1; incr i)
|
|
else if c = '(' then (emit Lparen c0 1; incr i)
|
|
else if c = ')' then (emit Rparen c0 1; incr i)
|
|
else if c = '"' then begin
|
|
let buf = Buffer.create 16 in
|
|
let start_line = !line in
|
|
incr i;
|
|
let closed = ref false in
|
|
while !i < n && not !closed do
|
|
if src.[!i] = '"' then (closed := true; incr i)
|
|
else if src.[!i] = '\\' && !i + 1 < n then begin
|
|
Buffer.add_char buf (escape src.[!i + 1]);
|
|
i := !i + 2
|
|
end
|
|
else begin
|
|
if src.[!i] = '\n' then (incr line; bol := !i + 1);
|
|
Buffer.add_char buf src.[!i];
|
|
incr i
|
|
end
|
|
done;
|
|
if not !closed then
|
|
Diag.error ~span:(Span.make file start_line c0 1) "unterminated string";
|
|
emit (Str (Buffer.contents buf)) c0 (max 1 (col () - c0))
|
|
end
|
|
else begin
|
|
let start = !i in
|
|
let fin = ref false in
|
|
while !i < n && not !fin do
|
|
if src.[!i] = '$' && !i + 1 < n && src.[!i + 1] = '{' then begin
|
|
i := !i + 2;
|
|
while !i < n && src.[!i] <> '}' && src.[!i] <> '\n' do incr i done;
|
|
if !i < n && src.[!i] = '}' then incr i
|
|
else Diag.error ~span:(Span.make file !line c0 (!i - start)) "unclosed ${...}"
|
|
end
|
|
else if is_delim src.[!i] then fin := true
|
|
else incr i
|
|
done;
|
|
emit (Word (String.sub src start (!i - start))) c0 (!i - start)
|
|
end
|
|
done;
|
|
emit Eof (col ()) 1;
|
|
List.rev !out
|