let tokenize ?(is_whitespace=is_whitespace) ?(tokens=[]) str =
let lst = ref [] in
let buf = Buffer.create 13 in
let idx = ref 0 in
let push () =
if Buffer.length buf > 0 then
begin
lst := Buffer.contents buf :: !lst;
Buffer.clear buf
end
in
let match_token () =
List.exists
(fun tok ->
if starts_with ~what:tok ~offset:!idx str then
begin
push ();
lst := tok :: !lst;
idx := !idx + (String.length tok);
true
end
else
false)
tokens
in
while !idx < String.length str do
let c = str.[!idx] in
if is_whitespace c then
begin
push ();
incr idx
end
else if match_token () then
begin
()
end
else
begin
Buffer.add_char buf c;
incr idx
end
done;
push ();
List.rev !lst