Invalid error span for eoi

[KrosFire]

I'm using my own lexer that returns vector of spanned tokens like this:

pub struct Token(pub String);
pub type Spanned<T> = (T, Span);

fn tokenize(input: &str) -> Vec<Spanned<Token>>

For input like this:

let input = "2 as";
let tokens = tokenize(input); // [(Token("2"), 0..1), (Token("as"), 2..4)]

I've noticed an incorrect span for error when compiler looks something like this:

fn ident<'a>(ident_name: String) -> ... {
    any()
        .try_map(move |token: Token, span| {
            let word = token.to_string();
            let mut chars = word.chars();

            let first_char = chars.next().unwrap();

            if !first_char.is_ascii_alphabetic() && first_char != '_' {
                return Err(Rich::custom(
                    span,
                    "identifier must start with a letter or an underscore",
                ));
            }

            for char in chars {
                if !char.is_ascii_alphanumeric() && char != '_' {
                    return Err(Rich::custom(
                        span,
                        "identifier must contain only alphanumeric characters or underscores",
                    ));
                }
            }

            if KEYWORDS.contains(&word.as_str()) {
                return Err(Rich::custom(
                    span,
                    format!("keyword used as {ident_name} name"),
                ));
            }

            Ok(word)
        })
}

let int = any().try_map(|token: Token, span: SimpleSpan| {
    let word = token.to_string();

    for char in word.chars() {
        if !char.is_ascii_digit() {
            return Err(Rich::custom(span, "int must contain only digits"));
        }
    }

    Ok(word)
});

let number = int
    .then(just(T!['.']).ignore_then(int).or_not())
    .map(|(int, float)| {
        let float = float.unwrap_or('0'.to_string());

        format!("{}.{}", int, float)
    })
    .from_str::<f32>()
    .unwrapped()
    .map_with(|num, e| (Expression::Number((num, e.span())), e.span()))
    .boxed();

let parser = number
    .then(
        just(T!["as"])
            .ignore_then(
                ident("type".to_string())
                    .map_with(|txt, e| (txt, e.span()))
                    .recover_with(via_parser(
                        any()
                            .or(end().map(|_| T![""]))
                            .map_with(|_, e| ("ERROR".to_string(), e.span())),
                    )),
            )
            .boxed(),
    )
    .map_with(|(exp, ty), e| (Expression::Cast(Box::new(exp), ty), e.span()))
    .boxed();

let spanned_tokens = tokens.spanned(SimpleSpan::new(eoi, eoi));

parser.parse(spanned_tokens)

Parsing results in:

output: Some((
Cast(
  (Number((2.0, 0..1)), 0..1),
  ("ERROR", 0..4),
), 0..4))
errors: "found end of input at 0..4 expected something else"

Error has a span of 0..4 but It should have 4..4

[zesterer]

Interesting, that seems odd. Any chance you could cut bits off until you have a minimum testable example that exhibits the behaviour?

[KrosFire]

Shorter example:

let parser = just(T!["as"])
    .map_with(
        |t, e| (t, e.span()),
    )
    .then(
        just(T!["text"])
            .recover_with(via_parser(any().or_not().map(|_| T!["error"])))
            .map_with(|t, e| (t, e.span())),
    )
    .map_with(|(t1, t2), e| (t1, t2, e.span()));

For input: "as"

tokens: [("as", 0..2)]

eoi: 2..2

Result is:

output: Some(("as", 0..2), ("error", 0..2), 0..2)
errs: found end of input at 0..2 expected "text"