Improve matcher string: support unicode string

nextest-filtering/Cargo.toml

@@ -24,4 +24,3 @@ nom-tracable = "0.8.0"
 nom_locate = "4.0.0"
 regex = "1.5.5"
 thiserror = "1.0.30"
-unicode-xid = "0.2.2"

nextest-filtering/src/compile.rs

@@ -7,8 +7,10 @@ use guppy::{
 };
 use std::collections::HashSet;
 
-use crate::expression::*;
-use crate::parsing::{Expr, SetDef};
+use crate::{
+    expression::*,
+    parsing::{Expr, SetDef},
+};
 
 pub(crate) fn compile(expr: &Expr, graph: &PackageGraph) -> FilteringExpr {
     let in_workspace_packages: Vec<_> = graph

nextest-filtering/src/error.rs

@@ -16,14 +16,14 @@ pub enum Error {
     InvalidRegex(#[label("Invalid regex")] SourceSpan),
     #[error("Expected close regex")]
     ExpectedCloseRegex(#[label("Missing '/'")] SourceSpan),
-    #[error("Invalid identifier")]
-    InvalidIdentifier(
-        #[label("Identifier can only contain xid_continue characters and ':'")] SourceSpan,
-    ),
-    #[error("Expected identifier")]
-    ExpectedIdentifier(#[label("Missing identifier")] SourceSpan),
+    #[error("Expected matcher input")]
+    ExpectedMatcherInput(#[label("Missing matcher content")] SourceSpan),
     #[error("Unexpected name matcher")]
     UnexpectedNameMatcher(#[label("This set doesn't take en argument")] SourceSpan),
+    #[error("Invalid unicode string")]
+    InvalidUnicodeString(#[label("This is not a valid unicode string")] SourceSpan),
+    #[error("Expected close string")]
+    ExpectedCloseQuote(#[label("Missing `'`")] SourceSpan),
     #[error("Expected open parentheses")]
     ExpectedOpenParenthesis(#[label("Missing '('")] SourceSpan),
     #[error("Expected close parentheses")]

nextest-filtering/src/expression.rs

@@ -4,8 +4,10 @@
 use guppy::{graph::PackageGraph, PackageId};
 use std::{cell::RefCell, collections::HashSet};
 
-use crate::error::{Error, FilteringExprParsingError, State};
-use crate::parsing::{parse, ParsedExpr, Span};
+use crate::{
+    error::{Error, FilteringExprParsingError, State},
+    parsing::{parse, ParsedExpr, Span},
+};
 
 /// Matcher for name
 ///

nextest-filtering/src/parsing.rs

@@ -16,17 +16,17 @@
 use miette::SourceSpan;
 use nom::{
     branch::alt,
-    bytes::complete::{tag, take_till, take_until, take_while1},
+    bytes::complete::{tag, take_till, take_until},
     character::complete::char,
     combinator::{eof, map, recognize},
     multi::{fold_many0, many0},
     sequence::{delimited, pair, preceded, terminated},
 };
 use nom_tracable::tracable_parser;
-use unicode_xid::UnicodeXID;
 
-use crate::error::*;
-use crate::NameMatcher;
+mod unicode_string;
+
+use crate::{error::*, NameMatcher};
 
 pub type Span<'a> = nom_locate::LocatedSpan<&'a str, State<'a>>;
 type IResult<'a, T> = nom::IResult<Span<'a>, T>;
@@ -190,89 +190,78 @@ fn ws<'a, T, P: FnMut(Span<'a>) -> IResult<'a, T>>(
     }
 }
 
-fn is_identifier_char(c: char) -> bool {
-    // This is use for NameMatcher::Contains(_) and NameMatcher::Equal(_)
-    // The output should be valid part of a test-name or a package name.
-    c == ':' || c.is_xid_continue()
+// This parse will never fail
+#[tracable_parser]
+fn parse_matcher_text_no_quote(input: Span) -> IResult<Option<String>> {
+    map(
+        take_till::<_, _, nom::error::Error<Span>>(|c| c == ')'),
+        |res: Span| Some(res.fragment().trim().to_string()),
+    )(input)
 }
 
 #[tracable_parser]
-fn parse_identifier_part(input: Span) -> IResult<Option<String>> {
-    let start = input.location_offset();
-    match map(
-        recognize::<_, _, nom::error::Error<Span>, _>(take_while1(is_identifier_char)),
-        |res: Span| res.fragment().to_string(),
-    )(input.clone())
-    {
-        Ok((i1, res1)) => {
-            match recognize::<_, _, nom::error::Error<Span>, _>(take_till(|c| c == ')'))(i1.clone())
-            {
-                Ok((i, res)) => {
-                    if res.fragment().trim().is_empty() {
-                        Ok((i1, Some(res1)))
-                    } else {
-                        let end = i.location_offset() - start;
-                        let err = Error::InvalidIdentifier((start, end).into());
+fn parse_matcher_text_quote(input: Span) -> IResult<Option<String>> {
+    let (input, _) = ws(char('\''))(input)?;
+
+    let (i, res) =
+        match expect(unicode_string::parse_string, Error::InvalidUnicodeString)(input.clone()) {
+            Ok((i, res)) => (i, res),
+            Err(nom::Err::Incomplete(_)) => {
+                // no closing `'` found for this string
+                match take_till::<_, _, nom::error::Error<Span>>(|c| c == ')')(input.clone()) {
+                    Ok((i, _)) => {
+                        let start = i.location_offset();
+                        let err = Error::ExpectedCloseQuote((start, 0).into());
                         i.extra.report_error(err);
-                        Ok((i, None))
+                        return Ok((i, None));
                     }
+                    Err(_) => unreachable!(),
                 }
-                Err(_) => unreachable!(),
             }
-        }
-        Err(_) => {
-            match recognize::<_, _, nom::error::Error<Span>, _>(take_till(|c| c == ')'))(input) {
-                Ok((i, res)) => {
-                    let end = i.location_offset() - start;
-                    let err = if res.fragment().trim().is_empty() {
-                        Error::ExpectedIdentifier((start, end).into())
-                    } else {
-                        Error::InvalidIdentifier((start, end).into())
-                    };
-                    i.extra.report_error(err);
-                    Ok((i, None))
-                }
-                Err(_) => unreachable!(),
-            }
-        }
-    }
+            Err(_) => unreachable!(),
+        };
+
+    // by construction this can not fail
+    let (i, _) = char::<_, nom::error::Error<Span>>('\'')(i).unwrap();
+    Ok((i, res))
+}
+
+// This parse will never fail (because parse_matcher_text_no_quote never fails)
+#[tracable_parser]
+fn parse_matcher_text(input: Span) -> IResult<Option<String>> {
+    alt((parse_matcher_text_quote, parse_matcher_text_no_quote))(input)
 }
 
 // This parse will never fail
 #[tracable_parser]
 fn parse_contains_matcher(input: Span) -> IResult<Option<NameMatcher>> {
-    ws(map(parse_identifier_part, |res: Option<String>| {
+    map(parse_matcher_text, |res: Option<String>| {
         res.map(NameMatcher::Contains)
-    }))(input)
+    })(input)
 }
 
 #[tracable_parser]
 fn parse_equal_matcher(input: Span) -> IResult<Option<NameMatcher>> {
     ws(map(
-        preceded(char('='), ws(parse_identifier_part)),
+        preceded(char('='), parse_matcher_text),
         |res: Option<String>| res.map(NameMatcher::Equal),
     ))(input)
 }
 
 #[tracable_parser]
 fn parse_regex_(input: Span) -> IResult<Option<NameMatcher>> {
-    let (i, res) =
-        match recognize::<_, _, nom::error::Error<Span>, _>(take_until("/"))(input.clone()) {
-            Ok((i, res)) => (i, res),
-            Err(_) => {
-                match recognize::<_, _, nom::error::Error<Span>, _>(take_till(|c| c == ')'))(
-                    input.clone(),
-                ) {
-                    Ok((i, res)) => {
-                        let start = i.location_offset();
-                        let err = Error::ExpectedCloseRegex((start, 0).into());
-                        i.extra.report_error(err);
-                        (i, res)
-                    }
-                    Err(_) => return Ok((input, None)),
-                }
+    let (i, res) = match take_until::<_, _, nom::error::Error<Span>>("/")(input.clone()) {
+        Ok((i, res)) => (i, res),
+        Err(_) => match take_till::<_, _, nom::error::Error<Span>>(|c| c == ')')(input.clone()) {
+            Ok((i, _)) => {
+                let start = i.location_offset();
+                let err = Error::ExpectedCloseRegex((start, 0).into());
+                i.extra.report_error(err);
+                return Ok((i, None));
             }
-        };
+            Err(_) => unreachable!(),
+        },
+    };
     match regex::Regex::new(res.fragment()).map(NameMatcher::Regex) {
         Ok(res) => Ok((i, Some(res))),
         _ => {
@@ -462,6 +451,31 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_parse_name_matcher_quote() {
+        assert_eq!(
+            SetDef::Test(NameMatcher::Contains("something".to_string())),
+            parse_set("test('something')")
+        );
+        assert_eq!(
+            SetDef::Test(NameMatcher::Equal("something".to_string())),
+            parse_set("test(='something')")
+        );
+
+        assert_eq!(
+            SetDef::Test(NameMatcher::Contains(r"some'thing".to_string())),
+            parse_set(r"test('some\'thing')")
+        );
+        assert_eq!(
+            SetDef::Test(NameMatcher::Contains(r"some(thing)".to_string())),
+            parse_set(r"test('some(thing)')")
+        );
+        assert_eq!(
+            SetDef::Test(NameMatcher::Contains(r"some U".to_string())),
+            parse_set(r"test('some \u{55}')")
+        );
+    }
+
     #[test]
     fn test_parse_set_def() {
         assert_eq!(SetDef::All, parse_set("all()"));
@@ -625,15 +639,6 @@ mod tests {
         assert_error!(error, ExpectedCloseRegex, 12, 0);
     }
 
-    #[test]
-    fn test_invalid_identifier() {
-        let src = "package(a aa)";
-        let mut errors = parse_err(src);
-        assert_eq!(1, errors.len());
-        let error = errors.remove(0);
-        assert_error!(error, InvalidIdentifier, 8, 4);
-    }
-
     #[test]
     fn test_unexpected_argument() {
         let src = "all(aaa)";
@@ -665,12 +670,19 @@ mod tests {
     fn test_complex_error() {
         let src = "all) + package(/not) - deps(expr none)";
         let mut errors = parse_err(src);
-        assert_eq!(3, errors.len(), "{:?}", errors);
+        assert_eq!(2, errors.len(), "{:?}", errors);
         let error = errors.remove(0);
         assert_error!(error, ExpectedOpenParenthesis, 3, 0);
         let error = errors.remove(0);
         assert_error!(error, ExpectedCloseRegex, 19, 0);
+    }
+
+    #[test]
+    fn test_missing_string_close() {
+        let src = "test('thing)";
+        let mut errors = parse_err(src);
+        assert_eq!(1, errors.len(), "{:?}", errors);
         let error = errors.remove(0);
-        assert_error!(error, InvalidIdentifier, 28, 9);
+        assert_error!(error, ExpectedCloseQuote, 11, 0);
     }
 }