Strip a byte order mark if present

dtolnay · Sep 29, 2022 · e2327f0 · e2327f0
1 parent eeb2e5b
commit e2327f0
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/src/fallback.rs b/src/fallback.rs
@@ -182,7 +182,13 @@ impl FromStr for TokenStream {
 
     fn from_str(src: &str) -> Result<TokenStream, LexError> {
         // Create a dummy file & add it to the source map
-        let cursor = get_cursor(src);
+        let mut cursor = get_cursor(src);
+
+        // Strip a byte order mark if present
+        const BYTE_ORDER_MARK: &str = "\u{feff}";
+        if cursor.starts_with(BYTE_ORDER_MARK) {
+            cursor = cursor.advance(BYTE_ORDER_MARK.len());
+        }
 
         parse::token_stream(cursor)
     }

diff --git a/src/parse.rs b/src/parse.rs
@@ -14,7 +14,7 @@ pub(crate) struct Cursor<'a> {
 }
 
 impl<'a> Cursor<'a> {
-    fn advance(&self, bytes: usize) -> Cursor<'a> {
+    pub fn advance(&self, bytes: usize) -> Cursor<'a> {
         let (_front, rest) = self.rest.split_at(bytes);
         Cursor {
             rest,
@@ -23,7 +23,7 @@ impl<'a> Cursor<'a> {
         }
     }
 
-    fn starts_with(&self, s: &str) -> bool {
+    pub fn starts_with(&self, s: &str) -> bool {
         self.rest.starts_with(s)
     }