Manual impl of Debug on Token (#11958)

## Summary

I look at the token stream a lot, not specifically in the playground but
in the terminal output and it's annoying to scroll a lot to find
specific location. Most of the information is also redundant.

The final format we end up with is: `<kind> <range> (flags = ...)` e.g.,
`String 0..4 (flags = BYTE_STRING)` where the flags part is only
populated if there are any flags set.

crates/ruff_dev/Cargo.toml

@@ -22,7 +22,6 @@ ruff_python_formatter = { workspace = true }
 ruff_python_parser = { workspace = true }
 ruff_python_stdlib = { workspace = true }
 ruff_python_trivia = { workspace = true }
-ruff_text_size = { workspace = true }
 ruff_workspace = { workspace = true, features = ["schemars"] }
 
 anyhow = { workspace = true }

crates/ruff_dev/src/print_tokens.rs

@@ -8,7 +8,6 @@ use anyhow::Result;
 use ruff_linter::source_kind::SourceKind;
 use ruff_python_ast::PySourceType;
 use ruff_python_parser::parse_unchecked_source;
-use ruff_text_size::Ranged;
 
 #[derive(clap::Args)]
 pub(crate) struct Args {
@@ -27,12 +26,7 @@ pub(crate) fn main(args: &Args) -> Result<()> {
     })?;
     let parsed = parse_unchecked_source(source_kind.source_code(), source_type);
     for token in parsed.tokens() {
-        println!(
-            "{start:#?} {kind:#?} {end:#?}",
-            start = token.start(),
-            end = token.end(),
-            kind = token.kind(),
-        );
+        println!("{token:#?}");
     }
     Ok(())
 }

crates/ruff_python_parser/src/token.rs

@@ -16,7 +16,7 @@ use ruff_python_ast::str_prefix::{
 use ruff_python_ast::{AnyStringFlags, BoolOp, Int, IpyEscapeKind, Operator, StringFlags, UnaryOp};
 use ruff_text_size::{Ranged, TextRange};
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Clone, Copy, PartialEq, Eq)]
 pub struct Token {
     /// The kind of the token.
     kind: TokenKind,
@@ -81,6 +81,26 @@ impl Ranged for Token {
     }
 }
 
+impl fmt::Debug for Token {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?} {:?}", self.kind, self.range)?;
+        if !self.flags.is_empty() {
+            f.write_str(" (flags = ")?;
+            let mut first = true;
+            for (name, _) in self.flags.iter_names() {
+                if first {
+                    first = false;
+                } else {
+                    f.write_str(" | ")?;
+                }
+                f.write_str(name)?;
+            }
+            f.write_str(")")?;
+        }
+        Ok(())
+    }
+}
+
 /// A kind of a token.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)]
 pub enum TokenKind {

crates/ruff_wasm/src/lib.rs

@@ -261,7 +261,7 @@ impl Workspace {
     pub fn tokens(&self, contents: &str) -> Result<String, Error> {
         let parsed = parse_unchecked(contents, Mode::Module);
 
-        Ok(format!("{:#?}", parsed.tokens()))
+        Ok(format!("{:#?}", parsed.tokens().as_ref()))
     }
 }