feat: 🚧 add slack broadcast tokenizer

src/utils/markdown_parser/tokenizers/index.ts

@@ -1,3 +1,4 @@
 export * from "./slack_user_mention";
 export * from "./slack_channel_mention";
 export * from "./slack_user_group_mention";
+export * from "./slack_broadcast";

src/utils/markdown_parser/tokenizers/slack_broadcast/index.ts

@@ -0,0 +1 @@
+export * from "./tokenizer";

src/utils/markdown_parser/tokenizers/slack_broadcast/match.ts

@@ -0,0 +1,95 @@
+import type { INodeInterval, INodePoint } from "@yozora/character";
+import { AsciiCodePoint } from "@yozora/character";
+import type {
+  IMatchInlineHookCreator,
+  IResultOfFindDelimiters,
+  IResultOfProcessSingleDelimiter,
+  ITokenDelimiter,
+} from "@yozora/core-tokenizer";
+import { eatOptionalCharacters } from "@yozora/core-tokenizer";
+import { SlackBroadcastType, type IDelimiter, type IThis, type IToken, type T } from "./types";
+
+export const match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = function (api) {
+  return { findDelimiter, processSingleDelimiter };
+
+  function* findDelimiter(): IResultOfFindDelimiters<IDelimiter> {
+    const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints();
+    const blockStartIndex: number = api.getBlockStartIndex();
+    const blockEndIndex: number = api.getBlockEndIndex();
+
+    const potentialDelimiters: ITokenDelimiter[] = [];
+    for (let i = blockStartIndex; i < blockEndIndex; ++i) {
+      const c = nodePoints[i]?.codePoint;
+      if (
+        c === AsciiCodePoint.OPEN_ANGLE &&
+        nodePoints[i + 1]?.codePoint === AsciiCodePoint.NUMBER_SIGN
+      ) {
+        const j = eatOptionalCharacters(nodePoints, i + 2, blockEndIndex, AsciiCodePoint.AT_SIGN);
+        if (j < blockEndIndex) {
+          potentialDelimiters.push({
+            type: "opener",
+            startIndex: i,
+            endIndex: j,
+          });
+        }
+      } else if (c === AsciiCodePoint.CLOSE_ANGLE) {
+        potentialDelimiters.push({
+          type: "closer",
+          startIndex: i,
+          endIndex: i + 1,
+        });
+      }
+    }
+
+    let pIndex = 0;
+    let lastEndIndex = -1;
+    let delimiter: IDelimiter | null = null;
+    while (pIndex < potentialDelimiters.length) {
+      const [startIndex, endIndex] = yield delimiter;
+
+      if (lastEndIndex === endIndex) {
+        if (delimiter == null || delimiter.startIndex >= startIndex) continue;
+      }
+      lastEndIndex = endIndex;
+
+      let openerDelimiter: INodeInterval | null = null;
+      let closerDelimiter: INodeInterval | null = null;
+      for (; pIndex < potentialDelimiters.length; ++pIndex) {
+        const delimiter = potentialDelimiters[pIndex]!;
+        if (delimiter.startIndex >= startIndex && delimiter.type !== "closer") {
+          openerDelimiter = delimiter;
+          break;
+        }
+      }
+
+      for (let i = pIndex + 1; i < potentialDelimiters.length; ++i) {
+        const delimiter = potentialDelimiters[i]!;
+        if (delimiter.type === "closer") {
+          closerDelimiter = delimiter;
+          break;
+        }
+      }
+
+      if (closerDelimiter == null) return;
+
+      delimiter = {
+        type: "full",
+        startIndex: openerDelimiter!.startIndex,
+        endIndex: closerDelimiter.endIndex,
+        thickness: closerDelimiter.endIndex - closerDelimiter.startIndex,
+      };
+    }
+  }
+
+  function processSingleDelimiter(
+    delimiter: IDelimiter,
+  ): IResultOfProcessSingleDelimiter<T, IToken> {
+    const token: IToken = {
+      nodeType: SlackBroadcastType,
+      startIndex: delimiter.startIndex,
+      endIndex: delimiter.endIndex,
+      thickness: delimiter.thickness,
+    };
+    return [token];
+  }
+};

src/utils/markdown_parser/tokenizers/slack_broadcast/parse.ts

@@ -0,0 +1,21 @@
+import type { INodePoint } from "@yozora/character";
+import { calcStringFromNodePoints } from "@yozora/character";
+import type { IParseInlineHookCreator } from "@yozora/core-tokenizer";
+import { SlackBroadcastType, type INode, type IThis, type IToken, type T } from "./types";
+
+export const parse: IParseInlineHookCreator<T, IToken, INode, IThis> = function (api) {
+  return {
+    parse: (tokens) =>
+      tokens.map((token) => {
+        const nodePoints: ReadonlyArray<INodePoint> = api.getNodePoints();
+        let startIndex: number = token.startIndex + 2; // Skip `<#`
+        let endIndex: number = token.endIndex - 1; // Skip `>`
+
+        const value = calcStringFromNodePoints(nodePoints, startIndex, endIndex);
+        const node: INode = api.shouldReservePosition
+          ? { type: SlackBroadcastType, position: api.calcPosition(token), value }
+          : { type: SlackBroadcastType, value };
+        return node;
+      }),
+  };
+};

src/utils/markdown_parser/tokenizers/slack_broadcast/tokenizer.ts

@@ -0,0 +1,32 @@
+import type {
+  IInlineTokenizer,
+  IMatchInlineHookCreator,
+  IParseInlineHookCreator,
+} from "@yozora/core-tokenizer";
+import { BaseInlineTokenizer, TokenizerPriority } from "@yozora/core-tokenizer";
+import { match } from "./match";
+import { parse } from "./parse";
+import {
+  SlackBroadcastType,
+  type IDelimiter,
+  type INode,
+  type IThis,
+  type IToken,
+  type ITokenizerProps,
+  type T,
+} from "./types";
+
+export class SlackBroadcastTokenizer
+  extends BaseInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
+  implements IInlineTokenizer<T, IDelimiter, IToken, INode, IThis>
+{
+  constructor(props: ITokenizerProps = {}) {
+    super({
+      name: SlackBroadcastType,
+      priority: props.priority || TokenizerPriority.ATOMIC,
+    });
+  }
+
+  public override readonly match: IMatchInlineHookCreator<T, IDelimiter, IToken, IThis> = match;
+  public override readonly parse: IParseInlineHookCreator<T, IToken, INode, IThis> = parse;
+}

src/utils/markdown_parser/tokenizers/slack_broadcast/types.ts

@@ -0,0 +1,23 @@
+import { Literal } from "@yozora/ast";
+import type {
+  IBaseInlineTokenizerProps,
+  IPartialInlineToken,
+  ITokenDelimiter,
+  ITokenizer,
+} from "@yozora/core-tokenizer";
+
+export const SlackBroadcastType = "slack_broadcast";
+export type T = typeof SlackBroadcastType;
+export type INode = Literal<typeof SlackBroadcastType>;
+
+export interface IToken extends IPartialInlineToken<T> {
+  thickness: number;
+}
+
+export interface IDelimiter extends ITokenDelimiter {
+  type: "full";
+  thickness: number;
+}
+
+export type IThis = ITokenizer;
+export type ITokenizerProps = Partial<IBaseInlineTokenizerProps>;