foi-toy: added web tool, with tokens, syntax highlighting, and gramma…

…r-validator
getify · Dec 30, 2022 · 9390b96 · 9390b96
1 parent b5bec1d
commit 9390b96
Show file tree

Hide file tree

Showing 8 changed files with 310 additions and 8 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,7 @@
 node_modules/
+foi-toy/web/foi-grammar.txt
+foi-toy/web/syntax-color.html
+foi-toy/web/js/grammar-checker.js
 foi-toy/web/js/tokenizer.js
 foi-toy/web/js/highlighter.js
 package-lock.json
diff --git a/foi-toy/scripts/build-web b/foi-toy/scripts/build-web
@@ -18,7 +18,8 @@ var depMap = {
 
 const ROOT_DIR = path.join(__dirname,"..");
 const SRC_DIR = path.join(ROOT_DIR,"src");
-const WEB_JS_DIR = path.join(ROOT_DIR,"web","js");
+const WEB_DIR = path.join(ROOT_DIR,"web");
+const WEB_JS_DIR = path.join(WEB_DIR,"js");
 
 
 main().catch(console.log);
@@ -31,14 +32,20 @@ async function main() {
 
 	var [
 		packageJSON,
+		grammarMD,
 		copyrightHeader,
 		tokenizerCode,
-		highlighterCode
+		highlighterCode,
+		tmplHTMLCode,
+		tmplCSSCode,
 	] = await Promise.all([
 		fsp.readFile(path.join(ROOT_DIR,"..","package.json"),"utf-8"),
+		fsp.readFile(path.join(ROOT_DIR,"..","Grammar.md"),"utf-8"),
 		fsp.readFile(path.join(SRC_DIR,"copyright-header.txt"),"utf-8"),
 		fsp.readFile(path.join(SRC_DIR,"tokenizer.js"),"utf-8"),
 		fsp.readFile(path.join(SRC_DIR,"highlighter.js"),"utf-8"),
+		fsp.readFile(path.join(SRC_DIR,"tmpl.html"),"utf-8"),
+		fsp.readFile(path.join(SRC_DIR,"tmpl.css"),"utf-8"),
 	]);
 
 	packageJSON = JSON.parse(packageJSON);
@@ -47,11 +54,16 @@ async function main() {
 	var version = packageJSON.version;
 	var year = (new Date()).getFullYear();
 
+	// extract grammar
+	var grammar = (grammarMD.match(/^```ebnf$\s([^]*?)\s^```$/m) || [null,""])[1];
+
+	// process copyright header
 	copyrightHeader = copyrightHeader
 		.replace(/`/g,"")
 		.replace(/#VERSION#/g,version)
 		.replace(/#YEAR#/g,year);
 
+	// transpile CJS to ESM
 	var { esm: { code: tokenizerESM, }, } = build(
 		config,
 		"src/tokenizer.js",
@@ -67,10 +79,19 @@ async function main() {
 	tokenizerESM = `${copyrightHeader.replace("#FILENAME#","tokenizer.js")}${tokenizerESM}`;
 	highlighterESM = `${copyrightHeader.replace("#FILENAME#","highlighter.js")}${highlighterESM}`;
 
+	var syntaxColorHTML = tmplHTMLCode.replace("<style></style>",`<style>\n${tmplCSSCode}</style>`);
+
 	await Promise.all([
-		fsp.writeFile(path.join(__dirname,"..","web","js","tokenizer.js"),tokenizerESM,"utf-8"),
-		fsp.writeFile(path.join(__dirname,"..","web","js","highlighter.js"),highlighterESM,"utf-8"),
+		fsp.writeFile(path.join(WEB_DIR,"foi-grammar.txt"),grammar,"utf-8"),
+		fsp.writeFile(path.join(WEB_DIR,"syntax-color.html"),syntaxColorHTML,"utf-8"),
+		fsp.writeFile(path.join(WEB_JS_DIR,"tokenizer.js"),tokenizerESM,"utf-8"),
+		fsp.writeFile(path.join(WEB_JS_DIR,"highlighter.js"),highlighterESM,"utf-8"),
 	]);
 
+	await fsp.copyFile(
+		path.join(SRC_DIR,"grammar-checker.js"),
+		path.join(WEB_JS_DIR,"grammar-checker.js")
+	);
+
 	console.log("Complete.");
 }
diff --git a/foi-toy/web/css/style.css b/foi-toy/web/css/style.css
@@ -0,0 +1,26 @@
+#input {
+	width: 90%;
+	height: 10em;
+}
+
+#token-list {
+	width: 75%;
+	min-height: 10em;
+	max-height: 20em;
+	overflow-y: scroll;
+}
+
+#syntax-color {
+	width: 90%;
+	min-height: 10em;
+	max-height: 30em;
+	border: none;
+}
+
+summary {
+	cursor: pointer;
+}
+
+.hidden {
+	display: none;
+}
diff --git a/foi-toy/web/index.html b/foi-toy/web/index.html
@@ -7,6 +7,28 @@
 </head>
 <body>
 <h1>Foi-Toy</h1>
-</body>
+
+<textarea id="input">
+def x: "Hello world!";
+log(x);
+</textarea>
+
+<pre id="check-syntax"></pre>
+
+<iframe id="syntax-color" class="hidden"></iframe>
+
+<details id="token-details" class="hidden">
+	<summary>Click to expand token list</summary>
+
+	<textarea id="token-list"></textarea>
+</details>
+
+<br><br>
+
+
+<!------------------------------------->
+
 <script type="module" src="js/app.js"></script>
+
+</body>
 </html>
diff --git a/foi-toy/web/js/app.js b/foi-toy/web/js/app.js
@@ -1,15 +1,116 @@
+import Scheduler from "./scheduler.js";
 import { tokenize, } from "./tokenizer.js";
 import { highlight, } from "./highlighter.js";
 
+var inputEl;
+var syntaxColorEl;
+var checkSyntaxEl;
+var tokenDetailsEl;
+var tokenListEl;
+var grammar;
+var syntaxColorTmpl;
+var validator;
+var updater = Scheduler(50,300);
+var checker = Scheduler(100,750);
 
 main().catch(console.log);
 
+
 // ****************************
 
 async function main() {
-	var tokens = await tokenize("def x: \"Hello world!\"; log(x);");
+	inputEl = document.getElementById("input");
+	syntaxColorEl = document.getElementById("syntax-color");
+	checkSyntaxEl = document.getElementById("check-syntax");
+	tokenDetailsEl = document.getElementById("token-details");
+	tokenListEl = document.getElementById("token-list");
+
+	inputEl.addEventListener("input",onInput);
+
+	validator = new Worker("/js/grammar-checker-manager.js");
+	validator.addEventListener("message",onWorkerMessage);
+
+	[
+		syntaxColorTmpl,
+		grammar
+	] = await Promise.all([
+		fetch("/syntax-color.html",{
+			method: "GET",
+			cache: "no-store",
+			headers: {
+				"pragma": "no-cache",
+			},
+		}).then(res => res.text()),
+
+		fetch("/foi-grammar.txt",{
+			method: "GET",
+			cache: "no-store",
+			headers: {
+				"pragma": "no-cache",
+			},
+		}).then(res => res.text()),
+	]);
+
+	checkSyntaxEl.innerHTML = "Validating...";
+	validator.postMessage({ grammar, input: inputEl.value });
+
+	render();
+}
+
+function onInput(evt) {
+	checkSyntaxEl.innerHTML = "Validating...";
+	updater(render);
+	checker(checkInput);
+}
+
+async function render() {
+	var tokens = await tokenize(inputEl.value);
+	var tokensArr = [];
+	var tokensText = "";
+	for await (let token of tokens) {
+		tokensArr.push(token);
+		let attrs = Object.entries(token).map(([prop,value]) => `${prop}: ${JSON.stringify(value)}`);
+		tokensText += `{ ${attrs.join(", ")} }\n`;
+	}
+	tokenListEl.value = tokensText;
+	tokenDetailsEl.classList.remove("hidden");
 
-	for await (let htmlChunk of highlight(tokens)) {
-		console.log(htmlChunk);
+	await renderSyntaxColor(tokensArr);
+}
+
+async function renderSyntaxColor(tokens) {
+	syntaxColorEl.classList.remove("hidden");
+
+	var html = "";
+	if (tokens && tokens.length > 0) {
+		for await (let htmlChunk of highlight(tokens)) {
+			html += htmlChunk;
+		}
+	}
+
+	html = syntaxColorTmpl.replace("<pre></pre>",`<pre>${html}</pre>`);
+
+	syntaxColorEl.contentWindow.document.open();
+	syntaxColorEl.contentWindow.document.write(html);
+	syntaxColorEl.contentWindow.document.close();
+}
+
+function checkInput() {
+	validator.postMessage({ input: inputEl.value });
+}
+
+function onWorkerMessage({ data }) {
+	if (data.valid) {
+		checkSyntaxEl.innerHTML = "Valid!"
+	}
+	else if (data.invalid) {
+		checkSyntaxEl.innerHTML = data.invalid;
+		renderSyntaxColor();
+		syntaxColorEl.classList.add("hidden");
+		tokenListEl.innerHTML = "";
+		tokenDetailsEl.classList.add("hidden");
+	}
+	else {
+		console.log(data);
 	}
 }
diff --git a/foi-toy/web/js/grammar-checker-manager.js b/foi-toy/web/js/grammar-checker-manager.js
@@ -0,0 +1,56 @@
+"use strict";
+
+var grammar;
+var workerQueue = [];
+const QUEUE_LENGTH = 3;
+
+self.addEventListener("message",onParentMessage);
+
+
+// ****************************
+
+function addWorker() {
+	var worker = new Worker("/js/grammar-checker-worker.js");
+	worker.addEventListener("message",onWorkerMessage);
+	// initialize the worker with the grammar
+	if (grammar != null) {
+		worker.postMessage({ grammar, });
+	}
+	workerQueue.push({ pending: false, worker, });
+}
+
+function onParentMessage({ data }) {
+	if (data.grammar) {
+		if (grammar == null) {
+			grammar = data.grammar;
+		}
+		delete data.grammar;
+	}
+
+	if (
+		workerQueue.length > 0 &&
+		workerQueue[0].pending &&
+		(data.stop || data.input)
+	) {
+		let curWorker = workerQueue.shift();
+		addWorker();
+		curWorker.worker.removeEventListener("message",onWorkerMessage);
+		curWorker.worker.terminate();
+		curWorker.pending = false;
+	}
+
+	if (data.input) {
+		// prime the worker queue (if needed)
+		while (workerQueue.length < QUEUE_LENGTH) {
+			addWorker();
+		}
+
+		workerQueue[0].pending = true;
+		workerQueue[0].worker.postMessage(data);
+	}
+}
+
+function onWorkerMessage({ data }) {
+	workerQueue[0].pending = false;
+	self.postMessage(data);
+}
diff --git a/foi-toy/web/js/grammar-checker-worker.js b/foi-toy/web/js/grammar-checker-worker.js
@@ -0,0 +1,8 @@
+"use strict";
+
+importScripts(
+	"grammar-checker.js",
+	"https://rawgit.com/mdkrajnak/ebnftest/gh-pages/js/ebnftest.js"
+);
+
+initChecker(self);
diff --git a/foi-toy/web/js/scheduler.js b/foi-toy/web/js/scheduler.js
@@ -0,0 +1,65 @@
+export default Scheduler;
+export { Scheduler };
+
+
+// ***********************
+
+function Scheduler(debounceMin,throttleMax) {
+	var entries = new WeakMap();
+
+	return schedule;
+
+
+	// ***********************
+
+	function schedule(fn) {
+		var entry;
+
+		if (entries.has(fn)) {
+			entry = entries.get(fn);
+		}
+		else {
+			entry = {
+				last: 0,
+				timer: null,
+			};
+			entries.set(fn,entry);
+		}
+
+		var now = Date.now();
+
+		if (!entry.timer) {
+			entry.last = now;
+		}
+
+		if (
+			// no timer running yet?
+			entry.timer == null ||
+			// room left to debounce while still under the throttle-max?
+			(now - entry.last) < throttleMax
+		) {
+			if (entry.timer) {
+				clearTimeout(entry.timer);
+			}
+
+			let time = Math.min(debounceMin,Math.max(0,(entry.last + throttleMax) - now));
+			entry.timer = setTimeout(run,time,fn,entry);
+		}
+
+		if (!entry.cancelFn) {
+			entry.cancelFn = function cancel(){
+				if (entry.timer) {
+					clearTimeout(entry.timer);
+					entry.timer = entry.cancelFn = null;
+				}
+			};
+		}
+		return entry.cancelFn;
+	}
+
+	function run(fn,entry) {
+		entry.timer = entry.cancelFn = null;
+		entry.last = Date.now();
+		fn();
+	}
+}