omedusyo/syntax-lab
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

UI and a Lisp experiment

Browse source
This commit is contained in:
Yura Dupyn 2026-04-25 01:10:49 +02:00
parent 38ff06ea45
commit f55b437037
24 changed files with 2746 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

2
index.html
View file

@ -8,6 +8,6 @@
</head> </head>
<body> <body>
<div id="app"></div> <div id="app"></div>
<script type="module" src="/src/main.ts"></script> <script type="module" src="/src/main.tsx"></script>
</body> </body>
</html> </html>

2
libs/source-region

@ -1 +1 @@
Subproject commit 000949f3c30c73e645ae91b64f500dd1f02a65d8 Subproject commit 8471c60967eca6178d25fa3221035286a19856f2

1331
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

6
package.json
View file

@ -6,10 +6,14 @@
"scripts": { "scripts": {
"dev": "vite", "dev": "vite",
"build": "tsc && vite build", "build": "tsc && vite build",
"parser:experiments": "tsx src/parser.experiments.ts",
"preview": "vite preview" "preview": "vite preview"
}, },
"devDependencies": { "devDependencies": {
"solid-js": "^1.9.12",
"tsx": "^4.21.0",
"typescript": "~6.0.2", "typescript": "~6.0.2",
"vite": "^8.0.9" "vite": "^8.0.9",
"vite-plugin-solid": "^2.11.12"
} }
} }

81
src/main.ts
View file

@ -1,77 +1,4 @@
import './style.css' export { parseDocument } from './parser';
import { SourceText, SourceRegion, sourceText } from 'source-region'; export type { FoundSyntax, ParseDocumentResult, ParseError } from './parser';
import type { SourceLocation, Span } from 'source-region'; export { ConcreteSyntax, Expr } from './syntax';
export type { ConcreteSyntax as ConcreteSyntaxNode, Expr as ExprNode } from './syntax';
type Expr =
| { tag: "literal", value: Literal }
| { tag: "list", values: Expr[] }
namespace Expr {
export function number(value: number): Expr {
return { tag: "literal", value: { tag: "number", value } };
}
export function identifier(value: Identifier): Expr {
return { tag: "literal", value: { tag: "identifier", value } };
}
export function list(values: Expr[]): Expr {
return { tag: "list", values };
}
export function show(e: Expr): string {
switch (e.tag) {
case "literal":
return showLiteral(e.value);
case "list":
return `(${e.values.map(show).join(" ")})`;
}
}
function showLiteral(e: Literal): string {
switch (e.tag) {
case "number":
return `${e.value}`;
case "identifier":
return `${e.value}`;
}
}
}
type Literal =
| { tag: "number", value: number }
| { tag: "identifier", value: Identifier }
type Identifier = string
// === Examples ===
function example00() {
const v: Expr = Expr.list([Expr.identifier("f"), Expr.number(123), Expr.number(512)]);
console.log(v);
console.log(Expr.show(v));
}
function example01() {
const str = `hello, world!
foo
bar `;
const source = sourceText(str);
const region = source.fullRegion();
console.log(region);
console.log(region.lineCount);
region.forEachLine((span, lineNo) => {
console.log(lineNo, region.stringOf(span));
});
}
[
example00,
example01,
].forEach((f, i) => {
console.log(`====${i}===`);
f();
})

5
src/main.tsx Normal file
View file

@ -0,0 +1,5 @@
import { render } from 'solid-js/web';
import './style.css';
import { App } from './ui/App';
render(() => <App />, document.getElementById('app') as HTMLElement);

52
src/parser.experiments.ts Normal file
View file

@ -0,0 +1,52 @@
import { sourceText } from 'source-region';
import { parseDocument } from './parser';
import { Expr } from './syntax';
// === Experiments ===
function experiment00_emptyDocument(): void {
logParse("empty document", "");
}
function experiment01_topLevelExpressions(): void {
logParse("top-level expressions", "foo 123 (bar baz_1 qux-2)");
}
function experiment02_nestedLists(): void {
logParse("nested lists", "(define square (_ x) (* x x))");
}
function experiment03_unclosedList(): void {
logParse("unclosed list", "(foo 123\n (bar 456)");
}
function experiment04_recoverAtDocumentLevel(): void {
logParse("document recovery", "foo ) @@@ (bar 1) 99");
}
function experiment05_recoverInsideList(): void {
logParse("list recovery", "(foo @@@ 1 (bar # 2) baz)");
}
function experiment06_unicodeSpans(): void {
logParse("unicode spans", "alpha 💥 (beta 2)");
}
function logParse(name: string, input: string): void {
const region = sourceText(input).fullRegion();
const result = parseDocument(region);
console.log(`==== parser:${name} ====`);
console.log(input);
console.log(result.values.map(Expr.show));
console.dir(result.errors, { depth: null });
}
[
experiment00_emptyDocument,
experiment01_topLevelExpressions,
experiment02_nestedLists,
experiment03_unclosedList,
experiment04_recoverAtDocumentLevel,
experiment05_recoverInsideList,
experiment06_unicodeSpans,
].forEach((experiment) => experiment());

316
src/parser.ts Normal file
View file

@ -0,0 +1,316 @@
import {
CARRIAGE_RETURN,
NEW_LINE,
SPACE,
TAB,
char,
isDigit,
} from 'source-region';
import type {
CodePoint,
CodePointIndex,
CodePointSpan,
SourceRegion,
} from 'source-region';
import { ConcreteSyntax } from './syntax';
// Whitespace convention:
// - parseDocument consumes leading whitespace before each top-level expression.
// - parseExpr assumes leading whitespace has already been consumed.
// - Successful expression parsers stop immediately after the expression.
// - parseList owns whitespace between list elements and before the closing paren.
//
// Recovery policy:
// - At document level, invalid input is skipped until EOF or a plausible expression
// start. Unexpected ")" is reported and consumed immediately.
// - Inside lists, invalid input is skipped until EOF, ")", or a plausible
// expression start. Recovery always consumes at least one code point when it
// cannot stop at a synchronization point.
//
// Span convention:
// - Parser internals and diagnostics use CodePointSpan.
// - Rendering can convert these later with SourceText.getSpan.
const OPEN_PAREN = char('(');
const CLOSE_PAREN = char(')');
const DASH = char('-');
const UNDERSCORE = char('_');
const LOWERCASE_A = char('a');
const LOWERCASE_Z = char('z');
const UPPERCASE_A = char('A');
const UPPERCASE_Z = char('Z');
export type ParseDocumentResult = {
values: ConcreteSyntax[];
errors: ParseError[];
};
export type ParseError =
| {
tag: "expected-expression";
span: CodePointSpan;
found: FoundSyntax;
}
| {
tag: "expected-close-paren";
span: CodePointSpan;
openParen: CodePointSpan;
found: FoundSyntax;
}
| {
tag: "unexpected-close-paren";
span: CodePointSpan;
}
| {
tag: "unexpected-code-point";
span: CodePointSpan;
found: FoundSyntax;
}
| {
tag: "invalid-number";
span: CodePointSpan;
text: string;
reason: "unsafe-integer";
};
export type FoundSyntax =
| { tag: "code-point"; value: CodePoint; span: CodePointSpan }
| { tag: "eof"; span: CodePointSpan };
export function parseDocument(region: SourceRegion): ParseDocumentResult {
return new Parser(region).parseDocument();
}
class Parser {
private index: CodePointIndex;
private readonly errors: ParseError[] = [];
constructor(private readonly region: SourceRegion) {
this.index = region.span.start.index;
}
parseDocument(): ParseDocumentResult {
const values: ConcreteSyntax[] = [];
while (true) {
this.skipWhitespace();
if (this.isAtEnd()) break;
const before = this.index;
const value = this.parseExpr();
if (value) {
values.push(value);
continue;
}
this.recoverDocument(before);
}
return { values, errors: this.errors };
}
private parseExpr(): ConcreteSyntax | undefined {
const cp = this.peek();
if (cp === undefined) {
this.errors.push({
tag: "expected-expression",
span: this.eofSpan(),
found: this.found(),
});
return undefined;
}
if (cp === CLOSE_PAREN) {
this.errors.push({
tag: "unexpected-close-paren",
span: this.currentSpan(),
});
return undefined;
}
if (cp === OPEN_PAREN) return this.parseList();
if (isDigit(cp)) return this.parseNumber();
if (isIdentifierStart(cp)) return this.parseIdentifier();
this.errors.push({
tag: "expected-expression",
span: this.currentSpan(),
found: this.found(),
});
return undefined;
}
private parseList(): ConcreteSyntax | undefined {
const start = this.index;
const openParen = this.currentSpan();
this.advance();
const values: ConcreteSyntax[] = [];
// === Body Parsing ===
while (true) {
this.skipWhitespace();
const cp = this.peek();
if (cp === CLOSE_PAREN) {
this.advance();
return ConcreteSyntax.list(values, this.spanFrom(start));
}
if (cp === undefined) {
this.errors.push({
tag: "expected-close-paren",
span: this.eofSpan(),
openParen,
found: this.found(),
});
return ConcreteSyntax.list(values, this.spanFrom(start));
}
const before = this.index;
const value = this.parseExpr();
if (value) {
values.push(value);
continue;
}
this.recoverList(before);
}
}
private parseNumber(): ConcreteSyntax {
const start = this.index;
while (isDigit(this.peekOrInvalid())) {
this.advance();
}
const span = this.spanFrom(start);
const text = this.slice(span);
const value = Number(text);
if (!Number.isSafeInteger(value)) {
this.errors.push({
tag: "invalid-number",
span,
text,
reason: "unsafe-integer",
});
}
return ConcreteSyntax.number(value, span);
}
private parseIdentifier(): ConcreteSyntax {
const start = this.index;
this.advance();
while (isIdentifierPart(this.peekOrInvalid())) {
this.advance();
}
const span = this.spanFrom(start);
return ConcreteSyntax.identifier(this.slice(span), span);
}
private recoverDocument(failedAt: CodePointIndex): void {
if (this.index === failedAt) this.advance();
while (!this.isAtEnd()) {
const cp = this.peek();
if (cp === CLOSE_PAREN) {
this.errors.push({
tag: "unexpected-close-paren",
span: this.currentSpan(),
});
this.advance();
return;
}
if (isExpressionStart(cp)) return;
this.advance();
}
}
private recoverList(failedAt: CodePointIndex): void {
if (this.index === failedAt) this.advance();
while (!this.isAtEnd()) {
const cp = this.peek();
if (cp === CLOSE_PAREN || isExpressionStart(cp)) return;
this.advance();
}
}
private skipWhitespace(): void {
while (isWhitespace(this.peekOrInvalid())) {
this.advance();
}
}
private peek(): CodePoint | undefined {
if (this.index >= this.region.span.end.index) return undefined;
return this.region.codePointAt(this.index);
}
private peekOrInvalid(): CodePoint {
return this.peek() ?? -1;
}
private advance(): CodePoint | undefined {
const cp = this.peek();
if (cp === undefined) return undefined;
this.index += 1;
return cp;
}
private isAtEnd(): boolean {
return this.index >= this.region.span.end.index;
}
private spanFrom(start: CodePointIndex): CodePointSpan {
return { start, end: this.index };
}
private currentSpan(): CodePointSpan {
const start = this.index;
const end = this.isAtEnd() ? start : start + 1;
return { start, end };
}
private eofSpan(): CodePointSpan {
return { start: this.region.span.end.index, end: this.region.span.end.index };
}
private found(): FoundSyntax {
const cp = this.peek();
if (cp === undefined) return { tag: "eof", span: this.eofSpan() };
return { tag: "code-point", value: cp, span: this.currentSpan() };
}
private slice(span: CodePointSpan): string {
return this.region.source.sliceByCp(span.start, span.end);
}
}
function isWhitespace(cp: CodePoint): boolean {
return cp === SPACE || cp === TAB || cp === NEW_LINE || cp === CARRIAGE_RETURN;
}
function isExpressionStart(cp: CodePoint | undefined): boolean {
return cp !== undefined && (cp === OPEN_PAREN || isDigit(cp) || isIdentifierStart(cp));
}
function isIdentifierStart(cp: CodePoint): boolean {
return isAsciiLetter(cp) || cp === DASH || cp === UNDERSCORE;
}
function isIdentifierPart(cp: CodePoint): boolean {
return isIdentifierStart(cp) || isDigit(cp);
}
function isAsciiLetter(cp: CodePoint): boolean {
return (LOWERCASE_A <= cp && cp <= LOWERCASE_Z)
|| (UPPERCASE_A <= cp && cp <= UPPERCASE_Z);
}

4
src/style.css
View file

@ -0,0 +1,4 @@
@import "/src/styles/base.css";
@import "/src/styles/layout.css";
@import "/src/styles/syntax-pane.css";
@import "/src/styles/source-grid.css";

51
src/styles/base.css Normal file
View file

@ -0,0 +1,51 @@
:root {
color-scheme: dark;
--hue: 248;
--bg: oklch(14% 0.018 var(--hue));
--panel: oklch(18% 0.016 var(--hue));
--panel-raised: oklch(22% 0.018 var(--hue));
--border: oklch(33% 0.02 var(--hue));
--border-strong: oklch(46% 0.035 var(--hue));
--text: oklch(91% 0.012 var(--hue));
--text-muted: oklch(68% 0.018 var(--hue));
--text-faint: oklch(52% 0.018 var(--hue));
--accent: oklch(76% 0.08 var(--hue));
--accent-bg: oklch(30% 0.04 var(--hue));
--error: oklch(73% 0.14 28);
--error-bg: oklch(25% 0.045 28);
--source-highlight: oklch(34% 0.06 var(--hue));
--font-ui: Inter, ui-sans-serif, system-ui, sans-serif;
--font-mono: "SFMono-Regular", Consolas, "Liberation Mono", monospace;
--font-codepoint: "SFMono-Regular", Consolas, "Liberation Mono", "Noto Color Emoji", "Apple Color Emoji", "Segoe UI Emoji", monospace;
--text-xs: 0.75rem;
--text-sm: 0.875rem;
--text-md: 1rem;
--text-lg: 1.125rem;
--gap-1: 0.25rem;
--gap-2: 0.5rem;
--gap-3: 0.75rem;
--gap-4: 1rem;
--gap-5: 1.5rem;
}
* {
box-sizing: border-box;
}
body {
margin: 0;
min-width: 1200px;
min-height: 100vh;
color: var(--text);
background: var(--bg);
font-family: var(--font-ui);
font-size: var(--text-md);
}
button,
textarea {
font: inherit;
}

99
src/styles/layout.css Normal file
View file

@ -0,0 +1,99 @@
.app-shell {
display: grid;
grid-template-columns: var(--left-width) 0.45rem var(--middle-width) 0.45rem minmax(360px, 1fr);
gap: var(--gap-2);
height: 100vh;
padding: var(--gap-4);
}
.pane {
min-width: 0;
min-height: 0;
display: flex;
flex-direction: column;
border: 1px solid var(--border);
background: var(--panel);
}
.pane-splitter {
position: relative;
min-width: 0.45rem;
cursor: col-resize;
touch-action: none;
}
.pane-splitter::before {
content: "";
position: absolute;
top: 0;
bottom: 0;
left: calc(50% - 1px);
width: 2px;
background: var(--border);
}
.pane-splitter:hover::before,
.pane-splitter:active::before {
background: var(--accent);
}
.is-resizing-pane {
cursor: col-resize;
user-select: none;
}
.pane-header {
display: flex;
align-items: baseline;
justify-content: space-between;
gap: var(--gap-3);
padding: var(--gap-3) var(--gap-4);
border-bottom: 1px solid var(--border);
}
.pane-header h1 {
margin: 0;
color: var(--text);
font-size: var(--text-lg);
font-weight: 650;
letter-spacing: 0;
}
.pane-detail,
.item-meta,
.section-label {
color: var(--text-muted);
font-size: var(--text-xs);
}
.source-input {
flex: 1;
width: 100%;
min-height: 0;
resize: none;
border: 0;
outline: 0;
padding: var(--gap-4);
color: var(--text);
background: var(--panel);
font-family: var(--font-mono);
font-size: var(--text-sm);
line-height: 1.6;
}
.scroll-stack,
.expr-list,
.source-grid-shell {
min-height: 0;
overflow: auto;
}
.scroll-stack,
.expr-list {
padding: var(--gap-3);
}
.empty-state {
padding: var(--gap-4);
color: var(--text-muted);
}

118
src/styles/source-grid.css Normal file
View file

@ -0,0 +1,118 @@
.source-grid-shell {
--cell-width: 0.78rem;
--cell-height: 1.42rem;
--gutter-size: 3.25rem;
padding: var(--gap-3);
font-family: var(--font-mono);
font-size: var(--text-sm);
}
.grid-status {
position: sticky;
top: 0;
z-index: 4;
display: flex;
justify-content: space-between;
gap: var(--gap-4);
min-width: max-content;
padding: 0 0 var(--gap-3);
color: var(--text-muted);
background: var(--panel);
font-family: var(--font-ui);
font-size: var(--text-xs);
}
.source-grid {
width: max-content;
}
.source-grid-row {
display: grid;
grid-template-columns: var(--gutter-size) repeat(var(--max-column), var(--cell-width));
min-width: max-content;
}
.row-header,
.grid-cell {
width: var(--cell-width);
height: var(--cell-height);
display: inline-flex;
align-items: center;
justify-content: center;
line-height: var(--cell-height);
user-select: none;
}
.row-header {
position: sticky;
left: 0;
z-index: 2;
width: var(--gutter-size);
justify-content: flex-end;
padding-right: var(--gap-2);
color: var(--text-faint);
background: var(--panel);
font-size: var(--text-xs);
}
.grid-cell {
position: relative;
color: var(--text);
border: 1px solid transparent;
font-family: var(--font-codepoint);
font-size: var(--text-sm);
overflow: visible;
}
.grid-cell-space,
.grid-cell-tab,
.grid-cell-newline,
.grid-cell-carriage-return,
.grid-cell-combining-mark,
.grid-cell-format,
.grid-cell-control {
color: var(--text-faint);
}
.grid-cell-format,
.grid-cell-control {
font-family: var(--font-ui);
font-size: 0.55rem;
}
.grid-cell-combining-mark {
font-size: 0.7rem;
}
.grid-cell.is-annotated {
color: var(--text);
background: var(--source-highlight);
}
.grid-cell.is-hover-row,
.grid-cell.is-hover-column {
background: oklch(25% 0.026 var(--hue));
}
.grid-cell.is-annotated.is-hover-row,
.grid-cell.is-annotated.is-hover-column {
background: oklch(39% 0.064 var(--hue));
}
.grid-cell.is-hover-cell {
z-index: 1;
color: var(--bg);
border-color: var(--accent);
background: var(--accent);
}
.zero-span-marker {
z-index: 2;
align-self: stretch;
justify-self: start;
width: 2px;
margin-left: -1px;
background: var(--accent);
pointer-events: none;
}

66
src/styles/syntax-pane.css Normal file
View file

@ -0,0 +1,66 @@
.section-label {
margin: var(--gap-2) 0;
text-transform: uppercase;
}
.error-list,
.expr-list,
.list-children {
display: flex;
flex-direction: column;
gap: var(--gap-2);
}
.error-card,
.expr-node {
border: 1px solid var(--border);
background: var(--panel-raised);
}
.error-card {
padding: var(--gap-3);
border-color: oklch(42% 0.05 28);
background: var(--error-bg);
}
.expr-node {
padding: var(--gap-2);
}
.error-card:hover,
.expr-node:hover {
border-color: var(--border-strong);
background: var(--accent-bg);
}
.item-title,
.node-kind {
color: var(--text);
font-size: var(--text-sm);
font-weight: 650;
}
.node-kind {
display: inline-flex;
min-width: 4.5rem;
color: var(--accent);
}
.node-value {
color: var(--text);
font-family: var(--font-mono);
font-size: var(--text-sm);
}
.list-node-header {
display: flex;
align-items: baseline;
justify-content: space-between;
gap: var(--gap-2);
}
.list-children {
margin-top: var(--gap-2);
padding-left: var(--gap-3);
border-left: 1px solid var(--border);
}

59
src/syntax.ts Normal file
View file

@ -0,0 +1,59 @@
import type { CodePointSpan } from 'source-region';
export type ConcreteSyntax = Expr<{ span: CodePointSpan }>
export type Expr<A> =
| { tag: "literal", value: Literal } & A
| { tag: "list", values: Expr<A>[] } & A
export namespace ConcreteSyntax {
export function number(value: number, span: CodePointSpan): ConcreteSyntax {
return { tag: "literal", value: { tag: "number", value }, span };
}
export function identifier(value: Identifier, span: CodePointSpan): ConcreteSyntax {
return { tag: "literal", value: { tag: "identifier", value }, span };
}
export function list(values: ConcreteSyntax[], span: CodePointSpan): ConcreteSyntax {
return { tag: "list", values, span };
}
}
export namespace Expr {
export function number(value: number): Expr<void> {
return { tag: "literal", value: { tag: "number", value } };
}
export function identifier(value: Identifier): Expr<void> {
return { tag: "literal", value: { tag: "identifier", value } };
}
export function list(values: Expr<void>[]): Expr<void> {
return { tag: "list", values };
}
export function show<A>(e: Expr<A>): string {
switch (e.tag) {
case "literal":
return showLiteral(e.value);
case "list":
return `(${e.values.map(show).join(" ")})`;
}
}
function showLiteral(e: Literal): string {
switch (e.tag) {
case "number":
return `${e.value}`;
case "identifier":
return `${e.value}`;
}
}
}
type Literal =
| { tag: "number", value: number }
| { tag: "identifier", value: Identifier }
type Identifier = string

127
src/ui/App.tsx Normal file
View file

@ -0,0 +1,127 @@
import { createMemo, createSignal, Show } from 'solid-js';
import { sourceText } from 'source-region';
import type { CodePointSpan, SourceRegion, SourceText } from 'source-region';
import { parseDocument } from '../parser';
import type { ParseError } from '../parser';
import type { ConcreteSyntax } from '../syntax';
import { spanLabel } from './format';
import { PaneHeader, PaneSplitter } from './Pane';
import { SourceGrid } from './SourceGrid';
import type { SourceGridAnnotation } from './SourceGrid';
import { ErrorList, ExpressionList } from './SyntaxPane';
import type { HoverTarget } from './types';
type ParsedDocument = {
source: SourceText;
region: SourceRegion;
values: ConcreteSyntax[];
errors: ParseError[];
};
const SAMPLE_INPUT = `(define square (_ x) (mul x x))
(add 1 2)
(define pyth (_ x y) (+ (square x) (square y)))
foo ) @@@ (bar 1)
(nested (list 123 abc_9 name-with-dash))`;
export function App() {
const [input, setInput] = createSignal(SAMPLE_INPUT);
const [hovered, setHovered] = createSignal<HoverTarget | undefined>();
const [leftWidth, setLeftWidth] = createSignal(420);
const [middleWidth, setMiddleWidth] = createSignal(420);
const parsed = createMemo<ParsedDocument>(() => {
const source = sourceText(input());
const region = source.fullRegion();
const result = parseDocument(region);
return { source, region, values: result.values, errors: result.errors };
});
return (
<main
class="app-shell"
style={{
"--left-width": `${leftWidth()}px`,
"--middle-width": `${middleWidth()}px`,
}}
>
<section class="pane input-pane">
<PaneHeader title="Source" detail={`${input().length} UTF-16 units`} />
<textarea
class="source-input"
spellcheck={false}
value={input()}
onInput={(event) => {
setInput(event.currentTarget.value);
setHovered(undefined);
}}
/>
</section>
<PaneSplitter
label="Resize source and structure panes"
onDrag={(delta) => {
setLeftWidth((width) => clamp(width + delta, 280, 760));
}}
/>
<section class="pane structure-pane">
<PaneHeader
title="Structure"
detail={`${parsed().values.length} expressions, ${parsed().errors.length} errors`}
/>
<Show
when={parsed().errors.length > 0}
fallback={
<ExpressionList
values={parsed().values}
onHover={setHovered}
/>
}
>
<ErrorList
errors={parsed().errors}
values={parsed().values}
onHover={setHovered}
/>
</Show>
</section>
<PaneSplitter
label="Resize structure and source grid panes"
onDrag={(delta) => {
setMiddleWidth((width) => clamp(width + delta, 260, 760));
}}
/>
<section class="pane source-pane">
<PaneHeader
title="Source Grid"
detail={hovered() ? spanLabel(hovered()!.span) : "nothing hovered"}
/>
<SourceGrid
source={parsed().source}
region={parsed().region}
annotations={hovered() ? [hoverAnnotation(hovered()!)] : []}
/>
</section>
</main>
);
}
function hoverAnnotation(target: HoverTarget): SourceGridAnnotation {
return {
id: "hovered",
span: target.span,
label: target.label,
cellClass: "annotation-hovered",
markerClass: "annotation-hovered-marker",
};
}
function clamp(value: number, min: number, max: number): number {
return Math.max(min, Math.min(max, value));
}

50
src/ui/Pane.tsx Normal file
View file

@ -0,0 +1,50 @@
export function PaneHeader(props: { title: string; detail: string }) {
return (
<header class="pane-header">
<h1>{props.title}</h1>
<div class="pane-detail">{props.detail}</div>
</header>
);
}
export function PaneSplitter(props: {
label: string;
onDrag: (deltaX: number) => void;
}) {
let startX = 0;
function onPointerDown(event: PointerEvent) {
startX = event.clientX;
const target = event.currentTarget as HTMLElement;
target.setPointerCapture(event.pointerId);
document.body.classList.add("is-resizing-pane");
}
function onPointerMove(event: PointerEvent) {
if (!(event.currentTarget as HTMLElement).hasPointerCapture(event.pointerId)) return;
const delta = event.clientX - startX;
startX = event.clientX;
props.onDrag(delta);
}
function onPointerUp(event: PointerEvent) {
const target = event.currentTarget as HTMLElement;
if (target.hasPointerCapture(event.pointerId)) {
target.releasePointerCapture(event.pointerId);
}
document.body.classList.remove("is-resizing-pane");
}
return (
<div
class="pane-splitter"
role="separator"
aria-label={props.label}
aria-orientation="vertical"
onPointerDown={onPointerDown}
onPointerMove={onPointerMove}
onPointerUp={onPointerUp}
onPointerCancel={onPointerUp}
/>
);
}

261
src/ui/SourceGrid.tsx Normal file
View file

@ -0,0 +1,261 @@
import { createMemo, createSignal, For, Show } from 'solid-js';
import type { JSX } from 'solid-js';
import {
CARRIAGE_RETURN,
NEW_LINE,
SPACE,
TAB,
} from 'source-region';
import type {
CodePoint,
CodePointIndex,
CodePointSpan,
SourceRegion,
SourceText,
} from 'source-region';
export type SourceGridAnnotation = {
id: string;
span: CodePointSpan;
label?: string;
cellClass?: string;
cellStyle?: JSX.CSSProperties;
markerClass?: string;
markerStyle?: JSX.CSSProperties;
};
type SourceGridModel = {
rows: SourceGridRow[];
maxColumn: number;
};
type SourceGridRow = {
lineNo: number;
cells: SourceGridCell[];
};
type SourceGridCell = {
index: CodePointIndex;
line: number;
column: number;
codePoint: CodePoint;
display: string;
kind: "normal" | "space" | "tab" | "newline" | "carriage-return" | "combining-mark" | "format" | "control";
};
type ZeroWidthMarker = {
line: number;
column: number;
annotation: SourceGridAnnotation;
};
export function SourceGrid(props: {
source: SourceText;
region: SourceRegion;
annotations: SourceGridAnnotation[];
}) {
const [hoveredCell, setHoveredCell] = createSignal<SourceGridCell | undefined>();
const grid = createMemo(() => makeSourceGrid(props.source, props.region));
const zeroMarkers = createMemo(() => zeroWidthMarkers(props.source, props.annotations));
const maxColumn = createMemo(() => {
const markerMax = zeroMarkers().reduce((max, marker) => Math.max(max, marker.column), 0);
return Math.max(1, grid().maxColumn, markerMax);
});
return (
<div class="source-grid-shell">
<div class="grid-status">
<span>{props.annotations[0]?.label ?? "Hover an error or expression."}</span>
<Show when={hoveredCell()}>
{(cell) => (
<span>
index {cell().index}, line {cell().line}, col {cell().column}, U+{cell().codePoint.toString(16).toUpperCase()}
</span>
)}
</Show>
</div>
<div
class="source-grid"
style={{ "--max-column": `${maxColumn()}` }}
>
<For each={grid().rows}>
{(row) => (
<div class="source-grid-row">
<div class="row-header">{row.lineNo}</div>
<For each={zeroMarkers().filter((marker) => marker.line === row.lineNo)}>
{(marker) => (
<div
class={markerClass(marker.annotation)}
style={{
"grid-column": `${marker.column + 1}`,
...marker.annotation.markerStyle,
}}
title={marker.annotation.label}
/>
)}
</For>
<For each={row.cells}>
{(cell) => (
<div
class={cellClass(cell, hoveredCell(), props.annotations)}
style={cellStyle(cell, props.annotations)}
title={cellTitle(cell, props.annotations)}
onMouseEnter={() => setHoveredCell(cell)}
onMouseLeave={() => setHoveredCell(undefined)}
>
{cell.display}
</div>
)}
</For>
</div>
)}
</For>
</div>
</div>
);
}
function makeSourceGrid(source: SourceText, region: SourceRegion): SourceGridModel {
const rows: SourceGridRow[] = [];
let maxColumn = 0;
for (let lineNo = region.span.start.line; lineNo <= region.span.end.line; lineNo++) {
const range = source.getLineRange(lineNo);
const start = Math.max(range.start, region.span.start.index);
const end = Math.min(range.end, region.span.end.index);
const cells: SourceGridCell[] = [];
for (let index = start; index < end; index++) {
const codePoint = source.codePointAt(index);
const location = source.getLocation(index);
const cell = {
index,
line: location.line,
column: location.column,
codePoint,
display: displayCodePoint(codePoint),
kind: codePointKind(codePoint),
};
cells.push(cell);
maxColumn = Math.max(maxColumn, location.column);
}
rows.push({ lineNo, cells });
}
return { rows, maxColumn };
}
function cellClass(cell: SourceGridCell, hovered: SourceGridCell | undefined, annotations: SourceGridAnnotation[]): string {
const classes = ["grid-cell", `grid-cell-${cell.kind}`];
for (const annotation of annotationsForCell(cell, annotations)) {
classes.push("is-annotated");
if (annotation.cellClass) classes.push(annotation.cellClass);
}
if (hovered) {
if (hovered.line === cell.line) classes.push("is-hover-row");
if (hovered.column === cell.column) classes.push("is-hover-column");
if (hovered.index === cell.index) classes.push("is-hover-cell");
}
return classes.join(" ");
}
function cellStyle(cell: SourceGridCell, annotations: SourceGridAnnotation[]): JSX.CSSProperties {
return annotationsForCell(cell, annotations).reduce<JSX.CSSProperties>(
(style, annotation) => ({ ...style, ...annotation.cellStyle }),
{},
);
}
function cellTitle(cell: SourceGridCell, annotations: SourceGridAnnotation[]): string {
const labels = annotationsForCell(cell, annotations)
.map((annotation) => annotation.label)
.filter((label) => label !== undefined);
const base = `index ${cell.index}, line ${cell.line}, column ${cell.column}`;
return labels.length > 0 ? `${base}\n${labels.join("\n")}` : base;
}
function annotationsForCell(cell: SourceGridCell, annotations: SourceGridAnnotation[]): SourceGridAnnotation[] {
return annotations.filter((annotation) =>
annotation.span.start < annotation.span.end
&& annotation.span.start <= cell.index
&& cell.index < annotation.span.end
);
}
function markerClass(annotation: SourceGridAnnotation): string {
return ["zero-span-marker", annotation.markerClass].filter(Boolean).join(" ");
}
function zeroWidthMarkers(source: SourceText, annotations: SourceGridAnnotation[]): ZeroWidthMarker[] {
return annotations
.filter((annotation) => annotation.span.start === annotation.span.end)
.map((annotation) => {
const location = source.getLocation(annotation.span.start);
return { line: location.line, column: location.column, annotation };
});
}
function displayCodePoint(cp: CodePoint): string {
switch (cp) {
case SPACE:
return "·";
case TAB:
return "⇥";
case NEW_LINE:
return "␊";
case CARRIAGE_RETURN:
return "␍";
default:
if (isCombiningMark(cp)) return `◌${String.fromCodePoint(cp)}`;
if (cp === 0x200C) return "ZWNJ";
if (cp === 0x200D) return "ZWJ";
if (isVariationSelector(cp)) return "VS";
if (isControlCodePoint(cp)) return codePointLabel(cp);
return String.fromCodePoint(cp);
}
}
function codePointKind(cp: CodePoint): SourceGridCell["kind"] {
switch (cp) {
case SPACE:
return "space";
case TAB:
return "tab";
case NEW_LINE:
return "newline";
case CARRIAGE_RETURN:
return "carriage-return";
default:
if (isCombiningMark(cp)) return "combining-mark";
if (cp === 0x200C || cp === 0x200D || isVariationSelector(cp)) return "format";
if (isControlCodePoint(cp)) return "control";
return "normal";
}
}
function isCombiningMark(cp: CodePoint): boolean {
return (0x0300 <= cp && cp <= 0x036F)
|| (0x1AB0 <= cp && cp <= 0x1AFF)
|| (0x1DC0 <= cp && cp <= 0x1DFF)
|| (0x20D0 <= cp && cp <= 0x20FF)
|| (0xFE20 <= cp && cp <= 0xFE2F);
}
function isVariationSelector(cp: CodePoint): boolean {
return (0xFE00 <= cp && cp <= 0xFE0F)
|| (0xE0100 <= cp && cp <= 0xE01EF);
}
function isControlCodePoint(cp: CodePoint): boolean {
return (0x0000 <= cp && cp <= 0x001F) || (0x007F <= cp && cp <= 0x009F);
}
function codePointLabel(cp: CodePoint): string {
return `U+${cp.toString(16).toUpperCase().padStart(4, "0")}`;
}

126
src/ui/SyntaxPane.tsx Normal file
View file

@ -0,0 +1,126 @@
import { For, Show } from 'solid-js';
import type { JSX } from 'solid-js';
import type { CodePointSpan } from 'source-region';
import type { ParseError } from '../parser';
import type { ConcreteSyntax } from '../syntax';
import { Expr } from '../syntax';
import { errorDetail, errorLabel, errorTitle } from './format';
import type { HoverTarget } from './types';
export function ErrorList(props: {
errors: ParseError[];
values: ConcreteSyntax[];
onHover: (target: HoverTarget | undefined) => void;
}) {
return (
<div class="scroll-stack">
<div class="section-label">Errors</div>
<div class="error-list">
<For each={props.errors}>
{(error) => (
<HoverBlock
class="error-card"
label={errorLabel(error)}
span={error.span}
onHover={props.onHover}
>
<div class="item-title">{errorTitle(error)}</div>
<div class="item-meta">{errorDetail(error)}</div>
</HoverBlock>
)}
</For>
</div>
<Show when={props.values.length > 0}>
<div class="section-label">Recovered Expressions</div>
<ExpressionList values={props.values} onHover={props.onHover} />
</Show>
</div>
);
}
export function ExpressionList(props: {
values: ConcreteSyntax[];
onHover: (target: HoverTarget | undefined) => void;
}) {
return (
<div class="expr-list">
<For each={props.values}>
{(value, index) => (
<ExprView
expr={value}
label={`expression ${index() + 1}`}
onHover={props.onHover}
/>
)}
</For>
</div>
);
}
function ExprView(props: {
expr: ConcreteSyntax;
label: string;
onHover: (target: HoverTarget | undefined) => void;
}) {
if (props.expr.tag === "literal") {
return (
<HoverBlock
class="expr-node literal-node"
label={`${props.label}: ${props.expr.value.tag}`}
span={props.expr.span}
onHover={props.onHover}
>
<span class="node-kind">{props.expr.value.tag}</span>
<span class="node-value">{literalValue(props.expr)}</span>
</HoverBlock>
);
}
return (
<HoverBlock
class="expr-node list-node"
label={`${props.label}: list`}
span={props.expr.span}
onHover={props.onHover}
>
<div class="list-node-header">
<span class="node-kind">list</span>
<span class="item-meta">{props.expr.values.length} children</span>
</div>
<div class="list-children">
<For each={props.expr.values}>
{(child, index) => (
<ExprView
expr={child}
label={`${props.label}.${index() + 1}`}
onHover={props.onHover}
/>
)}
</For>
</div>
</HoverBlock>
);
}
function HoverBlock(props: {
class: string;
label: string;
span: CodePointSpan;
onHover: (target: HoverTarget | undefined) => void;
children: JSX.Element;
}) {
return (
<div
class={props.class}
onMouseEnter={() => props.onHover({ label: props.label, span: props.span })}
onMouseLeave={() => props.onHover(undefined)}
>
{props.children}
</div>
);
}
function literalValue(expr: ConcreteSyntax): string {
return expr.tag === "literal" ? Expr.show(expr) : "";
}

49
src/ui/format.ts Normal file
View file

@ -0,0 +1,49 @@
import type { CodePointSpan } from 'source-region';
import type { FoundSyntax, ParseError } from '../parser';
export function errorTitle(error: ParseError): string {
switch (error.tag) {
case "expected-expression":
return "Expected expression";
case "expected-close-paren":
return "Expected closing paren";
case "unexpected-close-paren":
return "Unexpected closing paren";
case "unexpected-code-point":
return "Unexpected code point";
case "invalid-number":
return "Invalid number";
}
}
export function errorLabel(error: ParseError): string {
return `${errorTitle(error)} ${spanLabel(error.span)}`;
}
export function errorDetail(error: ParseError): string {
switch (error.tag) {
case "expected-expression":
return `found ${foundLabel(error.found)}`;
case "expected-close-paren":
return `opened at ${spanLabel(error.openParen)}, found ${foundLabel(error.found)}`;
case "unexpected-close-paren":
return spanLabel(error.span);
case "unexpected-code-point":
return `found ${foundLabel(error.found)}`;
case "invalid-number":
return `${error.reason}: ${error.text}`;
}
}
export function foundLabel(found: FoundSyntax): string {
switch (found.tag) {
case "eof":
return `EOF ${spanLabel(found.span)}`;
case "code-point":
return `${String.fromCodePoint(found.value)} ${spanLabel(found.span)}`;
}
}
export function spanLabel(span: CodePointSpan): string {
return `[${span.start}, ${span.end})`;
}

6
src/ui/types.ts Normal file
View file

@ -0,0 +1,6 @@
import type { CodePointSpan } from 'source-region';
export type HoverTarget = {
label: string;
span: CodePointSpan;
};

3
src/vite-env.d.ts vendored Normal file
View file

@ -0,0 +1,3 @@
/// <reference types="vite/client" />
declare module "*.css";

16
tmp_repl/tmp_repl.md
View file

@ -1,20 +1,20 @@
https://git.meatbagoverclocked.com/omedusyo/source-region.git VERY IMPORTANT: All the implementation work and running of commands I'll do by myself. You'll act as an intelligent rubber duck, or as a helpful assistant who will suggest particular commands for me to type in, or how certain functions work, or as a checker for what I'm doing etc. I don't want you to make decisions for me, only help me to check my work/thinking please. You can when appropriate generate snippets of code that could solve my concrete problem - I'll use them as an off ramp to write my own code.
CONTEXT:
I have this `https://git.meatbagoverclocked.com/omedusyo/source-region.git` ts package that I developed that basically defines its own Char/CodePoint type and uses it to have sane UTF8 strings in typescript. The resulting fat string (the `SourceText`) also tracks newline information and also defines basic abstractions for source location and spans. I have this `https://git.meatbagoverclocked.com/omedusyo/source-region.git` ts package that I developed that basically defines its own Char/CodePoint type and uses it to have sane UTF8 strings in typescript. The resulting fat string (the `SourceText`) also tracks newline information and also defines basic abstractions for source location and spans. It is included as a submodule.
I consider myself a language designer and often have many programming language ideas that I want to try out by making a new toy language. I consider myself a language designer and often have many programming language ideas that I want to try out by making a new toy language.
This time I wish to make something like a very simple Lisp - and the purpose of making it is to use the `source-region` library that I made and perhaps during the development develop another library that would allow to do scanning/tokenization pretty easily over my `SourceText` etc. This time I wish to make something like a very simple Lisp - and the purpose of making it is to use the `source-region` library that I made and perhaps during the development develop another library that would allow to do scanning/tokenization pretty easily over my `SourceText` etc.
Also I'm not really bound for the tokenization to be a simple linear stream of tokens. I'm also considering returning concrete syntax trees even in this basic phase. Also I'm not really bound for the tokenization to be a simple linear stream of tokens. I'm also considering returning concrete syntax trees even in this basic phase.
But before we do all that, let's create a new typescript project that submodules the `https://git.meatbagoverclocked.com/omedusyo/source-region.git` (via ssh preferably) and then creates a fat string from e.g. just a simple "hello, world" just to make sure everything is working correctly. I'm ok with using e.g. vite for development and doing some toy web app where later we'll implement a REPL/UI as a webapp. Right now I'm trying to think through of how to write a simple parser for a number. Let's say number is just a simple positive integer like `0` or `1` or `123` or `00122` etc.
All the implementation work and running of commands I'll do by myself. You'll act as an intelligent rubber duck, or as a helpful assistant who will suggest particular commands for me to type in, or how certain functions work, or as a checker for what I'm doing etc. I don't want you to make decisions for me, only help me to check my work/thinking please. Start of by reading the README.md of the source-region library.
```
git submodule add git.meatbagoverclocked.com:omedusyo/source-region.git libs/source-region npm run parser:experiments
```

3
tsconfig.json
View file

@ -3,10 +3,13 @@
"target": "es2023", "target": "es2023",
"module": "esnext", "module": "esnext",
"lib": ["ES2023", "DOM"], "lib": ["ES2023", "DOM"],
"jsx": "preserve",
"jsxImportSource": "solid-js",
"types": ["vite/client"], "types": ["vite/client"],
"skipLibCheck": true, "skipLibCheck": true,
"baseUrl": ".", "baseUrl": ".",
"ignoreDeprecations": "6.0",
"paths": { "paths": {
"source-region": [ "./libs/source-region/src/index.ts" ] "source-region": [ "./libs/source-region/src/index.ts" ]
}, },

2
vite.config.ts
View file

@ -1,7 +1,9 @@
import { defineConfig } from 'vite'; import { defineConfig } from 'vite';
import solid from 'vite-plugin-solid';
import path from 'path'; import path from 'path';
export default defineConfig({ export default defineConfig({
plugins: [solid()],
resolve: { resolve: {
alias: { alias: {
'source-region': path.resolve(__dirname, './libs/source-region/src/index.ts'), 'source-region': path.resolve(__dirname, './libs/source-region/src/index.ts'),