Academy — Sanjib Sikder

Your Code Is Just Text Until Something Reads It

When you write const x = 1 + 2, that is a string of characters. Before V8 can execute it, it must be understood — transformed from raw text into a structured representation the engine can reason about and optimise.

This transformation happens in three stages:

Source code (text)
    ↓  Lexer (tokeniser)
Stream of tokens
    ↓  Parser
Abstract Syntax Tree (AST)
    ↓  Interpreter / Compiler
Result

Babel, ESLint, TypeScript, Prettier, and V8 itself all begin here. Understanding this pipeline is what separates developers who use these tools from developers who can build them.

Stage 1 — The Lexer

A lexer (also called a tokeniser) reads raw source character by character and groups them into meaningful units called tokens. It does not understand structure — it just categorises text:

JavaScript

Ctrl+Enter to run

// A lexer for a simple arithmetic expression language
// Handles: numbers, +, -, *, /, (, ), identifiers

const TokenType = {
NUMBER: "NUMBER",
PLUS: "PLUS",
MINUS: "MINUS",
STAR: "STAR",
SLASH: "SLASH",
LPAREN: "LPAREN",
RPAREN: "RPAREN",
IDENT: "IDENT",
EOF: "EOF",
};

function lex(source) {
const tokens = [];
let i = 0;

while (i < source.length) {
// Skip whitespace
if (/\s/.test(source[i])) { i++; continue; }

    // Numbers (including decimals)
    if (/[0-9]/.test(source[i])) {
      let num = "";
      while (i < source.length && /[0-9.]/.test(source[i])) num += source[i++];
      tokens.push({ type: TokenType.NUMBER, value: Number(num) });
      continue;
    }

    // Identifiers (variable names)
    if (/[a-zA-Z_]/.test(source[i])) {
      let ident = "";
      while (i < source.length && /[a-zA-Z0-9_]/.test(source[i])) ident += source[i++];
      tokens.push({ type: TokenType.IDENT, value: ident });
      continue;
    }

    // Single-character tokens
    const single = {
      "+": TokenType.PLUS,  "-": TokenType.MINUS,
      "*": TokenType.STAR,  "/": TokenType.SLASH,
      "(": TokenType.LPAREN, ")": TokenType.RPAREN,
    };
    if (source[i] in single) {
      tokens.push({ type: single[source[i]], value: source[i] });
      i++;
      continue;
    }

    throw new Error(`Unexpected character: '${source[i]}' at position ${i}`);

}

tokens.push({ type: TokenType.EOF, value: null });
return tokens;
}

// See the raw token stream
const tokens = lex("3.14 * (radius + 2) / height");
tokens.forEach(t => console.log(`[${t.type.padEnd(8)}] ${JSON.stringify(t.value)}`));

// A lexer for a simple arithmetic expression language
// Handles: numbers, +, -, *, /, (, ), identifiers

const TokenType = {
NUMBER: "NUMBER",
PLUS: "PLUS",
MINUS: "MINUS",
STAR: "STAR",
SLASH: "SLASH",
LPAREN: "LPAREN",
RPAREN: "RPAREN",
IDENT: "IDENT",
EOF: "EOF",
};

function lex(source) {
const tokens = [];
let i = 0;

while (i < source.length) {
// Skip whitespace
if (/\s/.test(source[i])) { i++; continue; }

// Numbers (including decimals)
    if (/[0-9]/.test(source[i])) {
      let num = "";
      while (i < source.length && /[0-9.]/.test(source[i])) num += source[i++];
      tokens.push({ type: TokenType.NUMBER, value: Number(num) });
      continue;
    }

// Identifiers (variable names)
    if (/[a-zA-Z_]/.test(source[i])) {
      let ident = "";
      while (i < source.length && /[a-zA-Z0-9_]/.test(source[i])) ident += source[i++];
      tokens.push({ type: TokenType.IDENT, value: ident });
      continue;
    }

// Single-character tokens
    const single = {
      "+": TokenType.PLUS,  "-": TokenType.MINUS,
      "*": TokenType.STAR,  "/": TokenType.SLASH,
      "(": TokenType.LPAREN, ")": TokenType.RPAREN,
    };
    if (source[i] in single) {
      tokens.push({ type: single[source[i]], value: source[i] });
      i++;
      continue;
    }

throw new Error(`Unexpected character: '${source[i]}' at position ${i}`);

}

tokens.push({ type: TokenType.EOF, value: null });
return tokens;
}

// See the raw token stream
const tokens = lex("3.14 * (radius + 2) / height");
tokens.forEach(t => console.log(`[${t.type.padEnd(8)}] ${JSON.stringify(t.value)}`));

The lexer produced a flat list of tokens — it has no idea that (radius + 2) is a grouped sub-expression. That understanding is the parser's job.

Stage 2 — The Parser

A parser takes the token stream and builds a tree that reflects the structure and precedence of the expression. This tree is the AST — Abstract Syntax Tree:

JavaScript

Ctrl+Enter to run

// Reuse lex() from above — abbreviated here
function lex(src) {
  const T = { NUMBER:"NUMBER", PLUS:"PLUS", MINUS:"MINUS", STAR:"STAR", SLASH:"SLASH", LPAREN:"LPAREN", RPAREN:"RPAREN", IDENT:"IDENT", EOF:"EOF" };
  const tokens = []; let i = 0;
  while (i < src.length) {
    if (/\s/.test(src[i])) { i++; continue; }
    if (/[0-9]/.test(src[i])) { let n=""; while(i<src.length&&/[0-9.]/.test(src[i]))n+=src[i++]; tokens.push({type:T.NUMBER,value:Number(n)}); continue; }
    if (/[a-zA-Z_]/.test(src[i])) { let n=""; while(i<src.length&&/\w/.test(src[i]))n+=src[i++]; tokens.push({type:T.IDENT,value:n}); continue; }
    const m={"+":T.PLUS,"-":T.MINUS,"*":T.STAR,"/":T.SLASH,"(":T.LPAREN,")":T.RPAREN};
    if(src[i] in m){tokens.push({type:m[src[i]],value:src[i]});i++;continue;}
    throw new Error("Unexpected: "+src[i]);
  }
  tokens.push({type:T.EOF,value:null}); return tokens;
}

// Recursive descent parser — each function handles one precedence level
// Grammar (low → high precedence):
// expr → term (('+' | '-') term)_
// term → factor (('_' | '/') factor)*
// factor → NUMBER | IDENT | '(' expr ')'

function parse(source) {
const tokens = lex(source);
let pos = 0;

const peek = () => tokens[pos];
const consume = () => tokens[pos++];
const expect = (t) => {
if (peek().type !== t) throw new Error(`Expected ${t}, got ${peek().type}`);
return consume();
};

function expr() {
let node = term();
while (peek().type === "PLUS" || peek().type === "MINUS") {
const op = consume().value;
node = { type: "BinaryExpr", op, left: node, right: term() };
}
return node;
}

function term() {
let node = factor();
while (peek().type === "STAR" || peek().type === "SLASH") {
const op = consume().value;
node = { type: "BinaryExpr", op, left: node, right: factor() };
}
return node;
}

function factor() {
const tok = peek();
if (tok.type === "NUMBER") { consume(); return { type: "Number", value: tok.value }; }
if (tok.type === "IDENT") { consume(); return { type: "Ident", name: tok.value }; }
if (tok.type === "LPAREN") {
consume();
const node = expr();
expect("RPAREN");
return node;
}
throw new Error(`Unexpected token: ${tok.type}`);
}

return expr();
}

// See the AST structure
const ast = parse("2 + 3 _ 4");
console.log("2 + 3 _ 4:");
console.log(JSON.stringify(ast, null, 2));

// Note how _ binds tighter — it's deeper in the tree
const ast2 = parse("(2 + 3) _ 4");
console.log("\n(2 + 3) * 4:");
console.log(JSON.stringify(ast2, null, 2));

// Reuse lex() from above — abbreviated here
function lex(src) {
  const T = { NUMBER:"NUMBER", PLUS:"PLUS", MINUS:"MINUS", STAR:"STAR", SLASH:"SLASH", LPAREN:"LPAREN", RPAREN:"RPAREN", IDENT:"IDENT", EOF:"EOF" };
  const tokens = []; let i = 0;
  while (i < src.length) {
    if (/\s/.test(src[i])) { i++; continue; }
    if (/[0-9]/.test(src[i])) { let n=""; while(i<src.length&&/[0-9.]/.test(src[i]))n+=src[i++]; tokens.push({type:T.NUMBER,value:Number(n)}); continue; }
    if (/[a-zA-Z_]/.test(src[i])) { let n=""; while(i<src.length&&/\w/.test(src[i]))n+=src[i++]; tokens.push({type:T.IDENT,value:n}); continue; }
    const m={"+":T.PLUS,"-":T.MINUS,"*":T.STAR,"/":T.SLASH,"(":T.LPAREN,")":T.RPAREN};
    if(src[i] in m){tokens.push({type:m[src[i]],value:src[i]});i++;continue;}
    throw new Error("Unexpected: "+src[i]);
  }
  tokens.push({type:T.EOF,value:null}); return tokens;
}

// Recursive descent parser — each function handles one precedence level
// Grammar (low → high precedence):
// expr → term (('+' | '-') term)_
// term → factor (('_' | '/') factor)*
// factor → NUMBER | IDENT | '(' expr ')'

function parse(source) {
const tokens = lex(source);
let pos = 0;

const peek = () => tokens[pos];
const consume = () => tokens[pos++];
const expect = (t) => {
if (peek().type !== t) throw new Error(`Expected ${t}, got ${peek().type}`);
return consume();
};

return expr();
}

// See the AST structure
const ast = parse("2 + 3 _ 4");
console.log("2 + 3 _ 4:");
console.log(JSON.stringify(ast, null, 2));

// Note how _ binds tighter — it's deeper in the tree
const ast2 = parse("(2 + 3) _ 4");
console.log("\n(2 + 3) * 4:");
console.log(JSON.stringify(ast2, null, 2));

The tree for 2 + 3 * 4 has * deeper than + — because the parser's precedence rules correctly made 3 * 4 a sub-tree before adding 2. The tree is the precedence.

Stage 3 — The Interpreter

With an AST, evaluation is a simple recursive tree walk. Each node type has a handler — the interpreter pattern:

JavaScript

Ctrl+Enter to run

// Full pipeline: source → tokens → AST → value
// (lex and parse from above — inlined for completeness)

function lex(src) {
const T={NUMBER:"NUMBER",PLUS:"PLUS",MINUS:"MINUS",STAR:"STAR",SLASH:"SLASH",LPAREN:"LPAREN",RPAREN:"RPAREN",IDENT:"IDENT",EOF:"EOF"};
const tokens=[];let i=0;
while(i<src.length){if(/\s/.test(src[i])){i++;continue;}if(/[0-9]/.test(src[i])){let n="";while(i<src.length&&/[0-9.]/.test(src[i]))n+=src[i++];tokens.push({type:T.NUMBER,value:Number(n)});continue;}if(/[a-zA-Z_]/.test(src[i])){let n="";while(i<src.length&&/\w/.test(src[i]))n+=src[i++];tokens.push({type:T.IDENT,value:n});continue;}const m={"+":T.PLUS,"-":T.MINUS,"*":T.STAR,"/":T.SLASH,"(":T.LPAREN,")":T.RPAREN};if(src[i] in m){tokens.push({type:m[src[i]],value:src[i]});i++;continue;}throw new Error("Unexpected: "+src[i]);}
tokens.push({type:T.EOF,value:null});return tokens;
}

function parse(source) {
const tokens=lex(source);let pos=0;
const peek=()=>tokens[pos];const consume=()=>tokens[pos++];
const expect=t=>{if(peek().type!==t)throw new Error("Expected "+t);return consume();};
function expr(){let n=term();while(peek().type==="PLUS"||peek().type==="MINUS"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:term()};}return n;}
function term(){let n=factor();while(peek().type==="STAR"||peek().type==="SLASH"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:factor()};}return n;}
function factor(){const t=peek();if(t.type==="NUMBER"){consume();return{type:"Number",value:t.value};}if(t.type==="IDENT"){consume();return{type:"Ident",name:t.value};}if(t.type==="LPAREN"){consume();const n=expr();expect("RPAREN");return n;}throw new Error("Unexpected: "+t.type);}
return expr();
}

// The interpreter — evaluate an AST node given an environment (variable bindings)
function evaluate(node, env = {}) {
switch (node.type) {
case "Number":
return node.value;

    case "Ident":
      if (!(node.name in env)) throw new ReferenceError(`${node.name} is not defined`);
      return env[node.name];

    case "BinaryExpr": {
      const left  = evaluate(node.left,  env);
      const right = evaluate(node.right, env);
      switch (node.op) {
        case "+": return left + right;
        case "-": return left - right;
        case "*": return left * right;
        case "/":
          if (right === 0) throw new Error("Division by zero");
          return left / right;
      }
    }

    default:
      throw new Error(`Unknown node type: ${node.type}`);

}
}

// A complete calculator with variable support
function calc(source, env = {}) {
const ast = parse(source);
const result = evaluate(ast, env);
console.log(`${source.padEnd(30)} = ${result}`);
return result;
}

calc("2 + 3 _ 4");
calc("(2 + 3) _ 4");
calc("10 / (2 + 3)");
calc("3.14 _ radius _ radius", { radius: 7 });

// Physics formulas
const env = { g: 9.81, m: 5, h: 10, v: 20 };
calc("m _ g _ h", env); // potential energy
calc("m _ v _ v / 2", env); // kinetic energy (note: no *_ yet)
calc("(m _ g _ h) + (m _ v * v / 2)", env); // total mechanical energy

// Full pipeline: source → tokens → AST → value
// (lex and parse from above — inlined for completeness)

// The interpreter — evaluate an AST node given an environment (variable bindings)
function evaluate(node, env = {}) {
switch (node.type) {
case "Number":
return node.value;

case "Ident":
      if (!(node.name in env)) throw new ReferenceError(`${node.name} is not defined`);
      return env[node.name];

case "BinaryExpr": {
      const left  = evaluate(node.left,  env);
      const right = evaluate(node.right, env);
      switch (node.op) {
        case "+": return left + right;
        case "-": return left - right;
        case "*": return left * right;
        case "/":
          if (right === 0) throw new Error("Division by zero");
          return left / right;
      }
    }

default:
      throw new Error(`Unknown node type: ${node.type}`);

}
}

// A complete calculator with variable support
function calc(source, env = {}) {
const ast = parse(source);
const result = evaluate(ast, env);
console.log(`${source.padEnd(30)} = ${result}`);
return result;
}

calc("2 + 3 _ 4");
calc("(2 + 3) _ 4");
calc("10 / (2 + 3)");
calc("3.14 _ radius _ radius", { radius: 7 });

// Physics formulas
const env = { g: 9.81, m: 5, h: 10, v: 20 };
calc("m _ g _ h", env); // potential energy
calc("m _ v _ v / 2", env); // kinetic energy (note: no *_ yet)
calc("(m _ g _ h) + (m _ v * v / 2)", env); // total mechanical energy

The Visitor Pattern — Separating Traversal from Logic

Real tools like ESLint and Babel don't hard-code every operation into the interpreter. They use the Visitor pattern: the traversal walks the AST, and pluggable visitors define what to do at each node type:

JavaScript

Ctrl+Enter to run

// A general AST walker with visitor hooks
function walk(node, visitor) {
  visitor.enter?.(node);

// Recurse into children based on node type
switch (node.type) {
case "BinaryExpr":
walk(node.left, visitor);
walk(node.right, visitor);
break;
// Number and Ident are leaves — no children
}

visitor.exit?.(node);
}

// Parse a few expressions into ASTs
function lex(src){const T={NUMBER:"NUMBER",PLUS:"PLUS",MINUS:"MINUS",STAR:"STAR",SLASH:"SLASH",LPAREN:"LPAREN",RPAREN:"RPAREN",IDENT:"IDENT",EOF:"EOF"};const tokens=[];let i=0;while(i<src.length){if(/\s/.test(src[i])){i++;continue;}if(/[0-9]/.test(src[i])){let n="";while(i<src.length&&/[0-9.]/.test(src[i]))n+=src[i++];tokens.push({type:T.NUMBER,value:Number(n)});continue;}if(/[a-zA-Z_]/.test(src[i])){let n="";while(i<src.length&&/\w/.test(src[i]))n+=src[i++];tokens.push({type:T.IDENT,value:n});continue;}const m={"+":T.PLUS,"-":T.MINUS,"*":T.STAR,"/":T.SLASH,"(":T.LPAREN,")":T.RPAREN};if(src[i] in m){tokens.push({type:m[src[i]],value:src[i]});i++;continue;}throw new Error("Unexpected: "+src[i]);}tokens.push({type:T.EOF,value:null});return tokens;}
function parse(source){const tokens=lex(source);let pos=0;const peek=()=>tokens[pos];const consume=()=>tokens[pos++];const expect=t=>{if(peek().type!==t)throw new Error("Expected "+t);return consume();};function expr(){let n=term();while(peek().type==="PLUS"||peek().type==="MINUS"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:term()};}return n;}function term(){let n=factor();while(peek().type==="STAR"||peek().type==="SLASH"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:factor()};}return n;}function factor(){const t=peek();if(t.type==="NUMBER"){consume();return{type:"Number",value:t.value};}if(t.type==="IDENT"){consume();return{type:"Ident",name:t.value};}if(t.type==="LPAREN"){consume();const n=expr();expect("RPAREN");return n;}throw new Error("Unexpected: "+t.type);}return expr();}

// Visitor 1: collect all identifiers (like ESLint's no-undef rule)
function findIdents(source) {
const ast = parse(source);
const idents = new Set();
walk(ast, {
enter(node) {
if (node.type === "Ident") idents.add(node.name);
}
});
return [...idents];
}

// Visitor 2: count operations (like a complexity analyser)
function countOps(source) {
const ast = parse(source);
const count = {};
walk(ast, {
enter(node) {
if (node.type === "BinaryExpr") {
count[node.op] = (count[node.op] || 0) + 1;
}
}
});
return count;
}

// Visitor 3: code printer (like Prettier — serialise AST back to string)
function print(node) {
switch (node.type) {
case "Number": return String(node.value);
case "Ident": return node.name;
case "BinaryExpr":
const needsParens = node.op === "+" || node.op === "-";
const l = print(node.left);
const r = print(node.right);
return `${l} ${node.op} ${r}`;
}
}

const exprs = [
"radius * radius * 3.14",
"(a + b) * (a - b)",
"mass * gravity * height + mass * velocity * velocity / 2",
];

console.log("=== Identifiers found ===");
exprs.forEach(e => console.log(` "${e.slice(0,30)}…" → [${findIdents(e).join(", ")}]`));

console.log("\n=== Operation counts ===");
exprs.forEach(e => console.log(` "${e.slice(0,30)}…" →`, countOps(e)));

console.log("\n=== Reprinted ===");
exprs.forEach(e => console.log(` ${print(parse(e))}`));

// A general AST walker with visitor hooks
function walk(node, visitor) {
  visitor.enter?.(node);

// Recurse into children based on node type
switch (node.type) {
case "BinaryExpr":
walk(node.left, visitor);
walk(node.right, visitor);
break;
// Number and Ident are leaves — no children
}

visitor.exit?.(node);
}

// Parse a few expressions into ASTs
function lex(src){const T={NUMBER:"NUMBER",PLUS:"PLUS",MINUS:"MINUS",STAR:"STAR",SLASH:"SLASH",LPAREN:"LPAREN",RPAREN:"RPAREN",IDENT:"IDENT",EOF:"EOF"};const tokens=[];let i=0;while(i<src.length){if(/\s/.test(src[i])){i++;continue;}if(/[0-9]/.test(src[i])){let n="";while(i<src.length&&/[0-9.]/.test(src[i]))n+=src[i++];tokens.push({type:T.NUMBER,value:Number(n)});continue;}if(/[a-zA-Z_]/.test(src[i])){let n="";while(i<src.length&&/\w/.test(src[i]))n+=src[i++];tokens.push({type:T.IDENT,value:n});continue;}const m={"+":T.PLUS,"-":T.MINUS,"*":T.STAR,"/":T.SLASH,"(":T.LPAREN,")":T.RPAREN};if(src[i] in m){tokens.push({type:m[src[i]],value:src[i]});i++;continue;}throw new Error("Unexpected: "+src[i]);}tokens.push({type:T.EOF,value:null});return tokens;}
function parse(source){const tokens=lex(source);let pos=0;const peek=()=>tokens[pos];const consume=()=>tokens[pos++];const expect=t=>{if(peek().type!==t)throw new Error("Expected "+t);return consume();};function expr(){let n=term();while(peek().type==="PLUS"||peek().type==="MINUS"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:term()};}return n;}function term(){let n=factor();while(peek().type==="STAR"||peek().type==="SLASH"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:factor()};}return n;}function factor(){const t=peek();if(t.type==="NUMBER"){consume();return{type:"Number",value:t.value};}if(t.type==="IDENT"){consume();return{type:"Ident",name:t.value};}if(t.type==="LPAREN"){consume();const n=expr();expect("RPAREN");return n;}throw new Error("Unexpected: "+t.type);}return expr();}

// Visitor 1: collect all identifiers (like ESLint's no-undef rule)
function findIdents(source) {
const ast = parse(source);
const idents = new Set();
walk(ast, {
enter(node) {
if (node.type === "Ident") idents.add(node.name);
}
});
return [...idents];
}

// Visitor 2: count operations (like a complexity analyser)
function countOps(source) {
const ast = parse(source);
const count = {};
walk(ast, {
enter(node) {
if (node.type === "BinaryExpr") {
count[node.op] = (count[node.op] || 0) + 1;
}
}
});
return count;
}

// Visitor 3: code printer (like Prettier — serialise AST back to string)
function print(node) {
switch (node.type) {
case "Number": return String(node.value);
case "Ident": return node.name;
case "BinaryExpr":
const needsParens = node.op === "+" || node.op === "-";
const l = print(node.left);
const r = print(node.right);
return `${l} ${node.op} ${r}`;
}
}

const exprs = [
"radius * radius * 3.14",
"(a + b) * (a - b)",
"mass * gravity * height + mass * velocity * velocity / 2",
];

console.log("=== Identifiers found ===");
exprs.forEach(e => console.log(` "${e.slice(0,30)}…" → [${findIdents(e).join(", ")}]`));

console.log("\n=== Operation counts ===");
exprs.forEach(e => console.log(` "${e.slice(0,30)}…" →`, countOps(e)));

console.log("\n=== Reprinted ===");
exprs.forEach(e => console.log(` ${print(parse(e))}`));

This is exactly how ESLint plugins work: you register a visitor for node types you care about, ESLint walks the AST and calls your visitor, and you report problems. You never write a lexer or parser — ESLint provides the AST, you provide the logic.

AST Transformation — How Babel Works

Babel's core job is AST transformation: parse modern JavaScript, walk the AST, replace new syntax nodes with equivalent older syntax nodes, then serialise back to source. The transformation is just a visitor that returns new nodes:

JavaScript

Ctrl+Enter to run

// A mini transformer: optimise constant expressions at compile time
// "constant folding" — a real optimisation that compilers perform

function lex(src){const T={NUMBER:"NUMBER",PLUS:"PLUS",MINUS:"MINUS",STAR:"STAR",SLASH:"SLASH",LPAREN:"LPAREN",RPAREN:"RPAREN",IDENT:"IDENT",EOF:"EOF"};const tokens=[];let i=0;while(i<src.length){if(/\s/.test(src[i])){i++;continue;}if(/[0-9]/.test(src[i])){let n="";while(i<src.length&&/[0-9.]/.test(src[i]))n+=src[i++];tokens.push({type:T.NUMBER,value:Number(n)});continue;}if(/[a-zA-Z_]/.test(src[i])){let n="";while(i<src.length&&/\w/.test(src[i]))n+=src[i++];tokens.push({type:T.IDENT,value:n});continue;}const m={"+":T.PLUS,"-":T.MINUS,"*":T.STAR,"/":T.SLASH,"(":T.LPAREN,")":T.RPAREN};if(src[i] in m){tokens.push({type:m[src[i]],value:src[i]});i++;continue;}throw new Error("Unexpected: "+src[i]);}tokens.push({type:T.EOF,value:null});return tokens;}
function parse(source){const tokens=lex(source);let pos=0;const peek=()=>tokens[pos];const consume=()=>tokens[pos++];const expect=t=>{if(peek().type!==t)throw new Error("Expected "+t);return consume();};function expr(){let n=term();while(peek().type==="PLUS"||peek().type==="MINUS"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:term()};}return n;}function term(){let n=factor();while(peek().type==="STAR"||peek().type==="SLASH"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:factor()};}return n;}function factor(){const t=peek();if(t.type==="NUMBER"){consume();return{type:"Number",value:t.value};}if(t.type==="IDENT"){consume();return{type:"Ident",name:t.value};}if(t.type==="LPAREN"){consume();const n=expr();expect("RPAREN");return n;}throw new Error("Unexpected: "+t.type);}return expr();}
function print(node){if(node.type==="Number")return String(node.value);if(node.type==="Ident")return node.name;const l=print(node.left),r=print(node.right);return`(${l} ${node.op} ${r})`;}

// Transform: fold constant binary expressions into a single Number node
function constantFold(node) {
if (node.type !== "BinaryExpr") return node;

// Recursively fold children first (bottom-up)
const left = constantFold(node.left);
const right = constantFold(node.right);

// If both children are now Numbers, evaluate at compile time
if (left.type === "Number" && right.type === "Number") {
let value;
switch (node.op) {
case "+": value = left.value + right.value; break;
case "-": value = left.value - right.value; break;
case "_": value = left.value _ right.value; break;
case "/": value = left.value / right.value; break;
}
return { type: "Number", value }; // replace the whole subtree
}

return { ...node, left, right };
}

const exprs = [
"2 + 3 * 4", // all constants — fully foldable
"10 / 2 + 5 * 3", // all constants
"x * 3 + 2 + 4", // partially foldable (2+4 can fold, x*3 cannot)
"a + b", // no constants — nothing to fold
];

exprs.forEach(src => {
const ast = parse(src);
const optimised = constantFold(ast);
const before = print(ast);
const after = print(optimised);
const changed = before !== after;
console.log(`Before: ${before}`);
console.log(`After: ${after} ${changed ? "← optimised" : "← no change"}`);
console.log();
});

// A mini transformer: optimise constant expressions at compile time
// "constant folding" — a real optimisation that compilers perform

// Transform: fold constant binary expressions into a single Number node
function constantFold(node) {
if (node.type !== "BinaryExpr") return node;

// Recursively fold children first (bottom-up)
const left = constantFold(node.left);
const right = constantFold(node.right);

// If both children are now Numbers, evaluate at compile time
if (left.type === "Number" && right.type === "Number") {
let value;
switch (node.op) {
case "+": value = left.value + right.value; break;
case "-": value = left.value - right.value; break;
case "_": value = left.value _ right.value; break;
case "/": value = left.value / right.value; break;
}
return { type: "Number", value }; // replace the whole subtree
}

return { ...node, left, right };
}

const exprs = [
"2 + 3 * 4", // all constants — fully foldable
"10 / 2 + 5 * 3", // all constants
"x * 3 + 2 + 4", // partially foldable (2+4 can fold, x*3 cannot)
"a + b", // no constants — nothing to fold
];

exprs.forEach(src => {
const ast = parse(src);
const optimised = constantFold(ast);
const before = print(ast);
const after = print(optimised);
const changed = before !== after;
console.log(`Before: ${before}`);
console.log(`After: ${after} ${changed ? "← optimised" : "← no change"}`);
console.log();
});

This is constant folding — when V8 sees const PI_2 = 3.14159 * 2, it evaluates it at compile time and replaces the entire expression with 6.28318. You are watching a real compiler optimisation in action, built in twenty lines of JavaScript.

A Complete Mini Language

Extend the language to support let statements and sequences — a tiny but complete programming language:

JavaScript

Ctrl+Enter to run

// Mini language: let bindings + expressions
// Grammar:
//   program → statement* EOF
//   statement → 'let' IDENT '=' expr | expr
//   expr/term/factor — as before

function createInterpreter() {
function lexFull(src) {
const T={NUMBER:"NUMBER",PLUS:"PLUS",MINUS:"MINUS",STAR:"STAR",SLASH:"SLASH",LPAREN:"LPAREN",RPAREN:"RPAREN",IDENT:"IDENT",LET:"LET",EQ:"EQ",EOF:"EOF"};
const keywords={let:T.LET};const tokens=[];let i=0;
while(i<src.length){if(/\s/.test(src[i])){i++;continue;}if(/[0-9]/.test(src[i])){let n="";while(i<src.length&&/[0-9.]/.test(src[i]))n+=src[i++];tokens.push({type:T.NUMBER,value:Number(n)});continue;}if(/[a-zA-Z_]/.test(src[i])){let n="";while(i<src.length&&/\w/.test(src[i]))n+=src[i++];const type=keywords[n]||T.IDENT;tokens.push({type,value:n});continue;}const m={"+":T.PLUS,"-":T.MINUS,"*":T.STAR,"/":T.SLASH,"(":T.LPAREN,")":T.RPAREN,"=":T.EQ};if(src[i] in m){tokens.push({type:m[src[i]],value:src[i]});i++;continue;}throw new Error("Unexpected: "+src[i]);}
tokens.push({type:T.EOF,value:null});return tokens;
}

function parseFull(source) {
const tokens=lexFull(source);let pos=0;
const peek=()=>tokens[pos];const consume=()=>tokens[pos++];
const expect=t=>{if(peek().type!==t)throw new Error("Expected "+t+", got "+peek().type);return consume();};
function expr(){let n=term();while(peek().type==="PLUS"||peek().type==="MINUS"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:term()};}return n;}
function term(){let n=factor();while(peek().type==="STAR"||peek().type==="SLASH"){const op=consume().value;n={type:"BinaryExpr",op,left:n,right:factor()};}return n;}
function factor(){const t=peek();if(t.type==="NUMBER"){consume();return{type:"Number",value:t.value};}if(t.type==="IDENT"){consume();return{type:"Ident",name:t.value};}if(t.type==="LPAREN"){consume();const n=expr();expect("RPAREN");return n;}throw new Error("Unexpected token: "+t.type);}
function statement(){if(peek().type==="LET"){consume();const name=expect("IDENT").value;expect("EQ");const value=expr();return{type:"LetDecl",name,value};}return{type:"ExprStmt",expr:expr()};}
const stmts=[];while(peek().type!=="EOF")stmts.push(statement());
return{type:"Program",body:stmts};
}

return {
run(source) {
const ast = parseFull(source);
const env = {};

      function evalNode(node) {
        switch (node.type) {
          case "Number":     return node.value;
          case "Ident":
            if (!(node.name in env)) throw new ReferenceError(node.name + " is not defined");
            return env[node.name];
          case "BinaryExpr":
            const l = evalNode(node.left), r = evalNode(node.right);
            switch(node.op){case"+":return l+r;case"-":return l-r;case"*":return l*r;case"/":return l/r;}
          case "LetDecl":
            env[node.name] = evalNode(node.value);
            return env[node.name];
          case "ExprStmt":
            return evalNode(node.expr);
        }
      }

      let last;
      for (const stmt of ast.body) last = evalNode(stmt);
      return { result: last, env };
    }

};
}

const interp = createInterpreter();

const program = `
let radius = 7
let pi = 3.14159
let area = pi _ radius _ radius
let circumference = 2 _ pi _ radius
area + circumference
`;

const { result, env } = interp.run(program);
console.log("Environment after execution:");
for (const [k, v] of Object.entries(env)) {
console.log(` ${k} = ${v.toFixed(4)}`);
}
console.log("Final expression:", result.toFixed(4));

// Mini language: let bindings + expressions
// Grammar:
//   program → statement* EOF
//   statement → 'let' IDENT '=' expr | expr
//   expr/term/factor — as before

return {
run(source) {
const ast = parseFull(source);
const env = {};

let last;
      for (const stmt of ast.body) last = evalNode(stmt);
      return { result: last, env };
    }

};
}

const interp = createInterpreter();

const program = `
let radius = 7
let pi = 3.14159
let area = pi _ radius _ radius
let circumference = 2 _ pi _ radius
area + circumference
`;

const { result, env } = interp.run(program);
console.log("Environment after execution:");
for (const [k, v] of Object.entries(env)) {
console.log(` ${k} = ${v.toFixed(4)}`);
}
console.log("Final expression:", result.toFixed(4));

You just built a programming language. It has variables, arithmetic, operator precedence, and a runtime environment. Every language — from Python to Rust to JavaScript itself — is this pipeline at a larger scale.

Real-World AST Tools You Can Use Today

You do not need to build your own parser for production work — mature tools exist for every use case:

JavaScript

Ctrl+Enter to run

// The ecosystem — what each tool does at the AST level

const tools = {
"Babel": {
what: "Transpiler — parses modern JS/TS, transforms AST, generates older JS",
uses: ["JSX → JS", "TS → JS", "ES2024 → ES5", "Plugin API for custom transforms"],
parser: "@babel/parser",
example: "import React from 'react' → const React = require('react')",
},
"ESLint": {
what: "Linter — walks the AST, visitors report rule violations",
uses: ["no-unused-vars", "no-undef", "prefer-const", "custom rules"],
parser: "espree (uses acorn under the hood)",
example: "Rule visitor: CallExpression → check if it's eval()",
},
"Prettier": {
what: "Formatter — parses to AST, reprints with consistent style",
uses: ["Enforce consistent formatting regardless of input style"],
parser: "recast / custom per language",
example: "Ignores all whitespace/style, reprints from AST → canonical form",
},
"TypeScript": {
what: "Type checker — parses TS AST, performs type inference, emits JS",
uses: ["Type checking", "Language server (autocomplete)", "Declaration files"],
parser: "ts.createSourceFile()",
example: "VariableDeclaration → infer type → check all usages → emit error or JS",
},
"acorn / oxc / swc": {
what: "Raw parsers — produce AST, nothing else",
uses: ["Foundation for other tools", "Custom analysis pipelines"],
parser: "self",
example: "acorn.parse(source, {ecmaVersion: 2024}) → AST",
},
};

// What they all share: the ESTree AST spec
// Most JS tools use the same AST node types so plugins are interoperable

console.log("All major JS tools share the ESTree AST specification:");
console.log(" Program → body: [Statement]");
console.log(" VariableDeclaration → declarations: [VariableDeclarator]");
console.log(" FunctionDeclaration → id, params, body");
console.log(" BinaryExpression → operator, left, right");
console.log(" CallExpression → callee, arguments");
console.log("");
console.log("Tools in this ecosystem:");
for (const [name, info] of Object.entries(tools)) {
console.log(`\n ${name}`);
console.log(` ${info.what}`);
console.log(` Parser: ${info.parser}`);
}

// Try it yourself — paste any JS into https://astexplorer.net
// Select @babel/parser or acorn and see the live AST

// The ecosystem — what each tool does at the AST level

const tools = {
"Babel": {
what: "Transpiler — parses modern JS/TS, transforms AST, generates older JS",
uses: ["JSX → JS", "TS → JS", "ES2024 → ES5", "Plugin API for custom transforms"],
parser: "@babel/parser",
example: "import React from 'react' → const React = require('react')",
},
"ESLint": {
what: "Linter — walks the AST, visitors report rule violations",
uses: ["no-unused-vars", "no-undef", "prefer-const", "custom rules"],
parser: "espree (uses acorn under the hood)",
example: "Rule visitor: CallExpression → check if it's eval()",
},
"Prettier": {
what: "Formatter — parses to AST, reprints with consistent style",
uses: ["Enforce consistent formatting regardless of input style"],
parser: "recast / custom per language",
example: "Ignores all whitespace/style, reprints from AST → canonical form",
},
"TypeScript": {
what: "Type checker — parses TS AST, performs type inference, emits JS",
uses: ["Type checking", "Language server (autocomplete)", "Declaration files"],
parser: "ts.createSourceFile()",
example: "VariableDeclaration → infer type → check all usages → emit error or JS",
},
"acorn / oxc / swc": {
what: "Raw parsers — produce AST, nothing else",
uses: ["Foundation for other tools", "Custom analysis pipelines"],
parser: "self",
example: "acorn.parse(source, {ecmaVersion: 2024}) → AST",
},
};

// What they all share: the ESTree AST spec
// Most JS tools use the same AST node types so plugins are interoperable

console.log("All major JS tools share the ESTree AST specification:");
console.log(" Program → body: [Statement]");
console.log(" VariableDeclaration → declarations: [VariableDeclarator]");
console.log(" FunctionDeclaration → id, params, body");
console.log(" BinaryExpression → operator, left, right");
console.log(" CallExpression → callee, arguments");
console.log("");
console.log("Tools in this ecosystem:");
for (const [name, info] of Object.entries(tools)) {
console.log(`\n ${name}`);
console.log(` ${info.what}`);
console.log(` Parser: ${info.parser}`);
}

// Try it yourself — paste any JS into https://astexplorer.net
// Select @babel/parser or acorn and see the live AST

What You Have Learned

Parsing is the foundational technology beneath every developer tool. Understanding the pipeline means you can read Babel plugin docs, write ESLint rules, and understand TypeScript error messages at a structural level.

The key ideas:

Lexer: raw text → flat stream of typed tokens — no understanding of structure, just categorisation
Parser: token stream → AST — the tree captures operator precedence, grouping, and nesting naturally
Recursive descent: one function per grammar rule, naturally handles precedence through call depth
Interpreter: walk the AST recursively, evaluate each node type — the tree structure makes precedence evaluation automatic
Visitor pattern: decouple traversal from logic — pluggable visitors at each node type are how ESLint rules and Babel plugins work
Transformation: walk the AST, replace nodes with new nodes, serialise back — this is Babel's entire job
Constant folding: evaluate constant sub-trees at compile time — a real optimisation, trivially expressible as an AST transform
Every major JS tool (Babel, ESLint, Prettier, TypeScript) uses the same ESTree AST specification — learn the node shapes once, read all of them

In the final lesson: Security — XSS, CSRF, injection, prototype pollution, and the patterns that make JavaScript applications safe by construction rather than by luck.

Parsing & ASTs

Your Code Is Just Text Until Something Reads It

Stage 1 — The Lexer

Stage 2 — The Parser

Stage 3 — The Interpreter

The Visitor Pattern — Separating Traversal from Logic

AST Transformation — How Babel Works

A Complete Mini Language

Real-World AST Tools You Can Use Today

What You Have Learned