Previously in compiler basics:

1. lisp to assembly

2. user-defined functions and variables

3. LLVM

4. LLVM conditionals and compiling fibonacci

5. LLVM system calls

This post upgrades the ulisp x86 backend from using a limited set of registers (with no spilling support) to solely using the stack to pass values between expressions.

This is a slightly longer post since we've got a lot of catchup to do to get to feature parity with the LLVM backend. Namely:

"Infinite" locals, parameters

Function definitions

Variable references

Arithmetic and logical operations

If

Syscalls

We'll tackle the first four points first and finish up with the last two. This way we can support the same fibonacci program that prints integers to stdout that we support in the LLVM backend.

As always the code is available on Github.

But first a digression into how this is suddenly easy for us to do with x86 and one-pass (sorta) code generation.

Stack-based languages

Stack-based languages have the extremely convenient attribute that values are (by default) stored on the stack, which allows a code generator targeting a stack-based language the option to omit handling register allocation. And as it happens, x86 has enough support to make it easy to treat as a stack machine.

As we build out the code generator for x86 as a stack machine we need to keep two commitments in mind:

Every expression must pop all arguments/operands

Every expression must store one result back on the stack

In the future, we may replace the second commitment. But for now it is more than enough.

Boilerplate

We'll start with the existing x86 backend code and strip all the implementation code:

const cp = require('child_process'); const fs = require('fs'); const os = require('os'); let GLOBAL_COUNTER = 0; const SYSCALL_MAP = { darwin: { exit: '0x2000001', write: '0x2000004', }, linux: { exit: 60, write: 1, }, }[os.platform()]; class Scope {} class Compiler { constructor() { this.outBuffer = []; this.primitiveFunctions = { def: this.compileDefine.bind(this), begin: this.compileBegin.bind(this), if: this.compileIf.bind(this), ...this.prepareArithmeticWrappers(), ...this.prepareLogicalWrappers(), ...this.prepareSyscallWrappers(), }; } prepareArithmeticWrappers() {} prepareLogicalWrappers() {} prepareSyscallWrappers() {} emit(depth, args) { if (depth === undefined || args === undefined) { throw new Error('Invalid call to emit'); } const indent = new Array(depth + 1).join(' '); this.outBuffer.push(indent + args); } compileExpression(arg, scope, depth) {} compileIf([test, then, els], scope, depth) {} compileBegin(body, scope, depth, topLevel = false) {} compileDefine([name, params, ...body], scope, depth) {} compileCall(fun, args, scope, depth) {} emitPrefix() { this.emit(1, '.global _main

'); this.emit(1, '.text

'); } emitPostfix() { this.emit(0, '_main:'); this.emit(1, 'CALL main'); this.emit(1, 'MOV RDI, RAX'); // Set exit arg this.emit(1, `MOV RAX, ${SYSCALL_MAP['exit']}`); this.emit(1, 'SYSCALL'); } getOutput() { const output = this.outBuffer.join('

'); // Leave at most one empty line return output.replace(/





+/g, '



'); } } module.exports.compile = function(ast) { const c = new Compiler(); c.emitPrefix(); const s = new Scope(); c.compileBegin(ast, s, 1, true); c.emitPostfix(); return c.getOutput(); }; module.exports.build = function(buildDir, program) { const prog = 'prog'; fs.writeFileSync(`${buildDir}/${prog}.s`, program); cp.execSync( `gcc -mstackrealign -masm=intel -o ${buildDir}/${prog} ${buildDir}/${prog}.s`, ); };

The prefix and postfix stays mostly the same as the original implementation. But we'll assume a couple of new helpers to get us in parity with the LLVM backend:

compileDefine

compileBegin

compileIf

compileCall

prepareArithmeticWrappers

prepareLogicalWrappers

prepareSyscallWrappers

The prepareArithmeticWrappers helper will define wrappers for arithmetic instructions. The prepareLogicalWrappers helper will define wrappers for logical instructions. And the prepareSyscallWrappers helper will define a wrapper for syscalls and generate builtins based on the SYSCALL_MAP entries.

Scope

Similar to our LLVM backend's Context and Scope helpers we'll define our own for the x86 backend. Since we're placing all locals on the stack, the two most important things Scope will do for us are:

Map identifiers to escaped strings

Store and increment the location of the local on the stack

Here's what it will look like:

class Scope { constructor() { this.localOffset = 1; this.map = {}; } assign(name) { const safe = name.replace('-', '_'); this.map[safe] = this.localOffset++; return safe; } symbol() { return this.localOffset++; } lookup(name) { const safe = name.replace('-', '_'); if (this.map[safe]) { return { name: safe, offset: this.map[safe] }; } return null; } copy() { const s = new Scope(); // In the future we may need to store s.scopeOffset = this.scopeOffset + 1 // so we can read outer-scoped values at runtime. s.map = { ...this.map }; return s; } }

compileExpression

An expression will be one of:

A function call (possibly a builtin like def or + )

or ) A literal value (e.g. 29 )

) A reference (e.g. &c )

) An identifier (e.g. my-var )

We'll handle compiling an expression in that order. If the AST argument passed to compileExpression is an array, we will call compileCall and return.

compileExpression(arg, scope, depth) { // Is a nested function call, compile it if (Array.isArray(arg)) { this.compileCall(arg[0], arg.slice(1), scope, depth); return; } }

If the AST is a number, we will push the number onto the stack and return.

compileExpression(arg, scope, depth) { // Is a nested function call, compile it if (Array.isArray(arg)) { this.compileCall(arg[0], arg.slice(1), scope, depth); return; } if (Number.isInteger(arg)) { this.emit(depth, `PUSH ${arg}`); return; } }

If the AST is a string that starts with & we will look up the location of the identifier after the & , push its location onto the stack and return.

We count on the Scope storing its offset from the "frame pointer", which we will later set up to be stored in RBP .

Locals will be stored after the frame pointer and parameters will be stored before it. So we'll need to add or subtract from the frame pointer depending on if we need a positive or negative offset from it.

compileExpression(arg, scope, depth) { // Is a nested function call, compile it if (Array.isArray(arg)) { this.compileCall(arg[0], arg.slice(1), scope, depth); return; } if (Number.isInteger(arg)) { this.emit(depth, `PUSH ${arg}`); return; } if (arg.startsWith('&')) { const { offset } = scope.lookup(arg.substring(1)); // Copy the frame pointer so we can return an offset from it this.emit(depth, `MOV RAX, RBP`); const operation = offset < 0 ? 'ADD' : 'SUB'; this.emit(depth, `${operation} RAX, ${Math.abs(offset * 8)} # ${arg}`); this.emit(depth, `PUSH RAX`); return; } }

Finally, we'll look up the identifier and copy the value (in its offset on the stack) to the top of the stack.

compileExpression(arg, scope, depth) { // Is a nested function call, compile it if (Array.isArray(arg)) { this.compileCall(arg[0], arg.slice(1), scope, depth); return; } if (Number.isInteger(arg)) { this.emit(depth, `PUSH ${arg}`); return; } if (arg.startsWith('&')) { const { offset } = scope.lookup(arg.substring(1)); // Copy the frame pointer so we can return an offset from it this.emit(depth, `MOV RAX, RBP`); const operation = offset < 0 ? 'ADD' : 'SUB'; this.emit(depth, `${operation} RAX, ${Math.abs(offset * 8)} # ${arg}`); this.emit(depth, `PUSH RAX`); return; } // Variable lookup const { offset } = scope.lookup(arg); if (offset) { const operation = offset < 0 ? '+' : '-'; this.emit( depth, `PUSH [RBP ${operation} ${Math.abs(offset * 8)}] # ${arg}`, ); } else { throw new Error( 'Attempt to reference undefined variable or unsupported literal: ' + arg, ); } }

And that's it for handling expression! Let's add compileCall support now that we referenced it.

compileCall

A call will first check if the call is a builtin. If so, it will immediately pass control to the builtin.

compileCall(fun, args, scope, depth) { if (this.primitiveFunctions[fun]) { this.primitiveFunctions[fun](args, scope, depth); return; } }

Otherwise it will compile every argument to the call (which will leave all the resulting values on the stack.)

compileCall(fun, args, scope, depth) { if (this.primitiveFunctions[fun]) { this.primitiveFunctions[fun](args, scope, depth); return; } // Compile registers and store on the stack args.map((arg, i) => this.compileExpression(arg, scope, depth)); }

Then we will check that function is defined and call it.

compileCall(fun, args, scope, depth) { if (this.primitiveFunctions[fun]) { this.primitiveFunctions[fun](args, scope, depth); return; } // Compile registers and store on the stack args.map((arg, i) => this.compileExpression(arg, scope, depth)); const fn = scope.lookup(fun); if (fn) { this.emit(depth, `CALL ${fn.name}`); } else { throw new Error('Attempt to call undefined function: ' + fun); } }

Then we'll reset the stack pointer (to maintain our commitment) based on the number of arguments and push RAX (where the return result of the function call will be stored) onto the stack. We'll make two minor optimizations for when there is exactly zero or one argument to the function.

compileCall(fun, args, scope, depth) { if (this.primitiveFunctions[fun]) { this.primitiveFunctions[fun](args, scope, depth); return; } // Compile registers and store on the stack args.map((arg, i) => this.compileExpression(arg, scope, depth)); const fn = scope.lookup(fun); if (fn) { this.emit(depth, `CALL ${fn.name}`); } else { throw new Error('Attempt to call undefined function: ' + fun); } if (args.length > 1) { // Drop the args this.emit(depth, `ADD RSP, ${args.length * 8}`); } if (args.length === 1) { this.emit(depth, `MOV [RSP], RAX

`); } else { this.emit(depth, 'PUSH RAX

'); } }

When there is only one argument, we can just set the top value on the stack to be the return result of the call rather than resetting the stack pointer just to push onto it.

And that's it for compileCall ! Now that we've got a feel for expressions and function calls, let's add some simple arithmetic operations.

prepareArithmeticWrappers

There are two major kind of arithmetic instructions we'll wrap for now:

"General" instructions that operate on two arguments, putting the return result in the first argument

"RAX" instructions that operate on RAX and the first argument, putting the return result in RAX and possibly RDX

prepareGeneral

This helper will compile its two arguments and pop the second argument into RAX . This is because x86 instructions typically require one argument to be a register if one argument is allowed to be a memory address.

We'll use the stack address as the first argument so 1) that non-commutative operations are correct and 2) the result is stored right back onto the stack in the right location.

const prepareGeneral = (instruction) => (arg, scope, depth) => { depth++; this.emit(depth, `# ${instruction.toUpperCase()}`); // Compile first argument this.compileExpression(arg[0], scope, depth); // Compile second argument this.compileExpression(arg[1], scope, depth); this.emit(depth, `POP RAX`); // Compile operation this.emit(depth, `${instruction.toUpperCase()} [RSP], RAX`); this.emit(depth, `# End ${instruction.toUpperCase()}`); };

prepareRax

This helper will similarly compile its two arguments and pop the second argument into RAX . But the RAX-implicit instructions require the argument to be stored in a register so we'll use the XCHG instruction to swap RAX with the value on the top of the stack (the first argument).

const prepareRax = (instruction, outRegister = 'RAX') => ( arg, scope, depth, ) => { depth++; this.emit(depth, `# ${instruction.toUpperCase()}`); // Compile first argument, store in RAX this.compileExpression(arg[0], scope, depth); // Compile second argument this.compileExpression(arg[1], scope, depth); // POP second argument and swap with first this.emit(depth, `POP RAX`); this.emit(depth, `XCHG [RSP], RAX`);

This may seem roundabout but remember that we must pop all arguments to the instruction to maintain our commitment.

Next we'll zero out the RDX register if the operation is DIV , perform the operation, and store the result on the top of the stack.

const prepareRax = (instruction, outRegister = 'RAX') => ( arg, scope, depth, ) => { depth++; this.emit(depth, `# ${instruction.toUpperCase()}`); // Compile first argument, store in RAX this.compileExpression(arg[0], scope, depth); // Compile second argument this.compileExpression(arg[1], scope, depth); // POP second argument and swap with first this.emit(depth, `POP RAX`); this.emit(depth, `XCHG [RSP], RAX`); // Reset RDX for DIV if (instruction.toUpperCase() === 'DIV') { this.emit(depth, `XOR RDX, RDX`); } // Compiler operation this.emit(depth, `${instruction.toUpperCase()} QWORD PTR [RSP]`); // Swap the top of the stack this.emit(depth, `MOV [RSP], ${outRegister}`); };

We parameterize the out register because the % wrapper will call DIV but need RDX rather than RAX after the operation.

prepareArithmeticWrappers

Putting everything together we get:

prepareArithmeticWrappers() { // General operatations const prepareGeneral = (instruction) => (arg, scope, depth) => { depth++; this.emit(depth, `# ${instruction.toUpperCase()}`); // Compile first argument this.compileExpression(arg[0], scope, depth); // Compile second argument this.compileExpression(arg[1], scope, depth); this.emit(depth, `POP RAX`); // Compile operation this.emit(depth, `${instruction.toUpperCase()} [RSP], RAX`); this.emit(depth, `# End ${instruction.toUpperCase()}`); }; // Operations that use RAX implicitly const prepareRax = (instruction, outRegister = 'RAX') => ( arg, scope, depth, ) => { depth++; this.emit(depth, `# ${instruction.toUpperCase()}`); // Compile first argument, store in RAX this.compileExpression(arg[0], scope, depth); // Compile second argument this.compileExpression(arg[1], scope, depth); // POP second argument and swap with first this.emit(depth, `POP RAX`); this.emit(depth, `XCHG [RSP], RAX`); // Reset RDX for DIV if (instruction.toUpperCase() === 'DIV') { this.emit(depth, `XOR RDX, RDX`); } // Compiler operation this.emit(depth, `${instruction.toUpperCase()} QWORD PTR [RSP]`); // Swap the top of the stack this.emit(depth, `MOV [RSP], ${outRegister}`); }; return { '+': prepareGeneral('add'), '-': prepareGeneral('sub'), '&': prepareGeneral('and'), '|': prepareGeneral('or'), '=': prepareGeneral('mov'), '*': prepareRax('mul'), '/': prepareRax('div'), '%': prepareRax('div', 'RDX'), }; }

Next we'll tackle compileBegin and compileDefine .

compileBegin

A begin form is an expression made up of a series of expressions where all expression values are thrown away and the last expression value is the result of the begin form.

To compile this form we will compile each expression passed in and pop from the stack to throw its value away. If the expression is the last in the list we will not pop since it is the result of the begin form.

We will add one exception to this popping logic: if the begin is called from the top-level we will omit the popping.

compileBegin(body, scope, depth, topLevel = false) { body.forEach((expression, i) => { this.compileExpression(expression, scope, depth); if (!topLevel && i < body.length - 1) { this.emit(depth, `POP RAX # Ignore non-final expression`); } }); }

That's it for compileBegin !

compileDefine

The prelude for a function definition will add its name to scope, push the current frame pointer ( RBP ) onto the stack and store the current stack pointer ( RSP ) as the new frame pointer ( RBP ).

Remember that we use the frame pointer as a point of reference when setting and getting local and parameter values. It works out entirely by convention.

compileDefine([name, params, ...body], scope, depth) { // Add this function to outer scope const safe = scope.assign(name); // Copy outer scope so parameter mappings aren't exposed in outer scope. const childScope = scope.copy(); this.emit(0, `${safe}:`); this.emit(depth, `PUSH RBP`); this.emit(depth, `MOV RBP, RSP

`); }

Next we copy the parameters into local scope at their negative (from the frame pointer) location. In the future we may decide to actually copy in the parameter values into the local stack but for now there's no benefit.

compileDefine([name, params, ...body], scope, depth) { // Add this function to outer scope const safe = scope.assign(name); // Copy outer scope so parameter mappings aren't exposed in outer scope. const childScope = scope.copy(); this.emit(0, `${safe}:`); this.emit(depth, `PUSH RBP`); this.emit(depth, `MOV RBP, RSP

`); // Copy params into local scope params.forEach((param, i) => { childScope.map[param] = -1 * (params.length - i - 1 + 2); }); }

Next we'll compile the body of the function within a begin block.

compileDefine([name, params, ...body], scope, depth) { // Add this function to outer scope const safe = scope.assign(name); // Copy outer scope so parameter mappings aren't exposed in outer scope. const childScope = scope.copy(); this.emit(0, `${safe}:`); this.emit(depth, `PUSH RBP`); this.emit(depth, `MOV RBP, RSP

`); // Copy params into local scope params.forEach((param, i) => { childScope.map[param] = -1 * (params.length - i - 1 + 2); }); // Pass childScope in for reference when body is compiled. this.compileBegin(body, childScope, depth); }

Then in the postlude we'll pop the stack (for the return result of the begin form), save it in RAX, pop the previous frame pointer back to restore the calling frame, and return.

compileDefine([name, params, ...body], scope, depth) { // Add this function to outer scope const safe = scope.assign(name); // Copy outer scope so parameter mappings aren't exposed in outer scope. const childScope = scope.copy(); this.emit(0, `${safe}:`); this.emit(depth, `PUSH RBP`); this.emit(depth, `MOV RBP, RSP

`); // Copy params into local scope params.forEach((param, i) => { childScope.map[param] = -1 * (params.length - i - 1 + 2); }); // Pass childScope in for reference when body is compiled. this.compileBegin(body, childScope, depth); // Save the return value this.emit(0, ''); this.emit(depth, `POP RAX`); this.emit(depth, `POP RBP

`); this.emit(depth, 'RET

'); }

And now we're ready to compile a simple program!

Our first program

Here's a simple one we can support:

$ cat tests/meaning-of-life.lisp (def main () (+ 8 (* 2 17)))

We'll compile this program without the ulisp kernel (which contains a lisp library we cannot currently compile):

$ node src/ulisp.js tests/meaning-of-life.lisp --no-kernel --backend x86 $ ./build/prog $ echo $? 42

Not bad. Let's finish up with support for prepareLogicalWrappers , prepareSyscallWrappers , and compileIf .

prepareLogicalWrappers

Storing logical results as values is a bit of pain. Most of the internet wants you to use branching. And a better compiler may optimize an idiom like (if (> 5 2) ...) into a single branch.

But we're going to resort to an instruction I just learned about called CMOV . This allows us to conditionally assign a value based on flags, similar to how you can conditionally branch.

Otherwise we'll follow a pattern similar to our arithmetic wrappers. At the end of the procedure we will have a 0 or a 1 on the top of the stack.

prepareLogicalWrappers() { const prepareComparison = (operator) => { return { [operator]: (arg, scope, depth) => { depth++; this.emit(depth, `# ${operator}`); // Compile first argument, store in RAX this.compileExpression(arg[0], scope, depth); // Compile second argument this.compileExpression(arg[1], scope, depth); this.emit(depth, `POP RAX`); // Compile operation this.emit(depth, `CMP [RSP], RAX`); // Reset RAX to serve as CMOV* dest, MOV to keep flags (vs. XOR) this.emit(depth, `MOV RAX, 0`); // Conditional set [RSP] const operation = { '>': 'CMOVA', '>=': 'CMOVAE', '<': 'CMOVB', '<=': 'CMOVBE', '==': 'CMOVE', '!=': 'CMOVNE', }[operator]; // CMOV* requires the source to be memory or register this.emit(depth, `MOV DWORD PTR [RSP], 1`); // CMOV* requires the dest to be a register this.emit(depth, `${operation} RAX, [RSP]`); this.emit(depth, `MOV [RSP], RAX`); this.emit(depth, `# End ${operator}`); }, }; }; return { ...prepareComparison('>'), ...prepareComparison('>='), ...prepareComparison('<'), ...prepareComparison('<='), ...prepareComparison('=='), ...prepareComparison('!='), } }

prepareSyscallWrappers

This helper is similar to compileCall except for that it needs to follow the SYS V ABI and use the SYSCALL instruction rather than CALL .

prepareSyscallWrappers() { const registers = ['RDI', 'RSI', 'RDX', 'R10', 'R8', 'R9']; const wrappers = {}; Object.keys(SYSCALL_MAP).forEach((key, obj) => { wrappers[`syscall/${key}`] = (args, scope, depth) => { if (args.length > registers.length) { throw new Error(`Too many arguments to syscall/${key}`); } // Compile first args.forEach((arg) => this.compileExpression(arg, scope, depth)); // Then pop to avoid possible register contention args.forEach((_, i) => this.emit(depth, `POP ${registers[args.length - i - 1]}`), ); this.emit(depth, `MOV RAX, ${SYSCALL_MAP[key]}`); this.emit(depth, 'SYSCALL'); this.emit(depth, `PUSH RAX`); }; }); return wrappers; }

And we're set! Last up is compileIf .

compileIf

This is standard code generation but gets a little tricky due to our stack commitments. Testing must pop the test value off the stack. And then/else blocks must push a value onto the stack (even if there is no else block).

Here is an example we'd like to support:

(if (foo) (do-bar))

We compile the test and branch:

compileIf([test, then, els], scope, depth) { this.emit(depth, '# If'); // Compile test this.compileExpression(test, scope, depth); const branch = `else_branch` + GLOBAL_COUNTER++; // Must pop/use up argument in test this.emit(0, ''); this.emit(depth, `POP RAX`); this.emit(depth, `TEST RAX, RAX`); this.emit(depth, `JZ .${branch}

`); }

Then we compile the then block and jump to after the else block afterward.

compileIf([test, then, els], scope, depth) { this.emit(depth, '# If'); // Compile test this.compileExpression(test, scope, depth); const branch = `else_branch` + GLOBAL_COUNTER++; // Must pop/use up argument in test this.emit(0, ''); this.emit(depth, `POP RAX`); this.emit(depth, `TEST RAX, RAX`); this.emit(depth, `JZ .${branch}

`); // Compile then section this.emit(depth, `# If then`); this.compileExpression(then, scope, depth); this.emit(depth, `JMP .after_${branch}

`); }

Finally we compile the else block if it exists, and otherwise we push a zero onto the stack (possibly to represent null).

compileIf([test, then, els], scope, depth) { this.emit(depth, '# If'); // Compile test this.compileExpression(test, scope, depth); const branch = `else_branch` + GLOBAL_COUNTER++; // Must pop/use up argument in test this.emit(0, ''); this.emit(depth, `POP RAX`); this.emit(depth, `TEST RAX, RAX`); this.emit(depth, `JZ .${branch}

`); // Compile then section this.emit(depth, `# If then`); this.compileExpression(then, scope, depth); this.emit(depth, `JMP .after_${branch}

`); // Compile else section this.emit(depth, `# If else`); this.emit(0, `.${branch}:`); if (els) { this.compileExpression(els, scope, depth); } else { this.emit(1, 'PUSH 0 # Null else branch'); } this.emit(0, `.after_${branch}:`); this.emit(depth, '# End if'); }

And we're ready for an interesting program! Let's print (to stdout) the result of fib(20) .

Fibonacci

$ cat ./tests/fib.lisp (def fib (n) (if (< n 2) n (+ (fib (- n 1)) (fib (- n 2))))) (def main () (print (fib 20)))

And check out the kernel:

$ cat ./lib/kernel.lisp (def print-char (c) (syscall/write 1 &c 1)) (def print (n) (if (> n 9) (print (/ n 10))) (print-char (+ 48 (% n 10))))

Compile and run it:

$ node src/ulisp.js tests/fib.lisp --backend x86 $ ./build/prog 6765

And we're in business!

Please reply on Twitter with questions or comments.