work on metadata system, put tests in seperate file

3 files changed, 581 insertions, 393 deletions

diff --git a/twasm/README.md b/twasm/README.md
index be3f5ad..e6f386e 100644
--- a/twasm/README.md
+++ b/twasm/README.md
@@ -102,6 +102,7 @@ the `type` hex digit is defined as the following:
 | 0x0 | ignored  | `; this entire comment is 1 token` |
 | 0x1 | operator | `mov`, `hlt` |
 | 0x2 | register | `rsp`, `al` |
+| 0xF | unknown  | any token ID not represented in the lookup table |
 
 type metadata for the different types is as follows:
 
diff --git a/twasm/asm/main.asm b/twasm/asm/main.asm
index bc9b333..7128a7f 100644
--- a/twasm/asm/main.asm
+++ b/twasm/asm/main.asm
@@ -15,7 +15,9 @@ OUTPUT_SIZE equ 0x1000     ; max length of outputed binary
 
 STACK_ADDR equ 0x00060000 ; address to put the 64-bit stack at
 
-UNRECOGNISED_TOKEN_ID equ 0xFFFF ; id of an unrecognised token
+UNRECOGNISED_TOKEN_ID equ 0xFFFF  ; id of an unrecognised token
+UNRECOGNISED_ID_TYPE equ 0x0F     ; type of an unrecognised id
+UNRECOGNISED_ID_METADATA equ 0xFF ; metadata of an unrecognised id
 
 TEST_LINE_LENGTH equ 80 ; right border of test suite results
 
@@ -48,6 +50,7 @@ start:
 ; ------------------------------------------------------------------------------
 ; assemble
 ; TODO write tests
+; TODO make it work :/ putting the cart before the horse
 ;
 ; description:
 ; assembles the program from tokens located at TOKEN_TABLE_ADDR into a flat
@@ -64,6 +67,7 @@ assemble:
                                                        ; table
     jg .break                                          ; if overflown, break
 
+    mov rdi, [rax * TOKEN_TABLE_ENTRY_SIZE + TOKEN_TABLE_ADDR] ; rdi -> next
 
     inc rax ; move to next token
     jmp .loop
@@ -71,6 +75,80 @@ assemble:
     ret
 
 ; ------------------------------------------------------------------------------
+; get_tte_type
+;
+; description:
+; given a token table entry, returns the declared type in `tokens.by_id`. If
+; there is no entry, returns UNRECOGNISED_ID_TYPE
+;
+; parameters:
+; di = token table entry
+;
+; returned:
+; al = type of token, or UNRECOGNISED_ID_TYPE. The upper 4 bytes of al are
+;      zeroed; the rest of rax is zeroed.
+; ------------------------------------------------------------------------------
+
+get_tte_type:
+  and rdi, 0xFFFF ; mask input so it behaves as expected
+
+  .loop:
+    cmp rax, (tokens.by_id_end - tokens.by_id) / 4 ; make sure it's still in range
+    jg .not_found
+
+    mov cx, [tokens.by_id + rax * 4] ; next entry in tokens.by_id
+
+    cmp cx, di
+    je .found
+
+    inc rax
+    jmp .loop
+  .not_found:
+    xor rax, rax
+    mov al, UNRECOGNISED_ID_TYPE
+    ret
+  .found:
+    mov al, [2 + tokens.by_id + rax * 4]
+    ret
+
+; ------------------------------------------------------------------------------
+; get_tte_typed_metadata
+;
+; description:
+; given a token table entry, returns the declared typed metadata in
+; `tokens.by_id`. If there is no entry, returns UNRECOGNISED_ID_METADATA
+;
+; parameters:
+; di = token table entry
+;
+; returned:
+; al = typed metadata of token, or UNRECOGNISED_ID_METADATA; the rest of rax is
+;      zeroed.
+; ------------------------------------------------------------------------------
+
+get_tte_typed_metadata:
+  and rdi, 0xFFFF ; mask input so it behaves as expected
+
+  .loop:
+    cmp rax, (tokens.by_id_end - tokens.by_id) / 4 ; make sure it's still in range
+    jg .not_found
+
+    mov cx, [tokens.by_id + rax * 4] ; next entry in tokens.by_id
+
+    cmp cx, di
+    je .found
+
+    inc rax
+    jmp .loop
+  .not_found:
+    xor rax, rax
+    mov al, UNRECOGNISED_ID_METADATA
+    ret
+  .found:
+    mov al, [3 + tokens.by_id + rax * 4]
+    ret
+
+; ------------------------------------------------------------------------------
 ; tokenising
 ; ------------------------------------------------------------------------------
 
@@ -531,382 +609,7 @@ clear_output_arena:
   rep stosd
   ret
 
-; ------------------------------------------------------------------------------
-; tests
-; ------------------------------------------------------------------------------
-
-; ------------------------------------------------------------------------------
-; run_tests
-;
-; description:
-; runs all tests
-; ------------------------------------------------------------------------------
-
-run_tests:
-  mov rsi, .msg
-  call print
-
-  call clear_test_arena
-  call test_copy_byte
-
-  call clear_test_arena
-  call test_copy_token
-
-  call clear_test_arena
-  call test_elemb
-
-  call clear_test_arena
-  call test_identify_token
-
-  call clear_test_arena
-  call test_identify_next_token
-
-  ret
-  .msg db "running test suite...", 0x0A, 0x00
-
-; ------------------------------------------------------------------------------
-; test_copy_byte
-;
-; description:
-; tests copy_byte described functionality
-; ------------------------------------------------------------------------------
-
-test_copy_byte:
-  mov rsi, .msg
-  call print
-
-  mov rdi, test_byte       ; byte to be copied
-  mov rsi, TEST_ARENA_ADDR ; location of test
-  call copy_byte
-
-  mov cx, [rsi]
-  and ax, 0xFF ; only compare bottom byte
-  and cx, 0xFF
-  cmp ax, cx ; compare returned byte to copied byte
-  jne .fail
-  cmp al, [test_byte] ; compare returned byte to expected byte
-  jne .fail
-
-  .pass:
-    mov rsi, msg_pass
-    call print
-    ret
-  .fail:
-    mov rsi, msg_fail
-    call print
-    ret
-  .msg db "test_copy_byte...", 0x00
-
-; ------------------------------------------------------------------------------
-; test_copy_token
-;
-; description:
-; tests copy_token described functionality
-; ------------------------------------------------------------------------------
-
-test_copy_token:
-  mov rsi, .msg
-  call print
-
-  ; test case: space terminated
-
-  mov rdi, test_token_space ; read buffer
-  mov rsi, TEST_ARENA_ADDR  ; write buffer
-  call copy_token
-
-  ; check reported final indicies with the expected final indicies
-  cmp rax, test_token_space + 8 ; last byte read
-  jne .fail
-  cmp rdx, TEST_ARENA_ADDR + 8 ; last byte written
-  jne .fail
-
-  mov rsi, TEST_ARENA_ADDR
-  mov rcx, [rsi]
-  cmp rcx, [test_token_space] ; check if copied token matches expected token
-  jne .fail                   ; if not, fail
-
-  ; test case: null terminated
-
-  mov rdi, test_token_null ; read buffer
-  mov rsi, TEST_ARENA_ADDR ; write buffer
-  call copy_token
-
-  ; check reported final indicies with the expected final indicies
-  cmp rax, test_token_null + 8 ; last byte read
-  jne .fail
-  cmp rdx, TEST_ARENA_ADDR + 8 ; last byte written
-  jne .fail
-
-  mov rsi, TEST_ARENA_ADDR
-  mov rcx, [rsi]
-  cmp rcx, [test_token_null] ; check if copied token matches expected token
-  jne .fail                  ; if not, fail
-
-  .pass:
-    mov rsi, msg_pass
-    call print
-    ret
-  .fail:
-    mov rsi, msg_fail
-    call print
-    ret
-  .msg db "test_copy_token...", 0x00
-
-; ------------------------------------------------------------------------------
-; test_elemb
-;
-; description:
-; tests elemb described functionality
-; ------------------------------------------------------------------------------
-
-test_elemb:
-  mov rsi, .msg
-  call print
-
-  ; [0]
-  mov rdi, 5
-  mov rsi, test_elemb_5
-  mov dl, [test_elemb_5]
-  call elemb
-  cmp al, 1
-  jne .fail
-
-  ; [n - 1]
-  mov rdi, 5
-  mov rsi, test_elemb_5
-  mov dl, [test_elemb_5 + 4]
-  call elemb
-  cmp al, 1
-  jne .fail
-
-  ; [1]
-  mov rdi, 5
-  mov rsi, test_elemb_5
-  mov dl, [test_elemb_5 + 1]
-  call elemb
-  cmp al, 1
-  jne .fail
-
-  ; not present
-  mov rdi, 5
-  mov rsi, test_elemb_5
-  mov dl, 0xDA
-  call elemb
-  cmp al, 0
-  jne .fail
-
-  ; 0 length list
-  mov rdi, 0
-  mov rsi, test_elemb_0
-  mov dl, 0x34
-  call elemb
-  cmp al, 0
-  jne .fail
-
-  .pass:
-    mov rsi, msg_pass
-    call print
-    ret
-  .fail:
-    mov rsi, msg_fail
-    call print
-    ret
-  .msg db "test_elemb...", 0x00
-
-; ------------------------------------------------------------------------------
-; test_identify_token
-;
-; description:
-; tests identify_token described functionality
-; ------------------------------------------------------------------------------
-
-test_identify_token:
-  mov rsi, .msg
-  call print
-
-  ; length1 token that exists
-  mov byte [TEST_ARENA_ADDR], "*"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 1
-  call identify_token
-  cmp ax, 0x0064
-  jne .fail
-
-  ; length1 token that doesn't exist
-  mov byte [TEST_ARENA_ADDR], " "
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 1
-  call identify_token
-  cmp ax, 0xFFFF
-  jne .fail
-
-  ; length2 token that exists
-  mov word [TEST_ARENA_ADDR], "sp"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 2
-  call identify_token
-  cmp ax, 0x0026
-  jne .fail
-
-  ; length2 token that doesn't exist
-  mov word [TEST_ARENA_ADDR], "QQ"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 2
-  call identify_token
-  cmp ax, 0xFFFF
-  jne .fail
-
-  ; length3 token that exists
-  mov dword [TEST_ARENA_ADDR], "rax"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 3
-  call identify_token
-  cmp ax, 0x0000
-  jne .fail
-
-  ; length3 token that exists
-  mov dword [TEST_ARENA_ADDR], "cr0"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 3
-  call identify_token
-  cmp ax, 0x004A
-  jne .fail
-
-  ; length3 token that doesn't exist
-  mov dword [TEST_ARENA_ADDR], "r16"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 3
-  call identify_token
-  cmp ax, 0xFFFF
-  jne .fail
-
-  ; length4 token that exists
-  mov dword [TEST_ARENA_ADDR], "r10d"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 4
-  call identify_token
-  cmp ax, 0x001A
-  jne .fail
-
-  ; length4 token that exists
-  mov dword [TEST_ARENA_ADDR], "r15b"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 4
-  call identify_token
-  cmp ax, 0x003F
-  jne .fail
-
-  ; length4 token that doesn't exist
-  mov dword [TEST_ARENA_ADDR], "r15q"
-  mov rdi, TEST_ARENA_ADDR
-  mov rsi, 4
-  call identify_token
-  cmp ax, 0xFFFF
-  jne .fail
-
-  .pass:
-    mov rsi, msg_pass
-    call print
-    ret
-  .fail:
-    mov rsi, msg_fail
-    call print
-    ret
-  .msg db "test_identify_token...", 0x00
-
-; ------------------------------------------------------------------------------
-; test_identify_next_token
-;
-; description:
-; tests identify_next_token described functionality
-; ------------------------------------------------------------------------------
-
-test_identify_next_token:
-  mov rsi, .msg
-  call print
-
-  ; length1 token that exists
-  mov word [TEST_ARENA_ADDR], "* "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0x0064
-  jne .fail
-
-  ; length1 token that doesn't exist
-  mov word [TEST_ARENA_ADDR], "  "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0xFFFF
-  jne .fail
-
-  ; length2 token that exists
-  mov dword [TEST_ARENA_ADDR], "sp  "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0x0026
-  jne .fail
-
-  ; length2 token that doesn't exist
-  mov dword [TEST_ARENA_ADDR], "QQ  "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0xFFFF
-  jne .fail
-
-  ; length3 token that exists
-  mov dword [TEST_ARENA_ADDR], "rax "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0x0000
-  jne .fail
-
-  ; length3 token that exists
-  mov dword [TEST_ARENA_ADDR], "cr0 "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0x004A
-  jne .fail
-
-  ; length3 token that doesn't exist
-  mov dword [TEST_ARENA_ADDR], "r16 "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0xFFFF
-  jne .fail
-
-  ; length4 token that exists
-  mov dword [TEST_ARENA_ADDR], "r10d"
-  mov byte [TEST_ARENA_ADDR + 4], " "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0x001A
-  jne .fail
-
-  ; length4 token that exists
-  mov dword [TEST_ARENA_ADDR], "r15b"
-  mov byte [TEST_ARENA_ADDR + 4], " "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0x003F
-  jne .fail
-
-  ; length4 token that doesn't exist
-  mov dword [TEST_ARENA_ADDR], "r15q"
-  mov byte [TEST_ARENA_ADDR + 4], " "
-  mov rdi, TEST_ARENA_ADDR
-  call identify_next_token
-  cmp ax, 0xFFFF
-  jne .fail
-
-  .pass:
-    mov rsi, msg_pass
-    call print
-    ret
-  .fail:
-    mov rsi, msg_fail
-    call print
-    ret
-  .msg db "test_identify_next_token...", 0x00
+%include "asm/tests.asm"
 
 ; ------------------------------------------------------------------------------
 ; data
@@ -1123,25 +826,37 @@ tokens:
     dw 0x0059
   .by_name_5:
   .by_id:
+    dw 0x0053 ; xor
+    db 0x01   ; type: operator
+    db 0x02   ; # operands
+
+    dw 0x0010 ; eax
+    db 0x02   ; type: register
+    db 0x02   ; width: 32 bit
+
+    dw 0x0054 ; inc
+    db 0x01   ; type: operator
+    db 0x01   ; # operands
+
+    dw 0x0000 ; rax
+    db 0x02   ; type: register
+    db 0x03   ; width: 64 bit
+
+    dw 0x0056 ; mov
+    db 0x01   ; type: operator
+    db 0x02   ; # operands
+
+    dw 0x0003 ; rdx
+    db 0x02   ; type: register
+    db 0x03   ; width: 64 bit
+
+    dw 0x004F ; hlt
+    db 0x01   ; type: operator
+    db 0x00   ; # operands
+  .by_id_end:
 
 msg_welcome db "Welcome to Twasm", 0x0A, 0x00
 msg_halt db "halted.", 0x0A, 0x00
-msg_pass:
-  db 0x0A
-  times (TEST_LINE_LENGTH + .start - .end) db " ", ; right align
-  .start db "passed."
-  .end db 0x0A, 0x00
-msg_fail:
-  db 0x0A
-  times (TEST_LINE_LENGTH + .start - .end) db " ",
-  .start db "failed."
-  .end db 0x0A, 0x00
-
-test_byte db "Q" ; unterminated, just a byte chillin
-test_token_null db "TestTokn", 0x00 ; followed by null terminator. Quad word
-test_token_space db "TestTokn " ; followed by space. Quad word
-test_elemb_0: ; [This Page Intentionally Left Blank]
-test_elemb_5 db 0x54, 0x00, 0x21, 0x20, 0x34
 
 token_terminator_8 db 0x00, " ", 0x0A, 0x0D, ",", 0x00, 0x00, 0x00
 
diff --git a/twasm/asm/tests.asm b/twasm/asm/tests.asm
new file mode 100644
index 0000000..d5341f0
--- /dev/null
+++ b/twasm/asm/tests.asm
@@ -0,0 +1,472 @@
+; ------------------------------------------------------------------------------
+; tests
+; ------------------------------------------------------------------------------
+
+; ------------------------------------------------------------------------------
+; run_tests
+;
+; description:
+; runs all tests
+; ------------------------------------------------------------------------------
+
+run_tests:
+  mov rsi, .msg
+  call print
+
+  call clear_test_arena
+  call test_copy_byte
+
+  call clear_test_arena
+  call test_copy_token
+
+  call clear_test_arena
+  call test_elemb
+
+  call clear_test_arena
+  call test_identify_token
+
+  call clear_test_arena
+  call test_identify_next_token
+
+  call clear_test_arena
+  call test_get_tte_type
+
+  call clear_test_arena
+  call test_get_tte_typed_metadata
+
+  ret
+  .msg db "running test suite...", 0x0A, 0x00
+
+; ------------------------------------------------------------------------------
+; test_copy_byte
+;
+; description:
+; tests copy_byte described functionality
+; ------------------------------------------------------------------------------
+
+test_copy_byte:
+  mov rsi, .msg
+  call print
+
+  mov rdi, test_byte       ; byte to be copied
+  mov rsi, TEST_ARENA_ADDR ; location of test
+  call copy_byte
+
+  mov cx, [rsi]
+  and ax, 0xFF ; only compare bottom byte
+  and cx, 0xFF
+  cmp ax, cx ; compare returned byte to copied byte
+  jne .fail
+  cmp al, [test_byte] ; compare returned byte to expected byte
+  jne .fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_copy_byte...", 0x00
+
+; ------------------------------------------------------------------------------
+; test_copy_token
+;
+; description:
+; tests copy_token described functionality
+; ------------------------------------------------------------------------------
+
+test_copy_token:
+  mov rsi, .msg
+  call print
+
+  ; test case: space terminated
+
+  mov rdi, test_token_space ; read buffer
+  mov rsi, TEST_ARENA_ADDR  ; write buffer
+  call copy_token
+
+  ; check reported final indicies with the expected final indicies
+  cmp rax, test_token_space + 8 ; last byte read
+  jne .fail
+  cmp rdx, TEST_ARENA_ADDR + 8 ; last byte written
+  jne .fail
+
+  mov rsi, TEST_ARENA_ADDR
+  mov rcx, [rsi]
+  cmp rcx, [test_token_space] ; check if copied token matches expected token
+  jne .fail                   ; if not, fail
+
+  ; test case: null terminated
+
+  mov rdi, test_token_null ; read buffer
+  mov rsi, TEST_ARENA_ADDR ; write buffer
+  call copy_token
+
+  ; check reported final indicies with the expected final indicies
+  cmp rax, test_token_null + 8 ; last byte read
+  jne .fail
+  cmp rdx, TEST_ARENA_ADDR + 8 ; last byte written
+  jne .fail
+
+  mov rsi, TEST_ARENA_ADDR
+  mov rcx, [rsi]
+  cmp rcx, [test_token_null] ; check if copied token matches expected token
+  jne .fail                  ; if not, fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_copy_token...", 0x00
+
+; ------------------------------------------------------------------------------
+; test_elemb
+;
+; description:
+; tests elemb described functionality
+; ------------------------------------------------------------------------------
+
+test_elemb:
+  mov rsi, .msg
+  call print
+
+  ; [0]
+  mov rdi, 5
+  mov rsi, test_elemb_5
+  mov dl, [test_elemb_5]
+  call elemb
+  cmp al, 1
+  jne .fail
+
+  ; [n - 1]
+  mov rdi, 5
+  mov rsi, test_elemb_5
+  mov dl, [test_elemb_5 + 4]
+  call elemb
+  cmp al, 1
+  jne .fail
+
+  ; [1]
+  mov rdi, 5
+  mov rsi, test_elemb_5
+  mov dl, [test_elemb_5 + 1]
+  call elemb
+  cmp al, 1
+  jne .fail
+
+  ; not present
+  mov rdi, 5
+  mov rsi, test_elemb_5
+  mov dl, 0xDA
+  call elemb
+  cmp al, 0
+  jne .fail
+
+  ; 0 length list
+  mov rdi, 0
+  mov rsi, test_elemb_0
+  mov dl, 0x34
+  call elemb
+  cmp al, 0
+  jne .fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_elemb...", 0x00
+
+; ------------------------------------------------------------------------------
+; test_identify_token
+;
+; description:
+; tests identify_token described functionality
+; ------------------------------------------------------------------------------
+
+test_identify_token:
+  mov rsi, .msg
+  call print
+
+  ; length1 token that exists
+  mov byte [TEST_ARENA_ADDR], "*"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 1
+  call identify_token
+  cmp ax, 0x0064
+  jne .fail
+
+  ; length1 token that doesn't exist
+  mov byte [TEST_ARENA_ADDR], " "
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 1
+  call identify_token
+  cmp ax, 0xFFFF
+  jne .fail
+
+  ; length2 token that exists
+  mov word [TEST_ARENA_ADDR], "sp"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 2
+  call identify_token
+  cmp ax, 0x0026
+  jne .fail
+
+  ; length2 token that doesn't exist
+  mov word [TEST_ARENA_ADDR], "QQ"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 2
+  call identify_token
+  cmp ax, 0xFFFF
+  jne .fail
+
+  ; length3 token that exists
+  mov dword [TEST_ARENA_ADDR], "rax"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 3
+  call identify_token
+  cmp ax, 0x0000
+  jne .fail
+
+  ; length3 token that exists
+  mov dword [TEST_ARENA_ADDR], "cr0"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 3
+  call identify_token
+  cmp ax, 0x004A
+  jne .fail
+
+  ; length3 token that doesn't exist
+  mov dword [TEST_ARENA_ADDR], "r16"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 3
+  call identify_token
+  cmp ax, 0xFFFF
+  jne .fail
+
+  ; length4 token that exists
+  mov dword [TEST_ARENA_ADDR], "r10d"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 4
+  call identify_token
+  cmp ax, 0x001A
+  jne .fail
+
+  ; length4 token that exists
+  mov dword [TEST_ARENA_ADDR], "r15b"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 4
+  call identify_token
+  cmp ax, 0x003F
+  jne .fail
+
+  ; length4 token that doesn't exist
+  mov dword [TEST_ARENA_ADDR], "r15q"
+  mov rdi, TEST_ARENA_ADDR
+  mov rsi, 4
+  call identify_token
+  cmp ax, 0xFFFF
+  jne .fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_identify_token...", 0x00
+
+; ------------------------------------------------------------------------------
+; test_identify_next_token
+;
+; description:
+; tests identify_next_token described functionality
+; ------------------------------------------------------------------------------
+
+test_identify_next_token:
+  mov rsi, .msg
+  call print
+
+  ; length1 token that exists
+  mov word [TEST_ARENA_ADDR], "* "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0x0064
+  jne .fail
+
+  ; length1 token that doesn't exist
+  mov word [TEST_ARENA_ADDR], "  "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0xFFFF
+  jne .fail
+
+  ; length2 token that exists
+  mov dword [TEST_ARENA_ADDR], "sp  "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0x0026
+  jne .fail
+
+  ; length2 token that doesn't exist
+  mov dword [TEST_ARENA_ADDR], "QQ  "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0xFFFF
+  jne .fail
+
+  ; length3 token that exists
+  mov dword [TEST_ARENA_ADDR], "rax "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0x0000
+  jne .fail
+
+  ; length3 token that exists
+  mov dword [TEST_ARENA_ADDR], "cr0 "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0x004A
+  jne .fail
+
+  ; length3 token that doesn't exist
+  mov dword [TEST_ARENA_ADDR], "r16 "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0xFFFF
+  jne .fail
+
+  ; length4 token that exists
+  mov dword [TEST_ARENA_ADDR], "r10d"
+  mov byte [TEST_ARENA_ADDR + 4], " "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0x001A
+  jne .fail
+
+  ; length4 token that exists
+  mov dword [TEST_ARENA_ADDR], "r15b"
+  mov byte [TEST_ARENA_ADDR + 4], " "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0x003F
+  jne .fail
+
+  ; length4 token that doesn't exist
+  mov dword [TEST_ARENA_ADDR], "r15q"
+  mov byte [TEST_ARENA_ADDR + 4], " "
+  mov rdi, TEST_ARENA_ADDR
+  call identify_next_token
+  cmp ax, 0xFFFF
+  jne .fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_identify_next_token...", 0x00
+
+
+; ------------------------------------------------------------------------------
+; test_get_tte_type
+;
+; description:
+; tests get_tte_type described functionality
+; ------------------------------------------------------------------------------
+
+test_get_tte_type:
+  mov rsi, .msg
+  call print
+
+  mov di, 0x0053    ; xor
+  call get_tte_type
+  cmp al, 0x01      ; operator
+  jne .fail
+
+  mov di, 0x0003    ; rdx
+  call get_tte_type
+  cmp al, 0x02      ; register
+  jne .fail
+
+  mov di, 0xFFFF    ; unrecognised token
+  call get_tte_type
+  cmp al, UNRECOGNISED_ID_TYPE
+  jne .fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_get_tte_type...", 0x00
+
+; ------------------------------------------------------------------------------
+; test_get_tte_typed_metadata
+;
+; description:
+; tests get_tte_typed_metadata described functionality
+; ------------------------------------------------------------------------------
+
+test_get_tte_typed_metadata:
+  mov rsi, .msg
+  call print
+
+  mov di, 0x0053              ; xor
+  call get_tte_typed_metadata
+  cmp al, 0x02                ; # operands
+  jne .fail
+
+  mov di, 0x0003              ; rdx
+  call get_tte_typed_metadata
+  cmp al, 0x03                ; width: 64 bits
+  jne .fail
+
+  mov di, 0xFFFF              ; unrecognised token
+  call get_tte_typed_metadata
+  cmp al, UNRECOGNISED_ID_METADATA
+  jne .fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_get_tte_type...", 0x00
+
+msg_pass:
+  db 0x0A
+  times (TEST_LINE_LENGTH + .start - .end) db " ", ; right align
+  .start db "passed."
+  .end db 0x0A, 0x00
+msg_fail:
+  db 0x0A
+  times (TEST_LINE_LENGTH + .start - .end) db " ",
+  .start db "failed."
+  .end db 0x0A, 0x00
+
+test_byte db "Q" ; unterminated, just a byte chillin
+test_token_null db "TestTokn", 0x00 ; followed by null terminator. Quad word
+test_token_space db "TestTokn " ; followed by space. Quad word
+test_elemb_0: ; [This Page Intentionally Left Blank]
+test_elemb_5 db 0x54, 0x00, 0x21, 0x20, 0x34