change token table entries, remove temporary file :shame:

This commit is contained in:
andromeda
2026-03-08 11:15:05 +01:00
parent d90c39b6bc
commit f4cadcfff9
3 changed files with 8 additions and 9 deletions

Binary file not shown.

View File

@@ -35,11 +35,11 @@ each word represents a token on the token table.
each token gets loaded into the token table with the following form:
```
+----------+----------+
| 31 16 | 15 0 |
+----------+----------+
| reserved | token id |
+----------+----------+
+----------+
| 15 0 |
+----------+
| token id |
+----------+
```
### token IDs

View File

@@ -5,8 +5,8 @@ TEST_ARENA_SIZE equ 0x1000 ; maximum size tests can use
TOKEN_TABLE_ADDR equ 0x00060000 ; address the token table is loaded at
TOKEN_TABLE_SIZE equ 0x1000 ; max length of table
TOKEN_TABLE_ENTRY_SIZE equ 8 ; size of token table entry; a LOT of things
; may break if this ever changes
TOKEN_TABLE_ENTRY_SIZE equ 2 ; size of token table entry; things may break
; if this ever changes
OUTPUT_ADDR equ 0x00070000 ; address of outputed binary
OUTPUT_SIZE equ 0x1000 ; max length of outputed binary
@@ -84,10 +84,9 @@ tokenise:
mov edx, 1 ; byte length is 1
.continue:
add rdi, rdx ; current byte + length of token = next unread byte
mov [TOKEN_TABLE_ADDR + rcx * TOKEN_TABLE_ENTRY_SIZE], rax ; fill next entry
mov [TOKEN_TABLE_ADDR + rcx * TOKEN_TABLE_ENTRY_SIZE], ax ; fill next entry
; in token table
inc rcx ; +1 token processed
jmp .loop