add elemb, improve token checking

README.md

@@ -2,50 +2,18 @@ Call me Terry Davis because... actually please don't. I have visions: aspiration
 
 # bootle
 
-hobby kernel written in rust. It's just for playing around... for now :p
+hobby kernel
 
-run with `nix run git+https://git.mtgmonkey.net/andromeda/bootler#bootle`
+status: basically nothing, come back later
 
 # bootler
 
-hobby bootloader written in asm. It's just for playing around... for now :p
+hobby 1-stage legacy mode bootloader
 
-run with `nix run git+https://git.mtgmonkey.net/andromeda/bootler#bootler`
+status: gets to long mode, loads+jumps to kernel, starts idt and gdt... :)
 
-### memory map
+# twasm
 
-```
+hobby self-hosted assembler
-+------ 0x00100000 ------+
+
-| hardware, bios stuff   |
+status: basically nothing, come back later
-+------ 0x00080000 ------+
-|                        |
-|                        |
-+------ 0x00010200 ------+
-| x86_64 kernel          |
-+------ 0x00010000 ------+
-|                        |
-|                        |
-+------ 0x00009000 ------+
-| IDT                    |
-+------ 0x00008000 ------+
-|                        |
-|                        |
-+------ 0x00007E00 ------+
-| bootloader (boot.asm)  |
-+------ 0x00007C00 ------+
-| stack                  | TODO get real stack
-+------ 0x00005000 ------+
-| PT                     |
-+------ 0x00004000 ------+
-| PDT                    |
-+------ 0x00003000 ------+
-| PDPT                   |
-+------ 0x00002000 ------+
-| PML4T                  |
-+------ 0x00001000 ------+
-|                        |
-|                        |
-+------ 0x00000500 ------+
-| bios stuff             |
-+------ 0x00000000 ------+
-```

bootle/README.md

@@ -0,0 +1,28 @@
+# bootle
+
+hobby kernel written in rust. It's just for playing around... for now :p
+
+run with `nix run git+https://git.mtgmonkey.net/andromeda/bootler#bootle`
+
+### memory map
+
+```
++------ 0x00100000 ------+
+| hardware, bios stuff   |
++------ 0x00080000 ------+
+|                        |
+|                        |
++------ 0x00010200 ------+
+| kernel                 |
++------ 0x00010000 ------+
+| bootloader stuff       |
+| includes stack, gdt,   |
+| idt for the time being |
++------ 0x00000500 ------+
+| bios stuff             |
++------ 0x00000000 ------+
+```
+
+---
+
+this project follows [Common Changelog](https://common-changelog.org) guidelines

bootle/package.nix

@@ -6,27 +6,28 @@
   qemu,
   ...
 }: let
-  rust-toolchain = rust-bin.fromRustupToolchainFile ../../rust-toolchain.toml;
+  rust-toolchain = rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
   naersk' = callPackage naersk {
     cargo = rust-toolchain;
     rustc = rust-toolchain;
     clippy = rust-toolchain;
   };
 in (naersk'.buildPackage {
-  src = ../../.;
+  src = ./.;
 
   # deps for rust-src
   additionalCargoLock = "${rust-toolchain.availableComponents.rust-src}/lib/rustlib/src/rust/library/Cargo.lock";
 
   # just library build
   copyBins = false;
+  copyLibs = true;
   release = true;
 
   # build std
   cargoBuildOptions = x:
     x
     ++ [
-      "-Zbuild-std=core,compiler_builtins"
+      "-Zbuild-std"
     ];
 
   postInstall = ''
@@ -36,7 +37,7 @@ in (naersk'.buildPackage {
       -e _start \
       target/x86_64-unknown-none/release/libbootle.a
 
-    dd if=/dev/zero of=disk bs=512 count=2
+    dd if=/dev/zero of=disk bs=512 count=2880
     dd if=${bootler}/bin/boot.bin of=disk conv=notrunc
     dd if=kernel.bin of=disk bs=512 seek=1 conv=notrunc
 

bootle/src/lib.rs

@@ -12,11 +12,13 @@ pub extern "C" fn _start() -> ! {
 fn print_serial(s: &str) {
     let mut bytes = s.bytes();
     while let Some(b) = bytes.next() {
-        unsafe {core::arch::asm!(
+        unsafe {
-            "out dx, al"
+            core::arch::asm!(
-            , in("al") b
+                "out dx, al"
-        )};
+                , in("al") b
-    };
+            )
+        };
+    }
 }
 
 fn println_serial(s: &str) {
@@ -28,7 +30,11 @@ fn welcome_serial() {
     print_serial(ANSI_PINK);
     println_serial("\nWelcome to Bootle OS");
     println_serial("All code GPL licensed and freely available on git.mtgmonkey.net");
-    print_serial("Enjoy your time! Press "); print_serial(ANSI_RED); print_serial("ctrl+a x"); print_serial(ANSI_PINK); println_serial(" to escape Qemu");
+    print_serial("Enjoy your time! Press ");
+    print_serial(ANSI_RED);
+    print_serial("ctrl+a x");
+    print_serial(ANSI_PINK);
+    println_serial(" to escape Qemu");
     print_serial(ANSI_CLEAR);
 }
 
@@ -39,9 +45,7 @@ fn panic(_: &PanicInfo) -> ! {
 }
 
 fn halt() -> ! {
-    unsafe {core::arch::asm!(
+    unsafe { core::arch::asm!("hlt") };
-        "hlt"
-    )};
     halt()
 }
 

bootler/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Changelog
+
+## [0.1.0] - 2026-03-06
+
+### Added
+
+- initialised CHANGELOG.md

bootler/README.md

@@ -0,0 +1,43 @@
+# bootler
+
+hobby bootloader, 1-stage, legacy mode :p
+
+run with `nix run git+https://git.mtgmonkey.net/andromeda/bootler#bootler`
+
+### memory map
+
+```
+.                        .
+:                        :
+| longmode kernel        |
++------ 0x00010000 ------+
+|                        |
+|                        |
++------ 0x00009000 ------+
+| IDT                    |
++------ 0x00008000 ------+
+|                        |
+|                        |
++------ 0x00007E00 ------+
+| bootloader (boot.asm)  |
++------ 0x00007C00 ------+
+| stack                  |
++------ 0x00005000 ------+
+| PT                     |
++------ 0x00004000 ------+
+| PDT                    |
++------ 0x00003000 ------+
+| PDPT                   |
++------ 0x00002000 ------+
+| PML4T                  |
++------ 0x00001000 ------+
+|                        |
+|                        |
++------ 0x00000500 ------+
+| bios stuff             |
++------ 0x00000000 ------+
+```
+
+---
+
+this project follows [Common Changelog](https://common-changelog.org) guidelines

bootler/asm/boot.asm

@@ -6,7 +6,7 @@
 LOAD_ADDR equ 0x7C00
 
 KERNEL_START equ 2             ; first sector on disk to load kernel from; 1 indexed
-KERNEL_SIZE equ 1              ; length of kernel in sectors
+KERNEL_SIZE equ 16             ; length of kernel in sectors
 KERNEL_LOAD_ADDR_ES equ 0x1000 ; kernel to be loaded at es * 0x10 + 0x0000
 
 PAGE_TABLE_LOAD_ADDR equ 0x1000 ; start of page table; 4 * pt size 

bootler/package.nix

@@ -9,19 +9,18 @@ in
   stdenv.mkDerivation {
     pname = "bootler";
     version = "0.1.0";
-    src = ../../.;
+    src = ./.;
     buildPhase = ''
       ${nasm}/bin/nasm asm/boot.asm -o boot.bin
       ${nasm}/bin/nasm asm/kernel.asm -o dummy.bin
 
-      dd if=/dev/zero of=${bootImg} bs=512 count=2
+      dd if=/dev/zero of=${bootImg} bs=512 count=2880
       dd if=boot.bin of=${bootImg} conv=notrunc
       dd if=dummy.bin of=${bootImg} bs=512 seek=1 conv=notrunc
     '';
     installPhase = ''
       mkdir -p $out/bin
       cp boot.bin $out/bin
-      cp dummy.bin $out/bin
       cp ${bootImg} $out/bin
 
       # create emulation binary

flake.nix

@@ -18,20 +18,25 @@
     ...
   }: let
     system = "x86_64-linux";
-    pkgs = import nixpkgs {
+    pkgs = nixpkgs.legacyPackages.${system};
-      inherit system;
+    pkgsWithRustOverlay = pkgs.extend (import rust-overlay);
-      overlays = [(import rust-overlay)];
-    };
   in {
     packages.${system} = {
-      bootler = pkgs.callPackage ./nix/pkgs/bootler.nix {};
+      bootler = pkgs.callPackage ./bootler/package.nix {};
-      bootle = pkgs.callPackage ./nix/pkgs/bootle.nix {
+      bootle = pkgsWithRustOverlay.callPackage ./bootle/package.nix {
-        naersk = naersk;
+        inherit naersk;
+        bootler = self.packages.${system}.bootler;
+      };
+      twasm = pkgs.callPackage ./twasm/package.nix {
         bootler = self.packages.${system}.bootler;
       };
     };
     devShells.${system}.default = pkgs.mkShell {
-      inputsFrom = [self.packages.${system}.default];
+      inputsFrom = [
+        self.packages.${system}.bootle
+        self.packages.${system}.bootler
+        self.packages.${system}.twasm
+      ];
     };
   };
 }

nix/package.nix

@@ -1,47 +0,0 @@
-{
-  stdenv,
-  nasm,
-  qemu,
-  ...
-}: let
-  bootImg = "boot";
-in
-  stdenv.mkDerivation {
-    pname = "bootler";
-    version = "0.1.0";
-    src = ../.;
-    buildPhase = ''
-      ${nasm}/bin/nasm asm/boot.asm -o boot.bin
-      ${nasm}/bin/nasm asm/kernel.asm -o kernel.bin
-
-      dd if=/dev/zero of=${bootImg} bs=512 count=2
-      dd if=boot.bin of=${bootImg} conv=notrunc
-      dd if=kernel.bin of=${bootImg} bs=512 seek=1 conv=notrunc
-    '';
-    installPhase = ''
-      mkdir -p $out/bin
-      cp ${bootImg} $out/bin
-
-      # create emulation binary
-      cat<<EOF>$out/bin/bootler
-      #!/usr/bin/env bash
-
-      # create temp dir
-      mkdir -p ./.bootler
-      cp $(echo $out)/bin/${bootImg} ./.bootler/${bootImg}
-      chmod a+w ./.bootler/${bootImg}
-
-      # run image
-      ${qemu}/bin/qemu-system-x86_64 \
-        -nographic \
-        -drive file=./.bootler/${bootImg},format=raw,index=0,media=disk
-
-      # clean up
-      rm ./.bootler -r
-
-      EOF
-
-      chmod +x $out/bin/${bootImg}
-      chmod +x $out/bin/bootler
-    '';
-  }

twasm/README.md

@@ -0,0 +1,32 @@
+# twasm
+
+this will be a self hosted, very minimal subset of 64 bit asm
+
+### goals
+
+I want to compile Bootler and Twasm with the Twasm assembler
+
+### memory map
+
+```
++------ 0x00100000 ------+
+| hardware, bios stuff   |
++------ 0x00080000 ------+
+| output binary          |
++------ 0x00070000 ------+
+| token table            |
++------ 0x00060000 ------+
+|                        |
+|                        |
++------------------------+
+| input                  |
++------------------------+ this is lined up to a sector
+|                        | and this is less than a sector
++------------------------+
+| assembler              |
++------ 0x00010000 ------+
+| bootloader, bios, etc. |
++------------------------+
+```
+
+each word represents a token on the token table.

twasm/asm/main.asm

@@ -0,0 +1,369 @@
+LOAD_ADDR equ 0x00010000 ; address this program is loaded at
+
+TOKEN_TABLE_ADDR equ 0x00060000 ; address the token table is loaded at
+TOKEN_TABLE_SIZE equ 0x1000     ; max length of table
+
+TEST_ARENA_ADDR equ 0x00060000 ; address to run tests at
+TEST_ARENA_SIZE equ 0x1000     ; maximum size tests can use
+
+OUTPUT_ADDR equ 0x00070000 ; address of outputed binary
+OUTPUT_SIZE equ 0x1000     ; max length of outputed binary
+
+STACK_ADDR equ 0x00060000 ; address to put the 64-bit stack at
+
+[bits 64]
+[org LOAD_ADDR]
+
+start:
+  mov rsp, STACK_ADDR ; we might need more stack space, let's just be safe
+
+  mov rsi, msg_welcome
+  call print
+
+  call run_tests
+
+  jmp halt
+
+; ------------------------------------------------------------------------------
+; tokenising
+; ------------------------------------------------------------------------------
+
+; ------------------------------------------------------------------------------
+; copy_token 
+;
+; description:
+; copies a token from one spot in memory to another
+;
+; parameters:
+; rdi -> start of buffer to be read
+; rsi -> start of buffer to be written
+;
+; returned:
+; rax -> last byte read
+; rdx -> last byte written
+; ------------------------------------------------------------------------------
+
+copy_token:
+  .loop:
+
+    mov dl, [rdi] ; move bit to compare to current byte in read buffer
+
+    push rdi ; push incrementors to call elemb
+    push rsi ;
+
+    mov rdi, 8                  ; length of terminator list
+    mov rsi, token_terminator_8 ; start of terminator list
+                                ; dl set before pushing rdi
+    call elemb
+
+    pop rsi ;
+    pop rdi ; pop incrementors after call
+
+    cmp rax, 1 ; check if the next character is a token terminator
+    je .break  ; > if so, break the function
+
+    ; rdi and rsi set from previous loop iteration
+    call copy_byte ; if not, copy the current byte in read buffer
+
+    inc rdi ; read pointer
+    inc rsi ; write pointer
+
+    jmp .loop
+
+  .break:
+    mov rax, rdi ; -> last byte read
+    mov rdx, rsi ; -> last byte written
+    ret
+    
+; ------------------------------------------------------------------------------
+; copy_byte
+;
+; description:
+; copies a byte from one spot in memory to another
+;
+; parameters:
+; rdi -> word to be read
+; rsi -> word to be written
+;
+; returned:
+; al = byte that was read; the rest of rax is zeroed
+; ------------------------------------------------------------------------------
+
+copy_byte:
+  xor rax, rax ; zero out so it returns fine
+  mov al, [rdi]
+  mov [rsi], al
+  ret
+
+; ------------------------------------------------------------------------------
+; utilities
+; ------------------------------------------------------------------------------
+
+; ------------------------------------------------------------------------------
+; print
+;
+; description:
+; prints a null-terminated string
+;
+; parameters:
+; rsi -> start of null-terminated string
+; ------------------------------------------------------------------------------
+
+print:
+  push rdx
+  mov rdx, 0x3F8
+  .loop:
+    mov al, [rsi]
+    test al, al
+    jz .done
+    out dx, al
+    inc rsi
+    jmp .loop
+  .done:
+    pop rdx
+    ret
+
+; ------------------------------------------------------------------------------
+; halt
+;
+; description:
+; halts the program, silly :)
+; ------------------------------------------------------------------------------
+
+halt:
+  mov rsi, msg_halt
+  call print
+  hlt
+  jmp halt
+
+; ------------------------------------------------------------------------------
+; elemb
+;
+; description:
+; checks if given byte is element of the specified list
+;
+; parameters:
+; rdi = size of list
+; rsi -> start of list
+; dl =  given byte
+;
+; returned:
+; rax = 0: is not an element
+;       1: is an element
+; ------------------------------------------------------------------------------
+
+elemb:
+  .loop
+    cmp rdi, 0    ; check if remaining length 0
+    je .not_found ; if so, break; dl not an element of list
+
+    mov al, [rsi]
+    cmp al, dl    ; check if current byte in list is the desired byte
+    je .found     ; if so, break; dl an element of list
+
+    inc rsi ; move to next byte
+    dec rdi ; and reduce remaining length
+
+    jmp .loop
+
+  .not_found
+    xor rax, rax ; return 0; dl not an element of list
+    ret
+
+  .found
+    xor rax, rax
+    mov rax, 1 ; return 1; dl an element of list
+    ret
+
+  .f db "found", 0x0D, 0x0A, 0x00
+  .nf db "not found", 0x0D, 0x0A, 0x00
+
+; ------------------------------------------------------------------------------
+; tests
+; ------------------------------------------------------------------------------
+
+; ------------------------------------------------------------------------------
+; run_tests
+;
+; description:
+; runs all tests
+; ------------------------------------------------------------------------------
+
+run_tests:
+  mov rsi, .msg
+  call print
+
+  call test_copy_byte
+  call test_copy_token
+  call test_elemb
+
+  ret
+  .msg db "running test suite...", 0x0D, 0x0A, 0x00
+
+; ------------------------------------------------------------------------------
+; test_copy_byte
+;
+; description:
+; tests copy_byte described functionality
+; ------------------------------------------------------------------------------
+
+test_copy_byte:
+  mov rsi, .msg
+  call print
+
+  mov rdi, test_byte       ; byte to be copied
+  mov rsi, TEST_ARENA_ADDR ; location of test
+  call copy_byte
+
+  mov cx, [rsi]
+  and ax, 0xFF ; only compare bottom byte
+  and cx, 0xFF
+  cmp ax, cx ; compare returned byte to copied byte
+  jne .fail
+  cmp al, [test_byte] ; compare returned byte to expected byte
+  jne .fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_copy_byte...", 0x00
+
+; ------------------------------------------------------------------------------
+; test_copy_token
+;
+; description:
+; tests copy_token described functionality
+; ------------------------------------------------------------------------------
+
+test_copy_token:
+  mov rsi, .msg
+  call print
+
+  ; test case: space terminated
+
+  mov rdi, test_token_space ; read buffer
+  mov rsi, TEST_ARENA_ADDR  ; write buffer
+  call copy_token
+
+  ; check reported final indicies with the expected final indicies
+  cmp rax, test_token_space + 8 ; last byte read
+  jne .fail
+  cmp rdx, TEST_ARENA_ADDR + 8 ; last byte written
+  jne .fail
+
+  mov rsi, TEST_ARENA_ADDR
+  mov rcx, [rsi]
+  cmp rcx, [test_token_space] ; check if copied token matches expected token
+  jne .fail                   ; if not, fail
+
+  ; test case: null terminated
+
+  mov rdi, test_token_null ; read buffer
+  mov rsi, TEST_ARENA_ADDR ; write buffer
+  call copy_token
+
+  ; check reported final indicies with the expected final indicies
+  cmp rax, test_token_null + 8 ; last byte read
+  jne .fail
+  cmp rdx, TEST_ARENA_ADDR + 8 ; last byte written
+  jne .fail
+
+  mov rsi, TEST_ARENA_ADDR
+  mov rcx, [rsi]
+  cmp rcx, [test_token_null] ; check if copied token matches expected token
+  jne .fail                  ; if not, fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_copy_token...", 0x00
+
+; ------------------------------------------------------------------------------
+; test_elemb
+;
+; description:
+; tests elemb described functionality
+; ------------------------------------------------------------------------------
+
+test_elemb:
+  mov rsi, .msg
+  call print
+
+  ; [0]
+  mov rdi, 5
+  mov rsi, test_elemb_5
+  mov dl, [test_elemb_5]
+  call elemb
+  cmp al, 1
+  jne .fail
+
+  ; [n - 1]
+  mov rdi, 5
+  mov rsi, test_elemb_5
+  mov dl, [test_elemb_5 + 4]
+  call elemb
+  cmp al, 1
+  jne .fail
+
+  ; [1]
+  mov rdi, 5
+  mov rsi, test_elemb_5
+  mov dl, [test_elemb_5 + 1]
+  call elemb
+  cmp al, 1
+  jne .fail
+
+  ; not present
+  mov rdi, 5
+  mov rsi, test_elemb_5
+  mov dl, 0xDA
+  call elemb
+  cmp al, 0
+  jne .fail
+
+  ; 0 length list
+  mov rdi, 0
+  mov rsi, test_elemb_0
+  mov dl, 0x34
+  call elemb
+  cmp al, 0
+  jne .fail
+
+  .pass:
+    mov rsi, msg_pass
+    call print
+    ret
+  .fail:
+    mov rsi, msg_fail
+    call print
+    ret
+  .msg db "test_elemb...", 0x00
+
+; ------------------------------------------------------------------------------
+; data
+; ------------------------------------------------------------------------------
+
+msg_welcome db "Welcome to Twasm", 0x0D, 0x0A, 0x00
+msg_halt db "halted.", 0x0D, 0x0A, 0x00
+msg_pass db "passed.", 0x0D, 0x0A, 0x00
+msg_fail db "failed.", 0x0D, 0x0A, 0x00
+
+test_byte db "Q" ; unterminated, just a byte chillin
+test_token_null db "TestTokn", 0x00 ; followed by null terminator. Quad word
+test_token_space db "TestTokn " ; followed by space. Quad word
+test_elemb_0
+test_elemb_5 db 0x54, 0x00, 0x21, 0x20, 0x34
+
+token_terminator_8 db 0x00, " ", 0x0A, 0x0D, 0x00, 0x00, 0x00, 0x00
+
+debug_string db "debug_string", 0x0D, 0x0A, 0x00

twasm/package.nix

@@ -0,0 +1,42 @@
+{
+  bootler,
+  stdenv,
+  nasm,
+  qemu,
+  ...
+}:
+stdenv.mkDerivation {
+  pname = "twasm";
+  version = "0.1.0";
+  src = ./.;
+  buildPhase = ''
+    ${nasm}/bin/nasm asm/main.asm -o out.bin
+  '';
+  installPhase = ''
+    dd if=/dev/zero of=disk bs=512 count=2880
+    dd if=${bootler}/bin/boot.bin of=disk conv=notrunc
+    dd if=out.bin of=disk bs=512 seek=1 conv=notrunc
+
+    mkdir -p $out/bin
+
+    cat<<EOF>$out/bin/twasm
+    #!/usr/bin/env bash
+
+    mkdir -p ./.bootle
+    cp $(echo $out)/bin/disk ./.bootle/disk
+    chmod a+w ./.bootle/disk
+
+    ${qemu}/bin/qemu-system-x86_64 \
+      -nographic \
+      -s \
+      -drive file=./.bootle/disk,format=raw,index=0,media=disk
+
+    rm ./.bootle -r
+    EOF
+
+    chmod +x $out/bin/twasm
+
+    cp out.bin $out/bin
+    cp disk $out/bin
+  '';
+}