progressive loading inital support

src/webpage/media.ts

@@ -1,6 +1,7 @@
 import {Contextmenu} from "./contextmenu.js";
 import {I18n} from "./i18n.js";
 import {Dialog} from "./settings.js";
+import {ProgressiveArray} from "./utils/progessiveLoad.js";
 const menu = new Contextmenu<media, undefined>("media");
 menu.addButton(
 	() => I18n.media.download(),
@@ -312,60 +313,31 @@ class MediaPlayer {
 		}
 		let resMedio = (_: media) => {};
 		this.cache.set(url, new Promise<media>((res) => (resMedio = res)));
-		const controller = new AbortController();
+		const prog = new ProgressiveArray(url, {method: "get"});
+		await prog.ready;
 
-		const f = await fetch(url, {
-			method: "get",
-			signal: controller.signal,
-		});
-		if (!f.ok || !f.body) {
-			return null;
-		}
-
-		let index = 0;
-		const read = f.body.getReader();
-		let cbuff = (await read.read()).value;
 		const output: Partial<media> = {
 			src: url,
 		};
 		try {
-			let sizeLeft = 0;
+			const head = String.fromCharCode(await prog.next(), await prog.next(), await prog.next());
-			async function next() {
-				return (await get8BitArray(1))[0];
-			}
-			async function get8BitArray(size: number) {
-				sizeLeft -= size;
-				const arr = new Uint8Array(size);
-				let arri = 0;
-				while (size > 0) {
-					if (!cbuff) throw Error("ran out of file to read");
-					let itter = Math.min(size, cbuff.length - index);
-					size -= itter;
-					for (let i = 0; i < itter; i++, arri++, index++) {
-						arr[arri] = cbuff[index];
-					}
-
-					if (size !== 0) {
-						cbuff = (await read.read()).value;
-						index = 0;
-					}
-				}
-				return arr;
-			}
-			const head = String.fromCharCode(await next(), await next(), await next());
 			if (head === "ID3") {
-				const version = (await next()) + (await next()) * 256;
+				const version = (await prog.next()) + (await prog.next()) * 256;
 
 				if (version === 2) {
 					//TODO I'm like 90% I can ignore *all* of the flags, but I need to check more sometime
-					await next();
+					await prog.next();
 					//debugger;
-					const sizes = await get8BitArray(4);
+					const sizes = await prog.get8BitArray(4);
-					sizeLeft = (sizes[0] << 21) + (sizes[1] << 14) + (sizes[2] << 7) + sizes[3];
+					prog.sizeLeft = (sizes[0] << 21) + (sizes[1] << 14) + (sizes[2] << 7) + sizes[3];
 					const mappy = new Map<string, Uint8Array>();
-					while (sizeLeft > 0) {
+					while (prog.sizeLeft > 0) {
-						const Identify = String.fromCharCode(await next(), await next(), await next());
+						const Identify = String.fromCharCode(
-						const sizeArr = await get8BitArray(3);
+							await prog.next(),
+							await prog.next(),
+							await prog.next(),
+						);
+						const sizeArr = await prog.get8BitArray(3);
 						const size = (sizeArr[0] << 16) + (sizeArr[1] << 8) + sizeArr[2];
 						if (Identify === String.fromCharCode(0, 0, 0)) {
 							break;
@@ -378,10 +350,10 @@ class MediaPlayer {
 							break;
 						}
 						if (mappy.has(Identify)) {
-							await get8BitArray(size);
+							await prog.get8BitArray(size);
 							//console.warn("Got dupe", Identify);
 						} else {
-							mappy.set(Identify, await get8BitArray(size));
+							mappy.set(Identify, await prog.get8BitArray(size));
 						}
 					}
 					const pic = mappy.get("PIC");
@@ -429,25 +401,25 @@ class MediaPlayer {
 					}
 					//TODO more thoroughly check if these two are the same format
 				} else if (version === 3 || version === 4) {
-					const flags = await next();
+					const flags = await prog.next();
 					if (flags & 0b01000000) {
 						//TODO deal with the extended header
 					}
 					//debugger;
-					const sizes = await get8BitArray(4);
+					const sizes = await prog.get8BitArray(4);
-					sizeLeft = (sizes[0] << 21) + (sizes[1] << 14) + (sizes[2] << 7) + sizes[3];
+					prog.sizeLeft = (sizes[0] << 21) + (sizes[1] << 14) + (sizes[2] << 7) + sizes[3];
 					const mappy = new Map<string, Uint8Array>();
-					while (sizeLeft > 0) {
+					while (prog.sizeLeft > 0) {
 						const Identify = String.fromCharCode(
-							await next(),
+							await prog.next(),
-							await next(),
+							await prog.next(),
-							await next(),
+							await prog.next(),
-							await next(),
+							await prog.next(),
 						);
-						const sizeArr = await get8BitArray(4);
+						const sizeArr = await prog.get8BitArray(4);
 						const size = (sizeArr[0] << 24) + (sizeArr[1] << 16) + (sizeArr[2] << 8) + sizeArr[3];
 
-						const flags = await get8BitArray(2);
+						const flags = await prog.get8BitArray(2);
 						const compression = !!(flags[1] & 0b10000000);
 						if (compression) {
 							//TODO Honestly, I don't know if I can do this with normal JS
@@ -470,10 +442,10 @@ class MediaPlayer {
 							break;
 						}
 						if (mappy.has(Identify)) {
-							await get8BitArray(size);
+							await prog.get8BitArray(size);
 							//console.warn("Got dupe", Identify);
 						} else {
-							mappy.set(Identify, await get8BitArray(size));
+							mappy.set(Identify, await prog.get8BitArray(size));
 						}
 					}
 					const pic = mappy.get("APIC");
@@ -532,7 +504,7 @@ class MediaPlayer {
 			console.error(e);
 		} finally {
 			output.filename = url.split("/").at(-1);
-			controller.abort();
+			prog.close();
 			if (!output.length) {
 				output.length = new Promise<number>(async (res) => {
 					const audio = document.createElement("audio");

src/webpage/utils/progessiveLoad.ts

@@ -0,0 +1,309 @@
+export class ProgressiveArray {
+	read?: ReadableStreamDefaultReader<Uint8Array>;
+	controller: AbortController;
+	cbuff? = new Uint8Array(0);
+	index = 0;
+	sizeLeft = 0;
+	ready: Promise<void>;
+	constructor(url: string, req: RequestInit = {}) {
+		this.controller = new AbortController();
+		this.ready = fetch(url, {
+			...req,
+			signal: this.controller.signal,
+		}).then(async (f) => {
+			if (!f.ok || !f.body) {
+				throw new Error("request not ok");
+			}
+			const read = f.body.getReader();
+			this.cbuff = (await read.read()).value;
+			this.read = read;
+		});
+	}
+	async next() {
+		return (await this.get8BitArray(1))[0];
+	}
+	async get8BitArray(size: number) {
+		if (!this.read) throw new Error("not ready to read");
+		this.sizeLeft -= size;
+		const arr = new Uint8Array(size);
+		let arri = 0;
+		while (size > 0) {
+			if (!this.cbuff) throw Error("ran out of file to read");
+			let itter = Math.min(size, this.cbuff.length - this.index);
+			size -= itter;
+			for (let i = 0; i < itter; i++, arri++, this.index++) {
+				arr[arri] = this.cbuff[this.index];
+			}
+
+			if (size !== 0) {
+				this.cbuff = (await this.read.read()).value;
+				this.index = 0;
+			}
+		}
+		return arr;
+	}
+	decoder = new TextDecoder();
+	backChar?: string;
+	async getChar() {
+		if (this.backChar) {
+			const temp = this.backChar;
+			delete this.backChar;
+			return temp;
+		}
+		let char = "";
+		while (!char) {
+			char = this.decoder.decode((await this.get8BitArray(1)).buffer, {stream: true});
+		}
+		return char;
+	}
+	putBackChar(char: string) {
+		this.backChar = char;
+	}
+	close() {
+		this.controller.abort();
+	}
+}
+
+async function getNextNonWhiteSpace(prog: ProgressiveArray) {
+	let char = " ";
+	const whiteSpace = new Set("\n\t \r");
+	while (whiteSpace.has(char)) {
+		char = await prog.getChar();
+	}
+	return char;
+}
+async function identifyType(prog: ProgressiveArray) {
+	let char = await getNextNonWhiteSpace(prog);
+	switch (char) {
+		case "-":
+		case "0":
+		case "1":
+		case "2":
+		case "3":
+		case "4":
+		case "5":
+		case "6":
+		case "7":
+		case "8":
+		case "9": {
+			const validNumber = new Set("0123456789e.+-");
+			let build = "";
+			do {
+				build += char;
+				char = await prog.getChar();
+			} while (validNumber.has(char));
+			prog.putBackChar(char);
+			return Number(build);
+		}
+		case '"':
+			let build = "";
+			do {
+				build += char;
+				if (char == "\\") {
+					char = await prog.getChar();
+					build += char;
+				}
+				char = await prog.getChar();
+			} while (char !== '"');
+			build += char;
+			return JSON.parse(build) as string;
+		case "t":
+		case "f":
+		case "n": {
+			let build = char;
+			while (build.match(/(^tr?u?$)|(^fa?l?s?$)|(^nu?l?$)/)) {
+				char = await prog.getChar();
+				build += char;
+			}
+			return JSON.parse(build) as boolean | null;
+		}
+		case "[":
+			return await ArrayProgressive.make(prog);
+		case "{":
+			return await ObjectProgressive.make(prog);
+		default:
+			throw new Error("bad JSON");
+	}
+}
+class ArrayProgressive<T, X extends Array<T>> {
+	ondone = async () => {};
+	prog: ProgressiveArray;
+	done = false;
+	private constructor(prog: ProgressiveArray) {
+		this.prog = prog;
+	}
+	static async make(prog: ProgressiveArray) {
+		const o = new ArrayProgressive(prog);
+		await o.check();
+		return o;
+	}
+	async check() {
+		const lastChar = await getNextNonWhiteSpace(this.prog);
+		if (lastChar === "]") {
+			this.done = true;
+			await this.ondone();
+			return;
+		} else {
+			this.prog.putBackChar(lastChar);
+		}
+	}
+	awaiting = new Promise<void>((_) => _());
+
+	async doChecks(): Promise<() => void> {
+		let res1: () => void;
+		let cur = new Promise<void>((res) => {
+			res1 = res;
+		});
+		[cur, this.awaiting] = [this.awaiting, cur];
+		await cur;
+
+		return () => res1();
+	}
+	async getNext(): Promise<Progressive<T>> {
+		const checks = await this.doChecks();
+		if (this.done) throw new Error("no more array");
+		const ret = (await identifyType(this.prog)) as Progressive<T>;
+		const check = async () => {
+			const lastChar = await getNextNonWhiteSpace(this.prog);
+			if (lastChar === "]") {
+				this.done = true;
+				await this.ondone();
+			} else if (lastChar !== ",") throw Error("Bad JSON Object:" + lastChar);
+			checks();
+		};
+		if ((ret instanceof ArrayProgressive || ret instanceof ObjectProgressive) && !ret.done) {
+			ret.ondone = check;
+		} else {
+			await check();
+		}
+
+		return ret;
+	}
+	/**
+	 * this only gets what's left, not everything
+	 */
+	async getWhole(): Promise<X> {
+		const arr: T[] = [];
+		while (!this.done) {
+			let t = await this.getNext();
+			if (t instanceof ArrayProgressive) {
+				t = await t.getWhole();
+			}
+			if (t instanceof ObjectProgressive) {
+				t = await t.getWhole();
+			}
+			arr.push(t as T);
+		}
+
+		return arr as X;
+	}
+}
+class ObjectProgressive<X extends Object> {
+	ondone = async () => {};
+	prog: ProgressiveArray;
+	done = false;
+	private constructor(prog: ProgressiveArray) {
+		this.prog = prog;
+	}
+	static async make(prog: ProgressiveArray) {
+		const o = new ObjectProgressive(prog);
+		await o.check();
+		return o;
+	}
+	async check() {
+		const lastChar = await getNextNonWhiteSpace(this.prog);
+		if (lastChar === "}") {
+			this.done = true;
+			await this.ondone();
+			return;
+		} else {
+			this.prog.putBackChar(lastChar);
+		}
+	}
+	awaiting = new Promise<void>((_) => _());
+
+	async doChecks(): Promise<() => void> {
+		let res1: () => void;
+		let cur = new Promise<void>((res) => {
+			res1 = res;
+		});
+		[cur, this.awaiting] = [this.awaiting, cur];
+		await cur;
+		return () => res1();
+	}
+	async getNextPair(): Promise<{[K in keyof X]: {key: K; value: Progressive<X[K]>}}[keyof X]> {
+		const checks = await this.doChecks();
+		if (this.done) throw new Error("no more object");
+		const key = (await identifyType(this.prog)) as unknown;
+		if (typeof key !== "string") {
+			throw Error("Bad key:" + key);
+		}
+		const nextChar = await getNextNonWhiteSpace(this.prog);
+		if (nextChar !== ":") throw Error("Bad JSON");
+		const value = (await identifyType(this.prog)) as unknown;
+		const check = async () => {
+			const lastChar = await getNextNonWhiteSpace(this.prog);
+			if (lastChar === "}") {
+				this.done = true;
+				await this.ondone();
+			} else if (lastChar !== ",") throw Error("Bad JSON Object:" + lastChar);
+			checks();
+		};
+		if ((value instanceof ArrayProgressive || value instanceof ObjectProgressive) && !value.done) {
+			value.ondone = check;
+		} else {
+			await check();
+		}
+		return {key, value} as any;
+	}
+	/**
+	 * this only gets what's left, not everything
+	 */
+	async getWhole(): Promise<X> {
+		const obj: Partial<X> = {};
+		while (!this.done) {
+			let {key, value} = await this.getNextPair();
+			if (value instanceof ArrayProgressive) {
+				value = await value.getWhole();
+			}
+			if (value instanceof ObjectProgressive) {
+				value = await value.getWhole();
+			}
+			obj[key] = value as any;
+		}
+		return obj as X;
+	}
+}
+Object.entries;
+type Progressive<T> =
+	T extends Array<any>
+		? ArrayProgressive<T extends Array<infer X> ? X : never, T>
+		: T extends string
+			? T
+			: T extends boolean
+				? T
+				: T extends null
+					? T
+					: T extends number
+						? T
+						: T extends Object
+							? ObjectProgressive<T>
+							: T;
+/*
+ * this will progressively load a JSON object, you must read everything you get to get the next thing in line.
+ */
+export async function ProgessiveDecodeJSON<X>(
+	url: string,
+	req: RequestInit = {},
+): Promise<Progressive<X>> {
+	const prog = new ProgressiveArray(url, req);
+	await prog.ready;
+	return identifyType(prog) as Promise<Progressive<X>>;
+}
+const test = [1, 2, 3, 4, 5, 6];
+const blob = new Blob([JSON.stringify(test)]);
+ProgessiveDecodeJSON<typeof test>(blob)
+	.then(async (obj) => {
+		console.log(await obj.getWhole()); //returns the ping object
+	})
+	.then(console.warn);