From 2cbb5aecbf3be0cd08c8996fd180ced1f1d546d7 Mon Sep 17 00:00:00 2001 From: MathMan05 Date: Mon, 19 May 2025 15:57:46 -0500 Subject: [PATCH] start of stream work --- src/webpage/channel.ts | 104 +++++++++++- src/webpage/icons/stream.svg | 1 + src/webpage/jsontypes.ts | 29 +++- src/webpage/localuser.ts | 15 +- src/webpage/style.css | 53 +++++- src/webpage/voice.ts | 320 +++++++++++++++++++++++++---------- translations/en.json | 5 + 7 files changed, 436 insertions(+), 91 deletions(-) create mode 100644 src/webpage/icons/stream.svg diff --git a/src/webpage/channel.ts b/src/webpage/channel.ts index c690666..d9f5e0c 100644 --- a/src/webpage/channel.ts +++ b/src/webpage/channel.ts @@ -1105,11 +1105,17 @@ class Channel extends SnowFlake { return this.last_pin_timestamp !== this.lastpin; } boxMap = new Map(); + liveMap = new Map(); destUserBox(user: User) { const box = this.boxMap.get(user.id); if (!box) return; box.remove(); this.boxMap.delete(user.id); + const live = this.liveMap.get(user.id); + if (live) { + live.remove(); + this.liveMap.delete(user.id); + } } boxVid(id: string, elm: HTMLVideoElement) { //TODO make a loading screen thingy if the video isn't progressing in time yet @@ -1118,14 +1124,44 @@ class Channel extends SnowFlake { console.log("vid", elm); box.append(elm); } + decorateLive(id: string) { + if (!this.voice) return; + const box = this.liveMap.get(id); + if (!box) return; + box.innerHTML = ""; + const live = this.voice.getLive(id); + if (!this.voice.open) { + const span = document.createElement("span"); + span.textContent = I18n.vc.joinForStream(); + box.append(span); + } else if (live) { + const leave = document.createElement("button"); + leave.classList.add("leave"); + leave.textContent = I18n.vc.leavestream(); + leave.onclick = () => { + this.voice?.leaveLive(id); + }; + box.append(live, leave); + } else { + const joinB = document.createElement("button"); + joinB.textContent = I18n.vc.joinstream(); + joinB.classList.add("joinb"); + box.append(joinB); + joinB.onclick = () => { + if (!this.voice) return; + this.voice.joinLive(id); + }; + } + } purgeVid(id: string) { const box = this.boxMap.get(id); if (!box) return; const videos = Array.from(box.getElementsByTagName("video")); videos.forEach((_) => _.remove()); } - boxChange(id: string, change: {deaf: boolean; muted: boolean; video: boolean}) { + boxChange(id: string, change: {deaf: boolean; muted: boolean; video: boolean; live: boolean}) { const box = this.boxMap.get(id); + if (!this.voice) return; if (box) { console.warn("purge:" + id); @@ -1135,7 +1171,33 @@ class Channel extends SnowFlake { } else if (!change.video) { this.purgeVid(id); } + Array.from(box.getElementsByClassName("statBub")).forEach((_) => _.remove()); + const statBub = document.createElement("div"); + statBub.classList.add("statBub"); + if (change.muted) { + const span = document.createElement("span"); + span.classList.add("svg-micmute"); + statBub.append(span); + box.append(statBub); + } else if (change.video) { + const span = document.createElement("span"); + span.classList.add("svg-video"); + statBub.append(span); + box.append(statBub); + } } + + const live = this.liveMap.get(id); + if (live && !change.live) { + live.remove(); + this.liveMap.delete(id); + } else if (!live && change.live && box) { + const livediv = document.createElement("div"); + this.liveMap.set(id, livediv); + box.parentElement?.prepend(livediv); + this.decorateLive(id); + } + const tray = this.voiceTray.get(id); if (tray) { console.warn("tray build", tray, change); @@ -1180,6 +1242,10 @@ class Channel extends SnowFlake { span.classList.add("voiceUsername"); box.append(span); users.append(box); + if (!this.voice) return; + const change = this.voice.userids.get(user.id); + if (!change) return; + this.boxChange(user.id, change); } usersDiv = new WeakRef(document.createElement("div")); async setUpVoiceArea() { @@ -1250,7 +1316,22 @@ class Channel extends SnowFlake { }; video.classList.add("callVoiceIcon"); - buttonRow.append(mute, call, video); + const updateLiveIcon = () => { + lspan.classList.remove("svg-video", "svg-novideo"); + lspan.classList.add(true ? "svg-stream" : "svg-stopstream"); + }; + const live = document.createElement("div"); + const lspan = document.createElement("span"); + live.append(lspan); + updateLiveIcon(); + live.onclick = async () => { + const stream = await navigator.mediaDevices.getDisplayMedia(); + this.voice?.createLive(this.localuser.user.id, stream); + updateLiveIcon(); + }; + live.classList.add("callVoiceIcon"); + + buttonRow.append(mute, call, video, live); const users = document.createElement("div"); users.classList.add("voiceUsers"); @@ -1259,6 +1340,9 @@ class Channel extends SnowFlake { const user = await this.localuser.getUser(id); this.makeUserBox(user, users); }); + [...this.liveMap].forEach(([_, box]) => { + users.prepend(box); + }); this.usersDiv = new WeakRef(users); voiceArea.append(users, buttonRow); @@ -1266,12 +1350,28 @@ class Channel extends SnowFlake { console.warn("happened"); this.boxVid(id, vid); }; + this.voice.onGotStream = (_vid, id) => { + this.decorateLive(id); + }; + this.voice.onconnect = () => { + if (!this.voice) return; + for (const [_, user] of this.voice.users) { + this.decorateLive(user); + } + }; + this.voice.onLeaveStream = (id) => { + this.decorateLive(id); + }; this.voice.onLeave = () => { updateCallIcon(); for (const [id] of this.boxMap) { this.purgeVid(id); } + if (!this.voice) return; + for (const [_, user] of this.voice.users) { + this.decorateLive(user); + } }; } diff --git a/src/webpage/icons/stream.svg b/src/webpage/icons/stream.svg new file mode 100644 index 0000000..aacd09d --- /dev/null +++ b/src/webpage/icons/stream.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/webpage/jsontypes.ts b/src/webpage/jsontypes.ts index 0f8f6d0..da2915c 100644 --- a/src/webpage/jsontypes.ts +++ b/src/webpage/jsontypes.ts @@ -726,7 +726,9 @@ type wsjson = stickers: stickerJson[]; }; s: 3; - }; + } + | streamServerUpdate + | streamCreate; type memberChunk = { guild_id: string; @@ -748,8 +750,32 @@ export type voiceStatus = { self_deaf: boolean; self_mute: boolean; self_video: boolean; + self_stream: boolean; suppress: boolean; }; +export interface streamCreate { + op: 0; + t: "STREAM_CREATE"; + d: { + stream_key: string; + rtc_server_id: string; + viewer_ids: string[]; + region: "spacebar"; + paused: boolean; + }; + s: number; +} +export interface streamServerUpdate { + op: 0; + t: "STREAM_SERVER_UPDATE"; + d: { + token: string; + stream_key: string; + guild_id: null; //There is no way this ain't a server bug lol + endpoint: string; + }; + s: number; +} type voiceupdate = { op: 0; t: "VOICE_STATE_UPDATE"; @@ -836,6 +862,7 @@ type webRTCSocket = ssrc: 940464811; }; }; + type sdpback = { op: 4; d: { diff --git a/src/webpage/localuser.ts b/src/webpage/localuser.ts index 4766bb2..69f5a76 100644 --- a/src/webpage/localuser.ts +++ b/src/webpage/localuser.ts @@ -773,6 +773,18 @@ class Localuser { } break; + case "STREAM_SERVER_UPDATE": { + if (this.voiceFactory) { + this.voiceFactory.streamServerUpdate(temp); + } + break; + } + case "STREAM_CREATE": { + if (this.voiceFactory) { + this.voiceFactory.streamCreate(temp); + } + break; + } case "VOICE_SERVER_UPDATE": if (this.voiceFactory) { this.voiceFactory.voiceServerUpdate(temp); @@ -857,8 +869,9 @@ class Localuser { guild.onStickerUpdate(guild.stickers); break; } + default: { - //@ts-ignore + //@ts-expect-error console.warn("Unhandled case " + temp.t, temp); } } diff --git a/src/webpage/style.css b/src/webpage/style.css index b92baef..6380fa9 100644 --- a/src/webpage/style.css +++ b/src/webpage/style.css @@ -200,14 +200,35 @@ body { .speaking { outline: 3px solid var(--green); } +.voiceUsers > :hover .leave { + bottom: 10px; + opacity: 1; +} +.leave { + position: absolute; + bottom: 0px; + right: 10px; + background: var(--red); + opacity: 0; + transition: + bottom 0.4s, + opacity 0.2s, + background 0.1s; +} +.leave:hover { + background: color-mix(in srgb, var(--red) 85%, white); +} .voiceUsers > * { background: var(--accent_color, var(--primary-bg)); - padding: 80px 140px; - width: fit-content; border-radius: 8px; position: relative; box-sizing: border-box; margin: 8px; + width: 340px; + height: 220px; + display: flex; + justify-content: center; + align-items: center; img { width: 60px; @@ -340,6 +361,14 @@ iframe { audio::-webkit-media-controls-panel { background: var(--secondary-bg); } +.joinb { + background: var(--green); + border-radius: 200px; + transition: background 0.2s; +} +.joinb:hover { + background: color-mix(in srgb, var(--green) 80%, transparent); +} button, input::file-selector-button, select { @@ -494,6 +523,10 @@ textarea { display: block; mask-size: cover !important; } +.svg-stream { + mask: url(/icons/stream.svg); + mask-size: contain !important; +} .svg-video { mask: url(/icons/video.svg); mask-size: contain !important; @@ -1235,6 +1268,22 @@ span.instanceStatus { .hiddencat { rotate: -90deg; } +.statBub { + position: absolute; + bottom: 10px; + right: 10px; + background: color-mix(in srgb, var(--secondary-bg) 75%, transparent); + border-radius: 50%; + z-index: 1; + + * { + background: var(--primary-text); + width: 16px; + height: 16px; + display: block; + margin: 5px; + } +} .addchannel { height: 10px; width: 20px; diff --git a/src/webpage/voice.ts b/src/webpage/voice.ts index 2131470..572a5bb 100644 --- a/src/webpage/voice.ts +++ b/src/webpage/voice.ts @@ -1,4 +1,12 @@ -import {memberjson, sdpback, voiceserverupdate, voiceStatus, webRTCSocket} from "./jsontypes.js"; +import { + memberjson, + sdpback, + streamCreate, + streamServerUpdate, + voiceserverupdate, + voiceStatus, + webRTCSocket, +} from "./jsontypes.js"; class VoiceFactory { settings: {id: string}; handleGateway: (obj: Object) => void; @@ -34,6 +42,7 @@ class VoiceFactory { onLeave = (_voice: Voice) => {}; private imute = false; video = false; + stream = false; get mute() { return this.imute; } @@ -96,12 +105,94 @@ class VoiceFactory { channel_id: channelId, self_mute, self_deaf: false, //todo - self_video: false, //What is this? I have some guesses + self_video: false, flags: 2, //????? }, op: 4, }; } + live = new Map void>(); + steamTokens = new Map>(); + steamTokensRes = new Map void>(); + async joinLive(userid: string) { + const stream_key = `${this.curGuild === "@me" ? "call" : `guild:${this.curGuild}`}:${this.curChan}:${userid}`; + this.handleGateway({ + op: 20, + d: { + stream_key, + }, + }); + return new Promise(async (res) => { + this.live.set(stream_key, res); + this.steamTokens.set( + stream_key, + new Promise<[string, string]>((res) => { + this.steamTokensRes.set(stream_key, res); + }), + ); + }); + } + islive = false; + liveStream?: MediaStream; + async createLive(userid: string, stream: MediaStream) { + this.islive = true; + this.liveStream = stream; + const stream_key = `${this.curGuild === "@me" ? "call" : `guild:${this.curGuild}`}:${this.curChan}:${userid}`; + this.handleGateway({ + op: 18, + d: { + type: this.curGuild === "@me" ? "call" : "guild", + guild_id: this.curGuild === "@me" ? null : this.curGuild, + channel_id: this.curChan, + preferred_region: null, + }, + }); + return new Promise(async (res) => { + this.live.set(stream_key, res); + this.steamTokens.set( + stream_key, + new Promise<[string, string]>((res) => { + this.steamTokensRes.set(stream_key, res); + }), + ); + }); + } + async streamCreate(create: streamCreate) { + const prom1 = this.steamTokens.get(create.d.stream_key); + if (!prom1) throw new Error("oops"); + const [token, endpoint] = await prom1; + if (create.d.stream_key.startsWith("guild")) { + const [_, _guild, chan, user] = create.d.stream_key.split(":"); + const voice2 = this.voiceChannels.get(chan); + if (!voice2 || !voice2.session_id) throw new Error("oops"); + let stream: undefined | MediaStream = undefined; + console.error(user, this.settings.id); + if (user === this.settings.id) { + stream = this.liveStream; + } + const voice = new Voice( + this.settings.id, + { + bitrate: 10000, + stream: true, + live: stream, + }, + { + url: endpoint, + token, + }, + this, + ); + voice.join(); + voice.startWS(voice2.session_id, create.d.rtc_server_id); + + voice2.gotStream(voice, user); + } + } + streamServerUpdate(update: streamServerUpdate) { + const res = this.steamTokensRes.get(update.d.stream_key); + if (res) res([update.d.token, update.d.endpoint]); + } userMap = new Map(); voiceStateUpdate(update: voiceStatus) { const prev = this.userMap.get(update.user_id); @@ -144,8 +235,8 @@ class Voice { return this.pstatus; } readonly userid: string; - settings: {bitrate: number}; - urlobj: {url?: string; token?: string; geturl: Promise; gotUrl: () => void}; + settings: {bitrate: number; stream?: boolean; live?: MediaStream}; + urlobj: {url?: string; token?: string; geturl?: Promise; gotUrl?: () => void}; owner: VoiceFactory; constructor( userid: string, @@ -194,6 +285,7 @@ class Voice { //there's more for sure, but this is "good enough" for now this.onMemberChange(userid, false); } + async packet(message: MessageEvent) { const data = message.data; if (typeof data === "string") { @@ -223,7 +315,6 @@ class Voice { (!this.users.has(json.d.audio_ssrc) && json.d.audio_ssrc !== 0) || (!this.vidusers.has(json.d.video_ssrc) && json.d.video_ssrc !== 0) ) { - this.sendtosend12 = false; console.log("redo 12!"); this.makeOp12(); } @@ -323,10 +414,8 @@ a=group:BUNDLE ${bundles.join(" ")}\r`; let i = 0; for (const grouping of parsed.medias) { let mode = "inactive"; - for (const _ of this.senders) { - if (i < 2) { - mode = "sendrecv"; - } + if (i < 2) { + mode = "sendonly"; } if (grouping.media === "audio") { build += ` @@ -380,6 +469,7 @@ a=rtcp-mux\r`; i++; } build += "\n"; + console.log(build); return build; } counter?: string; @@ -437,7 +527,7 @@ a=rtcp-mux\r`; sender: RTCRtpSender | undefined | [RTCRtpSender, number] = this.ssrcMap.entries().next().value, ) { if (!this.ws) return; - if (!sender) throw new Error("sender doesn't exist"); + if (!sender) return; if (sender instanceof Array) { sender = sender[0]; } @@ -601,12 +691,43 @@ a=rtcp-mux\r`; deaf: false, muted: this.owner.mute, video: false, + live: this.owner.stream, }); } + liveMap = new Map(); + private voiceMap = new Map(); + getLive(id: string) { + return this.liveMap.get(id); + } + joinLive(id: string) { + return this.owner.joinLive(id); + } + createLive(id: string, stream: MediaStream) { + return this.owner.createLive(id, stream); + } + leaveLive(id: string) { + const v = this.voiceMap.get(id); + if (!v) return; + v.leave(); + this.voiceMap.delete(id); + this.liveMap.delete(id); + this.onLeaveStream(id); + } + onLeaveStream = (_user: string) => {}; + onGotStream = (_v: HTMLVideoElement, _user: string) => {}; + gotStream(voice: Voice, user: string) { + voice.onVideo = (video) => { + this.liveMap.set(user, video); + this.onGotStream(video, user); + }; + this.voiceMap.set(user, voice); + } videoStarted = false; - async startVideo(caml: MediaStream) { + async startVideo(caml: MediaStream, early = false) { console.warn("test test test test video sent!"); - if (!this.cam) return; + while (!this.cam) { + await new Promise((res) => setTimeout(res, 100)); + } const tracks = caml.getVideoTracks(); const [cam] = tracks; @@ -621,16 +742,20 @@ a=rtcp-mux\r`; video.autoplay = true; this.cam.direction = "sendonly"; const sender = this.cam.sender; - await sender.replaceTrack(cam); - this.pc?.setLocalDescription(); + if (!early) { + await sender.replaceTrack(cam); + this.pc?.setLocalDescription(); - this.owner.updateSelf(); + this.owner.updateSelf(); + } } + onconnect = () => {}; async startWebRTC() { this.status = "Making offer"; const pc = new RTCPeerConnection(); pc.ontrack = async (e) => { this.status = "Done"; + this.onconnect(); const media = e.streams[0]; if (!media) { console.log(e); @@ -667,36 +792,47 @@ a=rtcp-mux\r`; }; const audioStream = await navigator.mediaDevices.getUserMedia({video: false, audio: true}); const [track] = audioStream.getAudioTracks(); - //Add track - - this.setupMic(audioStream); - const sender = pc.addTrack(track); - this.cam = pc.addTransceiver("video", { - direction: "sendonly", - sendEncodings: [ - {active: true, maxBitrate: 2500000, scaleResolutionDownBy: 1, maxFramerate: 20}, - ], - }); - this.mic = sender; - this.micTrack = track; - track.enabled = !this.owner.mute; - this.senders.add(sender); - console.log(sender); - - for (let i = 0; i < 10; i++) { + if (!this.settings.stream) { + this.setupMic(audioStream); + const sender = pc.addTrack(track); + this.cam = pc.addTransceiver("video", { + direction: "sendonly", + sendEncodings: [ + {active: true, maxBitrate: 2500000, scaleResolutionDownBy: 1, maxFramerate: 20}, + ], + }); + this.mic = sender; + this.micTrack = track; + track.enabled = !this.owner.mute; + this.senders.add(sender); + console.log(sender); + } + const count = this.settings.stream ? 1 : 10; + for (let i = 0; i < count; i++) { pc.addTransceiver("audio", { direction: "inactive", streams: [], sendEncodings: [{active: true, maxBitrate: this.settings.bitrate}], }); } - for (let i = 0; i < 10; i++) { - pc.addTransceiver("video", { - direction: "inactive", - streams: [], - sendEncodings: [{active: true, maxBitrate: this.settings.bitrate}], + if (this.settings.live) { + this.cam = pc.addTransceiver("video", { + direction: "sendonly", + sendEncodings: [ + {active: true, maxBitrate: 2500000, scaleResolutionDownBy: 1, maxFramerate: 20}, + ], }); + this.startVideo(this.settings.live, true); + } else { + for (let i = 0; i < count; i++) { + pc.addTransceiver("video", { + direction: "inactive", + streams: [], + sendEncodings: [{active: true, maxBitrate: this.settings.bitrate}], + }); + } } + this.pc = pc; this.negotationneeded(); await new Promise((res) => setTimeout(res, 100)); @@ -873,8 +1009,11 @@ a=rtcp-mux\r`; this.status = "waiting for main WS"; } onMemberChange = (_member: memberjson | string, _joined: boolean) => {}; - userids = new Map(); - onUserChange = (_user: string, _change: {deaf: boolean; muted: boolean; video: boolean}) => {}; + userids = new Map(); + onUserChange = ( + _user: string, + _change: {deaf: boolean; muted: boolean; video: boolean; live: boolean}, + ) => {}; async voiceupdate(update: voiceStatus) { console.log("Update!"); if (!this.userids.has(update.user_id)) { @@ -884,6 +1023,7 @@ a=rtcp-mux\r`; deaf: update.deaf, muted: update.mute || update.self_mute, video: update.self_video, + live: update.self_stream, }; this.onUserChange(update.user_id, vals); this.userids.set(update.user_id, vals); @@ -896,54 +1036,59 @@ a=rtcp-mux\r`; this.status = "bad responce from WS"; return; } - if (!this.urlobj.url) { - this.status = "waiting for Voice URL"; - await this.urlobj.geturl; - if (!this.open) { - this.leave(); - return; - } - } - - const ws = new WebSocket(("ws://" + this.urlobj.url) as string); - this.ws = ws; - ws.onclose = () => { - this.leave(); - }; - this.status = "waiting for WS to open"; - ws.addEventListener("message", (m) => { - this.packet(m); - }); - await new Promise((res) => { - ws.addEventListener("open", () => { - res(); - }); - }); - if (!this.ws) { + this.session_id = update.session_id; + await this.startWS(update.session_id, update.guild_id); + } + } + session_id?: string; + async startWS(session_id: string, server_id: string) { + if (!this.urlobj.url) { + this.status = "waiting for Voice URL"; + await this.urlobj.geturl; + if (!this.open) { this.leave(); return; } - this.status = "waiting for WS to authorize"; - ws.send( - JSON.stringify({ - op: 0, - d: { - server_id: update.guild_id, - user_id: update.user_id, - session_id: update.session_id, - token: this.urlobj.token, - video: false, - streams: [ - { - type: "video", - rid: "100", - quality: 100, - }, - ], - }, - }), - ); } + + const ws = new WebSocket(("ws://" + this.urlobj.url) as string); + this.ws = ws; + ws.onclose = () => { + this.leave(); + }; + this.status = "waiting for WS to open"; + ws.addEventListener("message", (m) => { + this.packet(m); + }); + await new Promise((res) => { + ws.addEventListener("open", () => { + res(); + }); + }); + if (!this.ws) { + this.leave(); + return; + } + this.status = "waiting for WS to authorize"; + ws.send( + JSON.stringify({ + op: 0, + d: { + server_id, + user_id: this.userid, + session_id, + token: this.urlobj.token, + video: false, + streams: [ + { + type: "video", + rid: "100", + quality: 100, + }, + ], + }, + }), + ); } onLeave = () => {}; async leave() { @@ -951,7 +1096,12 @@ a=rtcp-mux\r`; this.open = false; this.status = "Left voice chat"; this.onLeave(); - this.onMemberChange(this.userid, false); + for (const thing of this.liveMap) { + this.leaveLive(thing[0]); + } + if (!this.settings.stream) { + this.onMemberChange(this.userid, false); + } this.userids.delete(this.userid); if (this.ws) { this.ws.close(); @@ -972,7 +1122,7 @@ a=rtcp-mux\r`; this.ssrcMap = new Map(); this.fingerprint = undefined; this.users = new Map(); - this.owner.disconect(); + if (!this.settings.stream) this.owner.disconect(); this.vidusers = new Map(); this.videos = new Map(); if (this.cammera) this.cammera.stop(); diff --git a/translations/en.json b/translations/en.json index 54f15a2..3ebe7c5 100644 --- a/translations/en.json +++ b/translations/en.json @@ -5,6 +5,11 @@ "locale": "en", "comment": "Don't know how often I'll update this top part lol" }, + "vc": { + "joinstream": "Watch stream", + "leavestream": "Leave stream", + "joinForStream": "Join the VC to watch" + }, "readableName": "English", "pinMessage": "Pin Message", "unableToPin": "Unable to pin message",