From a9cc26da66bc78a74d3f8a7fc1100b1ea0643adf Mon Sep 17 00:00:00 2001 From: MathMan05 Date: Thu, 15 May 2025 22:27:48 -0500 Subject: [PATCH] some video support --- src/webpage/channel.ts | 71 +++++++++++++- src/webpage/icons/novideo.svg | 1 + src/webpage/icons/video.svg | 1 + src/webpage/style.css | 17 ++++ src/webpage/voice.ts | 172 ++++++++++++++++++++++++++++------ 5 files changed, 232 insertions(+), 30 deletions(-) create mode 100644 src/webpage/icons/novideo.svg create mode 100644 src/webpage/icons/video.svg diff --git a/src/webpage/channel.ts b/src/webpage/channel.ts index c47ab40..633f2c5 100644 --- a/src/webpage/channel.ts +++ b/src/webpage/channel.ts @@ -1078,6 +1078,31 @@ class Channel extends SnowFlake { box.remove(); this.boxMap.delete(user.id); } + boxVid(id: string, elm: HTMLVideoElement) { + //TODO make a loading screen thingy if the video isn't progressing in time yet + const box = this.boxMap.get(id); + if (!box) return; + console.log("vid", elm); + box.append(elm); + } + purgeVid(id: string) { + const box = this.boxMap.get(id); + if (!box) return; + const videos = Array.from(box.getElementsByTagName("video")); + videos.forEach((_) => _.remove()); + } + boxChange(id: string, change: {deaf: boolean; muted: boolean; video: boolean}) { + const box = this.boxMap.get(id); + if (!box || !this.voice) return; + const vid = this.voice.videos.get(id); + if (vid) { + if (change.video) { + this.boxVid(id, vid); + } else { + this.purgeVid(id); + } + } + } async makeUserBox(user: User, users: HTMLElement) { const memb = Member.resolveMember(user, this.guild); const box = document.createElement("div"); @@ -1147,10 +1172,40 @@ class Channel extends SnowFlake { }; call.classList.add("callVoiceIcon"); - buttonRow.append(mute, call); + const updateVideoIcon = () => { + vspan.classList.remove("svg-video", "svg-novideo"); + vspan.classList.add(this.localuser.voiceFactory?.video ? "svg-video" : "svg-novideo"); + }; + const video = document.createElement("div"); + const vspan = document.createElement("span"); + video.append(vspan); + updateVideoIcon(); + video.onclick = async () => { + if (!this.voice) return; + if (this.localuser.voiceFactory?.video) { + this.voice.stopVideo(); + } else { + const cam = await navigator.mediaDevices.getUserMedia({ + video: { + advanced: [ + { + aspectRatio: 1.75, + }, + ], + }, + }); + if (!cam) return; + this.voice.startVideo(cam); + } + updateVideoIcon(); + }; + video.classList.add("callVoiceIcon"); + + buttonRow.append(mute, call, video); const users = document.createElement("div"); users.classList.add("voiceUsers"); + this.voice.userids.forEach(async (_, id) => { const user = await this.localuser.getUser(id); this.makeUserBox(user, users); @@ -1167,7 +1222,21 @@ class Channel extends SnowFlake { }; voiceArea.append(users, buttonRow); + this.voice.onVideo = (vid, id) => { + console.warn("happened"); + this.boxVid(id, vid); + }; + this.voice.onUserChange = (user, change) => { + this.boxChange(user, change); + }; + this.voice.onLeave = () => { + updateCallIcon(); + for (const [id] of this.boxMap) { + this.purgeVid(id); + } + }; } + async getHTML(addstate = true, getMessages: boolean | void = undefined) { if (getMessages === undefined) { getMessages = this.type !== 2 || !this.localuser.voiceAllowed; diff --git a/src/webpage/icons/novideo.svg b/src/webpage/icons/novideo.svg new file mode 100644 index 0000000..4a5e722 --- /dev/null +++ b/src/webpage/icons/novideo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/webpage/icons/video.svg b/src/webpage/icons/video.svg new file mode 100644 index 0000000..f0e4653 --- /dev/null +++ b/src/webpage/icons/video.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/webpage/style.css b/src/webpage/style.css index a005ce4..c171a09 100644 --- a/src/webpage/style.css +++ b/src/webpage/style.css @@ -215,6 +215,13 @@ body { height: 60px; cursor: unset; } + video { + position: absolute; + top: 0px; + left: 0px; + width: 100%; + height: 100%; + } } .buttonRow > * { margin-right: 6px; @@ -246,6 +253,7 @@ body { background: var(--secondary-bg); padding: 4px; border-radius: 6px; + z-index: 1; } .flexgrow { flex-grow: 1; @@ -478,6 +486,15 @@ textarea { display: block; mask-size: cover !important; } +.svg-video { + mask: url(/icons/video.svg); + mask-size: contain !important; +} +.svg-novideo { + mask: url(/icons/novideo.svg); + mask-size: contain !important; + background: var(--red); +} .svg-call { mask: url(/icons/call.svg); mask-size: contain !important; diff --git a/src/webpage/voice.ts b/src/webpage/voice.ts index 958f2ec..542a917 100644 --- a/src/webpage/voice.ts +++ b/src/webpage/voice.ts @@ -33,6 +33,7 @@ class VoiceFactory { onJoin = (_voice: Voice) => {}; onLeave = (_voice: Voice) => {}; private imute = false; + video = false; get mute() { return this.imute; } @@ -69,7 +70,7 @@ class VoiceFactory { channel_id: this.curChan, self_mute: this.imute, self_deaf: false, - self_video: false, + self_video: this.video, flags: 3, }, }); @@ -169,6 +170,7 @@ class Voice { } } users = new Map(); + vidusers = new Map(); readonly speakingMap = new Map(); onSpeakingChange = (_userid: string, _speaking: number) => {}; disconnect(userid: string) { @@ -217,22 +219,47 @@ class Voice { break; case 12: await this.figureRecivers(); - if (!this.users.has(json.d.audio_ssrc)) { + if ( + (!this.users.has(json.d.audio_ssrc) && json.d.audio_ssrc !== 0) || + (!this.vidusers.has(json.d.video_ssrc) && json.d.video_ssrc !== 0) + ) { console.log("redo 12!"); this.makeOp12(); } - if (this.pc) { - this.pc.addTransceiver(json.d.audio_ssrc ? "audio" : "video", { + if (this.pc && json.d.audio_ssrc) { + this.pc.addTransceiver("audio", { direction: "recvonly", sendEncodings: [{active: true}], }); this.getAudioTrans(this.users.size + 1).direction = "recvonly"; + this.users.set(json.d.audio_ssrc, json.d.user_id); } - this.users.set(json.d.audio_ssrc, json.d.user_id); + if (this.pc && json.d.video_ssrc) { + this.pc.addTransceiver("video", { + direction: "recvonly", + sendEncodings: [{active: true}], + }); + this.getVideoTrans(this.vidusers.size + 1).direction = "recvonly"; + this.vidusers.set(json.d.video_ssrc, json.d.user_id); + } + break; } } } + getVideoTrans(id: number) { + if (!this.pc) throw new Error("no pc"); + let i = 0; + for (const thing of this.pc.getTransceivers()) { + if (thing.receiver.track.kind === "video") { + if (id === i) { + return thing; + } + i++; + } + } + throw new Error("none by that id"); + } getAudioTrans(id: number) { if (!this.pc) throw new Error("no pc"); let i = 0; @@ -281,6 +308,7 @@ class Voice { const candidate = (parsed1.atr.get("candidate") as Set).values().next().value as string; const audioUsers = [...this.users]; + const videoUsers = [...this.vidusers]; console.warn(audioUsers); let build = `v=0\r @@ -290,6 +318,7 @@ t=0 0\r a=msid-semantic: WMS *\r a=group:BUNDLE ${bundles.join(" ")}\r`; let ai = -1; + let vi = -1; let i = 0; for (const grouping of parsed.medias) { let mode = "inactive"; @@ -338,13 +367,14 @@ a=extmap:14 urn:ietf:params:rtp-hdrext:toffset\r a=extmap:13 urn:3gpp:video-orientation\r a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay a=setup:passive -a=mid:${bundles[i]}\r -a=${mode}\r +a=mid:${bundles[i]}${videoUsers[vi] && videoUsers[vi][1] ? `\r\na=msid:${videoUsers[vi][1]}-${videoUsers[vi][0]} v${videoUsers[vi][1]}-${videoUsers[vi][0]}\r` : "\r"} +a=${videoUsers[vi] && videoUsers[vi][1] ? "sendonly" : mode}\r a=ice-ufrag:${ICE_UFRAG}\r a=ice-pwd:${ICE_PWD}\r a=fingerprint:${FINGERPRINT}\r -a=candidate:${candidate}\r +a=candidate:${candidate}${videoUsers[vi] && videoUsers[vi][1] ? `\r\na=ssrc:${videoUsers[vi][0]} cname:${videoUsers[vi][1]}-${videoUsers[vi][0]}\r` : "\r"} a=rtcp-mux\r`; + vi++; } i++; } @@ -396,13 +426,7 @@ a=rtcp-mux\r`; console.log("icegatheringstatechange", pc.iceGatheringState, this.pc, this.counter); if (this.pc && this.counter) { if (pc.iceGatheringState === "complete") { - console.log("icegatheringstatechange"); - const counter = this.counter; - const remote: {sdp: string; type: RTCSdpType} = { - sdp: this.cleanServerSDP(counter), - type: "answer", - }; - await pc.setRemoteDescription(remote); + pc.setLocalDescription(); } } }); @@ -415,6 +439,27 @@ a=rtcp-mux\r`; if (sender instanceof Array) { sender = sender[0]; } + let video_ssrc = 0; + let rtx_ssrc = 0; + let max_framerate = 20; + let width = 1280; + let height = 720; + if (this.cam && this.cammera) { + const stats = (await this.cam.sender.getStats()) as Map; + Array.from(stats).forEach((_) => { + if (_[1].ssrc) { + video_ssrc = _[1].ssrc; + } + if (_[1].rtxSsrc) { + rtx_ssrc = _[1].rtxSsrc; + console.log(_); + } + }); + const settings = this.cammera.getSettings(); + console.error(settings); + //width = settings.width || 0; + //height = settings.height || 0; + } if (this.ws) { console.log(this.ssrcMap); this.ws.send( @@ -422,23 +467,19 @@ a=rtcp-mux\r`; op: 12, d: { audio_ssrc: this.ssrcMap.get(sender), - video_ssrc: 0, - rtx_ssrc: 0, + video_ssrc, + rtx_ssrc, streams: [ { type: "video", rid: "100", - ssrc: 0, //TODO - active: false, + ssrc: video_ssrc, + active: !!video_ssrc, quality: 100, - rtx_ssrc: 0, //TODO + rtx_ssrc: rtx_ssrc, max_bitrate: 2500000, //TODO - max_framerate: 0, //TODO - max_resolution: { - type: "fixed", - width: 0, //TODO - height: 0, //TODO - }, + max_framerate, //TODO + max_resolution: {type: "fixed", width, height}, }, ], }, @@ -536,17 +577,74 @@ a=rtcp-mux\r`; } mic?: RTCRtpSender; micTrack?: MediaStreamTrack; + onVideo = (_video: HTMLVideoElement, _id: string) => {}; + videos = new Map(); + cam?: RTCRtpTransceiver; + cammera?: MediaStreamTrack; + stopVideo() { + if (!this.cam) return; + this.owner.video = false; + if (!this.cammera) return; + this.cammera.stop(); + this.cam.sender.replaceTrack(null); + + this.owner.updateSelf(); + this.cammera = undefined; + + this.videos.delete(this.userid); + this.onUserChange(this.userid, { + deaf: false, + muted: this.owner.mute, + video: false, + }); + + this.makeOp12(); + } + async startVideo(caml: MediaStream) { + if (!this.cam) return; + const tracks = caml.getVideoTracks(); + const [cam] = tracks; + + this.owner.video = true; + + this.cammera = cam; + + const video = document.createElement("video"); + this.onVideo(video, this.userid); + this.videos.set(this.userid, video); + video.srcObject = caml; + video.autoplay = true; + await this.cam.sender.replaceTrack(cam); + + //await this.pc?.setLocalDescription(); + await this.makeOp12(); + + this.owner.updateSelf(); + } async startWebRTC() { this.status = "Making offer"; const pc = new RTCPeerConnection(); pc.ontrack = async (e) => { this.status = "Done"; + const media = e.streams[0]; + if (!media) { + console.log(e); + return; + } + const userId = media.id.split("-")[0]; if (e.track.kind === "video") { + console.log(media, this.vidusers); + const video = document.createElement("video"); + this.onVideo(video, userId); + this.videos.set(userId, video); + video.srcObject = media; + + video.autoplay = true; + console.log("gotVideo?"); return; } - const media = e.streams[0]; console.log("got audio:", e); for (const track of media.getTracks()) { console.log(track); @@ -568,6 +666,12 @@ a=rtcp-mux\r`; this.setupMic(audioStream); const sender = pc.addTrack(track); + this.cam = pc.addTransceiver("video", { + direction: "sendonly", + sendEncodings: [ + {active: true, maxBitrate: 2500000, scaleResolutionDownBy: 1, maxFramerate: 20}, + ], + }); this.mic = sender; this.micTrack = track; track.enabled = !this.owner.mute; @@ -763,13 +867,16 @@ a=rtcp-mux\r`; this.status = "waiting for main WS"; } onMemberChange = (_member: memberjson | string, _joined: boolean) => {}; - userids = new Map(); + userids = new Map(); + onUserChange = (_user: string, _change: {deaf: boolean; muted: boolean; video: boolean}) => {}; async voiceupdate(update: voiceStatus) { console.log("Update!"); if (!this.userids.has(update.user_id)) { this.onMemberChange(update?.member || update.user_id, true); } - this.userids.set(update.user_id, {deaf: update.deaf, muted: update.mute}); + const vals = {deaf: update.deaf, muted: update.mute, video: update.self_video}; + this.onUserChange(update.user_id, vals); + this.userids.set(update.user_id, vals); if (update.user_id === this.userid && this.open && !(this.status === "Done")) { if (!update) { @@ -825,10 +932,12 @@ a=rtcp-mux\r`; ); } } + onLeave = () => {}; async leave() { console.warn("leave"); this.open = false; this.status = "Left voice chat"; + this.onLeave(); this.onMemberChange(this.userid, false); this.userids.delete(this.userid); if (this.ws) { @@ -852,6 +961,11 @@ a=rtcp-mux\r`; this.fingerprint = undefined; this.users = new Map(); this.owner.disconect(); + this.vidusers = new Map(); + this.videos = new Map(); + if (this.cammera) this.cammera.stop(); + this.cammera = undefined; + this.cam = undefined; console.log(this); } }