From 5cdb4d2184d6e3a45897c5e4f8743565a51f2065 Mon Sep 17 00:00:00 2001 From: MathMan05 Date: Fri, 11 Oct 2024 22:43:39 -0500 Subject: [PATCH] more VC work --- src/webpage/channel.ts | 59 ++++++++-- src/webpage/index.html | 25 ++-- src/webpage/localuser.ts | 64 ++++++----- src/webpage/style.css | 14 +++ src/webpage/user.ts | 2 +- src/webpage/voice.ts | 242 +++++++++++++++++++++++++++------------ 6 files changed, 279 insertions(+), 127 deletions(-) diff --git a/src/webpage/channel.ts b/src/webpage/channel.ts index 4804ff8..7d2fe0f 100644 --- a/src/webpage/channel.ts +++ b/src/webpage/channel.ts @@ -14,6 +14,7 @@ import{channeljson,embedjson,messageCreateJson,messagejson,readyjson,startTyping import{ MarkDown }from"./markdown.js"; import{ Member }from"./member.js"; import { Voice } from "./voice.js"; +import { User } from "./user.js"; declare global { interface NotificationOptions { @@ -332,8 +333,9 @@ class Channel extends SnowFlake{ } this.setUpInfiniteScroller(); this.perminfo ??= {}; - if(this.type===2){ - this.voice=new Voice(this); + if(this.type===2&&this.localuser.voiceFactory){ + this.voice=this.localuser.voiceFactory.makeVoice(this.guild.id,this.id,{bitrate:this.bitrate}); + this.setUpVoice(); } } get perminfo(){ @@ -456,6 +458,7 @@ class Channel extends SnowFlake{ get visable(){ return this.hasPermission("VIEW_CHANNEL"); } + voiceUsers=new WeakRef(document.createElement("div")); createguildHTML(admin = false): HTMLDivElement{ const div = document.createElement("div"); this.html = new WeakRef(div); @@ -562,6 +565,7 @@ class Channel extends SnowFlake{ const decoration = document.createElement("span"); div.appendChild(decoration); decoration.classList.add("space", "svgtheme", "svg-voice"); + }else if(this.type === 5){ // const decoration = document.createElement("span"); @@ -574,9 +578,51 @@ class Channel extends SnowFlake{ div.onclick = _=>{ this.getHTML(); }; + if(this.type===2){ + const voiceUsers=document.createElement("div"); + div.append(voiceUsers); + this.voiceUsers=new WeakRef(voiceUsers); + this.updateVoiceUsers(); + } } return div; } + async setUpVoice(){ + if(!this.voice) return; + this.voice.onMemberChange=async (memb,joined)=>{ + console.log(memb,joined); + if(typeof memb!=="string"){ + await Member.new(memb,this.guild); + } + this.updateVoiceUsers(); + } + } + async updateVoiceUsers(){ + const voiceUsers=this.voiceUsers.deref(); + if(!voiceUsers||!this.voice) return; + console.warn(this.voice.userids) + + const html=(await Promise.all(this.voice.userids.entries().toArray().map(async _=>{ + const user=await User.resolve(_[0],this.localuser); + console.log(user); + const member=await Member.resolveMember(user,this.guild); + const array=[member,_[1]] as [Member, typeof _[1]]; + return array; + }))).flatMap(([member,_obj])=>{ + if(!member){ + console.warn("This is weird, member doesn't exist :P"); + return []; + } + const div=document.createElement("div"); + const span=document.createElement("span"); + span.textContent=member.name; + div.append(span); + return div; + }); + + voiceUsers.innerHTML=""; + voiceUsers.append(...html); + } get myhtml(){ if(this.html){ return this.html.deref(); @@ -840,8 +886,7 @@ class Channel extends SnowFlake{ this.rendertyping(); this.localuser.getSidePannel(); if(this.voice){ - this.voice.onSatusChange=console.warn; - this.voice.join(); + this.localuser.joinVoice(this); } await this.putmessages(); await prom; @@ -1329,11 +1374,7 @@ class Channel extends SnowFlake{ } notititle(message: Message): string{ return( - message.author.username + - " > " + - this.guild.properties.name + - " > " + - this.name + message.author.username + " > " + this.guild.properties.name + " > " + this.name ); } notify(message: Message, deep = 0){ diff --git a/src/webpage/index.html b/src/webpage/index.html index e7b6b89..016b98a 100644 --- a/src/webpage/index.html +++ b/src/webpage/index.html @@ -35,18 +35,23 @@

Server Name

-
-
- - -
-

USERNAME

-

STATUS

-
+
+
+
+
+
+ -
- +
+

USERNAME

+

STATUS

+
+
+ +
+ +
diff --git a/src/webpage/localuser.ts b/src/webpage/localuser.ts index b9235b6..68fb87e 100644 --- a/src/webpage/localuser.ts +++ b/src/webpage/localuser.ts @@ -5,13 +5,13 @@ import{ AVoice }from"./audio.js"; import{ User }from"./user.js"; import{ Dialog }from"./dialog.js"; import{ getapiurls, getBulkInfo, setTheme, Specialuser }from"./login.js"; -import{channeljson,guildjson,mainuserjson,memberjson,memberlistupdatejson,messageCreateJson,presencejson,readyjson,startTypingjson,voiceupdate,wsjson,}from"./jsontypes.js"; +import{channeljson,guildjson,mainuserjson,memberjson,memberlistupdatejson,messageCreateJson,presencejson,readyjson,startTypingjson,wsjson,}from"./jsontypes.js"; import{ Member }from"./member.js"; import{ Form, FormError, Options, Settings }from"./settings.js"; import{ MarkDown }from"./markdown.js"; import { Bot } from "./bot.js"; import { Role } from "./role.js"; -import { Voice } from "./voice.js"; +import { VoiceFactory } from "./voice.js"; const wsCodesRetry = new Set([4000, 4003, 4005, 4007, 4008, 4009]); @@ -42,6 +42,7 @@ class Localuser{ errorBackoff = 0; channelids: Map = new Map(); readonly userMap: Map = new Map(); + voiceFactory?:VoiceFactory; instancePing = { name: "Unknown", }; @@ -74,6 +75,9 @@ class Localuser{ this.guildids = new Map(); this.user = new User(ready.d.user, this); this.user.setstatus("online"); + + this.voiceFactory=new VoiceFactory({id:this.user.id}); + this.handleVoice(); this.mfa_enabled = ready.d.user.mfa_enabled as boolean; this.userinfo.username = this.user.username; this.userinfo.pfpsrc = this.user.getpfpsrc(); @@ -117,6 +121,7 @@ class Localuser{ this.pingEndpoint(); this.userinfo.updateLocal(); + } outoffocus(): void{ const servers = document.getElementById("servers") as HTMLDivElement; @@ -477,15 +482,14 @@ class Localuser{ break; } case "VOICE_STATE_UPDATE": - if(this.waitingForVoice){ - this.waitingForVoice(temp); + if(this.voiceFactory){ + this.voiceFactory.voiceStateUpdate(temp) } break; case "VOICE_SERVER_UPDATE": - if(this.currentVoice){ - Voice.url=temp.d.endpoint; - Voice.gotUrl(); + if(this.voiceFactory){ + this.voiceFactory.voiceServerUpdate(temp) } break; } @@ -504,32 +508,30 @@ class Localuser{ }, this.heartbeat_interval); } } - waitingForVoice?:((arg:voiceupdate|undefined)=>void); - currentVoice?:Voice; - async joinVoice(voice:Voice){ - if(this.currentVoice){ - this.currentVoice.leave(); - } - this.currentVoice=voice; - if(this.ws){ - this.ws.send(JSON.stringify({ - d:{ - guild_id: voice.guild.id, - channel_id: voice.channel.id, - self_mute: true,//todo - self_deaf: false,//todo - self_video: false,//What is this? I have some guesses - flags: 2//????? - }, - op:4 - })); - if(this.waitingForVoice){ - this.waitingForVoice(undefined); - } - return await new Promise((res)=>{this.waitingForVoice=res;}) - } + get currentVoice(){ + return this.voiceFactory?.currentVoice; + } + async joinVoice(channel:Channel){ + if(!this.voiceFactory) return; + if(!this.ws) return; + this.ws.send(JSON.stringify(this.voiceFactory.joinVoice(channel.id,channel.guild.id))); return undefined; } + changeVCStatus(status:string){ + const statuselm=document.getElementById("VoiceStatus"); + if(!statuselm) throw new Error("Missing status element"); + statuselm.textContent=status; + } + handleVoice(){ + if(this.voiceFactory){ + this.voiceFactory.onJoin=voice=>{ + voice.onSatusChange=status=>{ + this.changeVCStatus(status); + } + } + } + } + heartbeat_interval: number = 0; updateChannel(json: channeljson): void{ const guild = this.guildids.get(json.guild_id); diff --git a/src/webpage/style.css b/src/webpage/style.css index e3bb30d..36e4661 100644 --- a/src/webpage/style.css +++ b/src/webpage/style.css @@ -2267,4 +2267,18 @@ form div{ >div{ opacity:.4; } +} +#VoiceBox{ + padding: .05in; + display: flex; + align-items: center; + justify-content: center; + align-content: center; + width: 100%; + box-sizing: border-box; + border-radius: .1in .1in 0 0; + border: solid .01in var(--black); +} +#VoiceStatus{ + font-weight: bold; } \ No newline at end of file diff --git a/src/webpage/user.ts b/src/webpage/user.ts index 8cddb12..0867ccf 100644 --- a/src/webpage/user.ts +++ b/src/webpage/user.ts @@ -301,7 +301,7 @@ class User extends SnowFlake{ localuser.info.api.toString() + "/users/" + id + "/profile", { headers: localuser.headers } ).then(res=>res.json()); - return new User(json, localuser); + return new User(json.user, localuser); } changepfp(update: string | null): void{ diff --git a/src/webpage/voice.ts b/src/webpage/voice.ts index e1be722..ada524f 100644 --- a/src/webpage/voice.ts +++ b/src/webpage/voice.ts @@ -1,11 +1,80 @@ -import { Channel } from "./channel.js"; -import { sdpback, webRTCSocket } from "./jsontypes.js"; +import { memberjson, sdpback, voiceserverupdate, voiceupdate, webRTCSocket } from "./jsontypes.js"; + +class VoiceFactory{ + settings:{id:string}; + constructor(usersettings:VoiceFactory["settings"]){ + this.settings=usersettings; + } + voices=new Map>(); + voiceChannels=new Map(); + currentVoice?:Voice; + guildUrlMap=new Map,gotUrl:()=>void}>(); + makeVoice(guildid:string,channelId:string,settings:Voice["settings"]){ + let guild=this.voices.get(guildid); + if(!guild){ + this.setUpGuild(guildid); + guild=new Map(); + this.voices.set(guildid,guild); + } + const urlobj=this.guildUrlMap.get(guildid); + if(!urlobj) throw new Error("url Object doesn't exist (InternalError)"); + const voice=new Voice(this.settings.id,settings,urlobj); + this.voiceChannels.set(channelId,voice); + guild.set(channelId,voice); + return voice; + } + onJoin=(_voice:Voice)=>{}; + onLeave=(_voice:Voice)=>{}; + joinVoice(channelId:string,guildId:string){ + if(this.currentVoice){ + this.currentVoice.leave(); + } + const voice=this.voiceChannels.get(channelId); + if(!voice) throw new Error(`Voice ${channelId} does not exist`); + voice.join(); + this.currentVoice=voice; + this.onJoin(voice); + return { + d:{ + guild_id: guildId, + channel_id: channelId, + self_mute: true,//todo + self_deaf: false,//todo + self_video: false,//What is this? I have some guesses + flags: 2//????? + }, + op:4 + } + } + userMap=new Map(); + voiceStateUpdate(update:voiceupdate){ + + const prev=this.userMap.get(update.d.user_id); + console.log(prev,this.userMap); + if(prev){ + prev.disconnect(update.d.user_id); + this.onLeave(prev); + } + const voice=this.voiceChannels.get(update.d.channel_id); + if(voice){ + this.userMap.set(update.d.user_id,voice); + voice.voiceupdate(update); + } + } + private setUpGuild(id:string){ + const obj:{url?:string,geturl?:Promise,gotUrl?:()=>void}={}; + obj.geturl=new Promise(res=>{obj.gotUrl=res}); + this.guildUrlMap.set(id,obj as {geturl:Promise,gotUrl:()=>void}); + } + voiceServerUpdate(update:voiceserverupdate){ + const obj=this.guildUrlMap.get(update.d.guild_id); + if(!obj) return; + obj.url=update.d.endpoint; + obj.gotUrl(); + } +} class Voice{ - owner:Channel; - static url?:string; - static gotUrl:()=>void; - static geturl=new Promise(res=>{this.gotUrl=res}) private pstatus:string="not connected"; public onSatusChange:(e:string)=>unknown=()=>{}; set status(e:string){ @@ -15,17 +84,13 @@ class Voice{ get status(){ return this.pstatus; } - get channel(){ - return this.owner; - } - get guild(){ - return this.owner.owner; - } - get localuser(){ - return this.owner.localuser; - } - constructor(owner:Channel){ - this.owner=owner; + readonly userid:string; + settings:{bitrate:number}; + urlobj:{url?:string,geturl:Promise,gotUrl:()=>void}; + constructor(userid:string,settings:Voice["settings"],urlobj:Voice["urlobj"]){ + this.userid=userid; + this.settings=settings; + this.urlobj=urlobj; } pc?:RTCPeerConnection; ws?:WebSocket; @@ -40,7 +105,28 @@ class Voice{ } readonly users= new Map(); readonly speakingMap= new Map(); - onSpeakingChange=(_:string,_2:number)=>{}; + onSpeakingChange=(_userid:string,_speaking:number)=>{}; + disconnect(userid:string){ + console.warn(userid); + if(userid===this.userid){ + this.leave(); + } + const ssrc=this.speakingMap.get(userid); + + if(ssrc){ + this.users.delete(ssrc); + for(const thing of this.ssrcMap){ + if(thing[1]===ssrc){ + this.ssrcMap.delete(thing[0]); + } + } + } + this.speakingMap.delete(userid); + this.userids.delete(userid); + console.log(this.userids,userid); + //there's more for sure, but this is "good enough" for now + this.onMemberChange(userid,false); + } packet(message:MessageEvent){ const data=message.data if(typeof data === "string"){ @@ -89,14 +175,7 @@ class Voice{ console.log(bundles); if(!this.offer) throw new Error("Offer is missing :P"); - let cline:string|undefined; - console.log(sdp); - for(const line of sdp.split("\n")){ - if(line.startsWith("c=")){ - cline=line; - break; - } - } + let cline=sdp.split("\n").find(line=>line.startsWith("c=")); if(!cline) throw new Error("c line wasn't found"); const parsed1=Voice.parsesdp(sdp).medias[0]; //const parsed2=Voice.parsesdp(this.offer); @@ -168,7 +247,6 @@ a=rtcp-mux\r`; i++ } build+="\n"; - console.log(build); return build; } counter?:string; @@ -298,18 +376,19 @@ a=rtcp-mux\r`; pc.addTransceiver("audio",{ direction:"recvonly", streams:[], - sendEncodings:[{active:true,maxBitrate:this.channel.bitrate}] + sendEncodings:[{active:true,maxBitrate:this.settings.bitrate}] }); } for(let i=0;i<10;i++){ pc.addTransceiver("video",{ direction:"recvonly", streams:[], - sendEncodings:[{active:true,maxBitrate:this.channel.bitrate}] + sendEncodings:[{active:true,maxBitrate:this.settings.bitrate}] }); } this.counter=data.d.sdp; pc.ontrack = async (e) => { + this.status="Done"; if(e.track.kind==="video"){ return; } @@ -496,58 +575,69 @@ a=rtcp-mux\r`; } return out; } + open=false; async join(){ console.warn("Joining"); + this.open=true this.status="waiting for main WS"; - const json = await this.localuser.joinVoice(this); - if(!json) { - this.status="bad responce from WS"; - return; - }; - if(!Voice.url){ - this.status="waiting for Voice URL"; - await Voice.geturl; - } - if(this.localuser.currentVoice!==this){this.status="closed";return} - const ws=new WebSocket("ws://"+Voice.url as string); - this.ws=ws; - ws.onclose=()=>{ - this.leave(); - } - this.status="waiting for WS to open"; - ws.addEventListener("message",(m)=>{ - this.packet(m); - }) - await new Promise(res=>{ - ws.addEventListener("open",()=>{ - res() - }) - }); - this.status="waiting for WS to authorize"; - ws.send(JSON.stringify({ - "op": 0, - "d": { - server_id: this.guild.id, - user_id: json.d.user_id, - session_id: json.d.session_id, - token: json.d.token, - video: false, - "streams": [ - { - type: "video", - rid: "100", - quality: 100 - } - ] + } + onMemberChange=(_member:memberjson|string,_joined:boolean)=>{}; + userids=new Map(); + async voiceupdate(update:voiceupdate){ + console.log("Update!"); + this.userids.set(update.d.member.id,{deaf:update.d.deaf,muted:update.d.mute}); + this.onMemberChange(update.d.member,true); + if(update.d.member.id===this.userid&&this.open){ + if(!update) { + this.status="bad responce from WS"; + return; + }; + if(!this.urlobj.url){ + this.status="waiting for Voice URL"; + await this.urlobj.geturl; + if(!this.open){this.leave();return} } - })); - /* - const pc=new RTCPeerConnection(); - this.pc=pc; - //pc.setRemoteDescription({sdp:json.d.token,type:""}) - */ + + const ws=new WebSocket("ws://"+this.urlobj.url as string); + this.ws=ws; + ws.onclose=()=>{ + this.leave(); + } + this.status="waiting for WS to open"; + ws.addEventListener("message",(m)=>{ + this.packet(m); + }) + await new Promise(res=>{ + ws.addEventListener("open",()=>{ + res() + }) + }); + if(!this.ws){ + this.leave(); + return; + } + this.status="waiting for WS to authorize"; + ws.send(JSON.stringify({ + "op": 0, + "d": { + server_id: update.d.guild_id, + user_id: update.d.user_id, + session_id: update.d.session_id, + token: update.d.token, + video: false, + "streams": [ + { + type: "video", + rid: "100", + quality: 100 + } + ] + } + })); + } } async leave(){ + this.open=false; this.status="Left voice chat"; if(this.ws){ this.ws.close(); @@ -559,4 +649,4 @@ a=rtcp-mux\r`; } } } -export {Voice}; +export {Voice,VoiceFactory};