Line data Source code
1 : import 'dart:async';
2 :
3 : import 'package:collection/collection.dart';
4 : import 'package:webrtc_interface/webrtc_interface.dart';
5 :
6 : import 'package:matrix/matrix.dart';
7 : import 'package:matrix/src/utils/cached_stream_controller.dart';
8 : import 'package:matrix/src/voip/models/call_options.dart';
9 : import 'package:matrix/src/voip/utils/stream_helper.dart';
10 : import 'package:matrix/src/voip/utils/user_media_constraints.dart';
11 :
12 : class MeshBackend extends CallBackend {
13 4 : MeshBackend({
14 : super.type = 'mesh',
15 : });
16 :
17 : final List<CallSession> _callSessions = [];
18 :
19 : /// participant:volume
20 : final Map<CallParticipant, double> _audioLevelsMap = {};
21 :
22 : /// The stream is used to prepare for incoming peer calls like registering listeners
23 : StreamSubscription<CallSession>? _callSetupSubscription;
24 :
25 : /// The stream is used to signal the start of an incoming peer call
26 : StreamSubscription<CallSession>? _callStartSubscription;
27 :
28 : Timer? _activeSpeakerLoopTimeout;
29 :
30 : final CachedStreamController<WrappedMediaStream> onStreamAdd =
31 : CachedStreamController();
32 :
33 : final CachedStreamController<WrappedMediaStream> onStreamRemoved =
34 : CachedStreamController();
35 :
36 : final CachedStreamController<GroupCallSession> onGroupCallFeedsChanged =
37 : CachedStreamController();
38 :
39 4 : @override
40 : Map<String, Object?> toJson() {
41 4 : return {
42 4 : 'type': type,
43 : };
44 : }
45 :
46 : CallParticipant? _activeSpeaker;
47 : WrappedMediaStream? _localUserMediaStream;
48 : WrappedMediaStream? _localScreenshareStream;
49 : final List<WrappedMediaStream> _userMediaStreams = [];
50 : final List<WrappedMediaStream> _screenshareStreams = [];
51 :
52 2 : List<WrappedMediaStream> _getLocalStreams() {
53 2 : final feeds = <WrappedMediaStream>[];
54 :
55 2 : if (localUserMediaStream != null) {
56 4 : feeds.add(localUserMediaStream!);
57 : }
58 :
59 2 : if (localScreenshareStream != null) {
60 0 : feeds.add(localScreenshareStream!);
61 : }
62 :
63 : return feeds;
64 : }
65 :
66 2 : Future<MediaStream> _getUserMedia(
67 : GroupCallSession groupCall,
68 : CallType type,
69 : ) async {
70 2 : final mediaConstraints = {
71 : 'audio': UserMediaConstraints.micMediaConstraints,
72 2 : 'video': type == CallType.kVideo
73 : ? UserMediaConstraints.camMediaConstraints
74 : : false,
75 : };
76 :
77 : try {
78 6 : return await groupCall.voip.delegate.mediaDevices
79 2 : .getUserMedia(mediaConstraints);
80 : } catch (e) {
81 0 : groupCall.setState(GroupCallState.localCallFeedUninitialized);
82 : rethrow;
83 : }
84 : }
85 :
86 0 : Future<MediaStream> _getDisplayMedia(GroupCallSession groupCall) async {
87 0 : final mediaConstraints = {
88 : 'audio': false,
89 : 'video': true,
90 : };
91 : try {
92 0 : return await groupCall.voip.delegate.mediaDevices
93 0 : .getDisplayMedia(mediaConstraints);
94 : } catch (e, s) {
95 0 : throw MatrixSDKVoipException('_getDisplayMedia failed', stackTrace: s);
96 : }
97 : }
98 :
99 2 : CallSession? _getCallForParticipant(
100 : GroupCallSession groupCall,
101 : CallParticipant participant,
102 : ) {
103 4 : return _callSessions.singleWhereOrNull(
104 2 : (call) =>
105 6 : call.groupCallId == groupCall.groupCallId &&
106 2 : CallParticipant(
107 2 : groupCall.voip,
108 2 : userId: call.remoteUserId!,
109 2 : deviceId: call.remoteDeviceId,
110 2 : ) ==
111 : participant,
112 : );
113 : }
114 :
115 : /// Register listeners for a peer call to use for the group calls, that is
116 : /// needed before even call is added to `_callSessions`.
117 : /// We do this here for onStreamAdd and onStreamRemoved to make sure we don't
118 : /// miss any events that happen before the call is completely started.
119 2 : void _registerListenersBeforeCallAdd(CallSession call) {
120 8 : call.onStreamAdd.stream.listen((stream) {
121 2 : if (!stream.isLocal()) {
122 0 : onStreamAdd.add(stream);
123 : }
124 : });
125 :
126 6 : call.onStreamRemoved.stream.listen((stream) {
127 0 : if (!stream.isLocal()) {
128 0 : onStreamRemoved.add(stream);
129 : }
130 : });
131 : }
132 :
133 2 : Future<void> _addCall(GroupCallSession groupCall, CallSession call) async {
134 4 : _callSessions.add(call);
135 2 : _initCall(groupCall, call);
136 4 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
137 : }
138 :
139 : /// init a peer call from group calls.
140 2 : void _initCall(GroupCallSession groupCall, CallSession call) {
141 2 : if (call.remoteUserId == null) {
142 0 : throw MatrixSDKVoipException(
143 : 'Cannot init call without proper invitee user and device Id',
144 : );
145 : }
146 :
147 6 : call.onCallStateChanged.stream.listen(
148 0 : ((event) async {
149 0 : await _onCallStateChanged(call, event);
150 : }),
151 : );
152 :
153 6 : call.onCallReplaced.stream.listen((CallSession newCall) async {
154 0 : await _replaceCall(groupCall, call, newCall);
155 : });
156 :
157 6 : call.onCallStreamsChanged.stream.listen((call) async {
158 0 : await call.tryRemoveStopedStreams();
159 0 : await _onStreamsChanged(groupCall, call);
160 : });
161 :
162 6 : call.onCallHangupNotifierForGroupCalls.stream.listen((event) async {
163 0 : await _onCallHangup(groupCall, call);
164 : });
165 : }
166 :
167 0 : Future<void> _replaceCall(
168 : GroupCallSession groupCall,
169 : CallSession existingCall,
170 : CallSession replacementCall,
171 : ) async {
172 0 : final existingCallIndex = _callSessions
173 0 : .indexWhere((element) => element.callId == existingCall.callId);
174 :
175 0 : if (existingCallIndex == -1) {
176 0 : throw MatrixSDKVoipException('Couldn\'t find call to replace');
177 : }
178 :
179 0 : _callSessions.removeAt(existingCallIndex);
180 0 : _callSessions.add(replacementCall);
181 :
182 0 : await _disposeCall(groupCall, existingCall, CallErrorCode.replaced);
183 0 : _registerListenersBeforeCallAdd(replacementCall);
184 0 : _initCall(groupCall, replacementCall);
185 :
186 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
187 : }
188 :
189 : /// Removes a peer call from group calls.
190 0 : Future<void> _removeCall(
191 : GroupCallSession groupCall,
192 : CallSession call,
193 : CallErrorCode hangupReason,
194 : ) async {
195 0 : await _disposeCall(groupCall, call, hangupReason);
196 :
197 0 : _callSessions.removeWhere((element) => call.callId == element.callId);
198 :
199 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.callsChanged);
200 : }
201 :
202 0 : Future<void> _disposeCall(
203 : GroupCallSession groupCall,
204 : CallSession call,
205 : CallErrorCode hangupReason,
206 : ) async {
207 0 : if (call.remoteUserId == null) {
208 0 : throw MatrixSDKVoipException(
209 : 'Cannot init call without proper invitee user and device Id',
210 : );
211 : }
212 :
213 0 : if (call.hangupReason == CallErrorCode.replaced) {
214 : return;
215 : }
216 :
217 0 : if (call.state != CallState.kEnded) {
218 : // no need to emit individual handleCallEnded on group calls
219 : // also prevents a loop of hangup and onCallHangupNotifierForGroupCalls
220 0 : await call.hangup(reason: hangupReason, shouldEmit: false);
221 : }
222 :
223 0 : final usermediaStream = _getUserMediaStreamByParticipantId(
224 0 : CallParticipant(
225 0 : groupCall.voip,
226 0 : userId: call.remoteUserId!,
227 0 : deviceId: call.remoteDeviceId,
228 0 : ).id,
229 : );
230 :
231 : if (usermediaStream != null) {
232 0 : await _removeUserMediaStream(groupCall, usermediaStream);
233 : }
234 :
235 0 : final screenshareStream = _getScreenshareStreamByParticipantId(
236 0 : CallParticipant(
237 0 : groupCall.voip,
238 0 : userId: call.remoteUserId!,
239 0 : deviceId: call.remoteDeviceId,
240 0 : ).id,
241 : );
242 :
243 : if (screenshareStream != null) {
244 0 : await _removeScreenshareStream(groupCall, screenshareStream);
245 : }
246 : }
247 :
248 0 : Future<void> _onStreamsChanged(
249 : GroupCallSession groupCall,
250 : CallSession call,
251 : ) async {
252 0 : if (call.remoteUserId == null) {
253 0 : throw MatrixSDKVoipException(
254 : 'Cannot init call without proper invitee user and device Id',
255 : );
256 : }
257 :
258 0 : final currentUserMediaStream = _getUserMediaStreamByParticipantId(
259 0 : CallParticipant(
260 0 : groupCall.voip,
261 0 : userId: call.remoteUserId!,
262 0 : deviceId: call.remoteDeviceId,
263 0 : ).id,
264 : );
265 :
266 0 : final remoteUsermediaStream = call.remoteUserMediaStream;
267 0 : final remoteStreamChanged = remoteUsermediaStream != currentUserMediaStream;
268 :
269 : if (remoteStreamChanged) {
270 : if (currentUserMediaStream == null && remoteUsermediaStream != null) {
271 0 : await _addUserMediaStream(groupCall, remoteUsermediaStream);
272 : } else if (currentUserMediaStream != null &&
273 : remoteUsermediaStream != null) {
274 0 : await _replaceUserMediaStream(
275 : groupCall,
276 : currentUserMediaStream,
277 : remoteUsermediaStream,
278 : );
279 : } else if (currentUserMediaStream != null &&
280 : remoteUsermediaStream == null) {
281 0 : await _removeUserMediaStream(groupCall, currentUserMediaStream);
282 : }
283 : }
284 :
285 0 : final currentScreenshareStream = _getScreenshareStreamByParticipantId(
286 0 : CallParticipant(
287 0 : groupCall.voip,
288 0 : userId: call.remoteUserId!,
289 0 : deviceId: call.remoteDeviceId,
290 0 : ).id,
291 : );
292 0 : final remoteScreensharingStream = call.remoteScreenSharingStream;
293 : final remoteScreenshareStreamChanged =
294 0 : remoteScreensharingStream != currentScreenshareStream;
295 :
296 : if (remoteScreenshareStreamChanged) {
297 : if (currentScreenshareStream == null &&
298 : remoteScreensharingStream != null) {
299 0 : _addScreenshareStream(groupCall, remoteScreensharingStream);
300 : } else if (currentScreenshareStream != null &&
301 : remoteScreensharingStream != null) {
302 0 : await _replaceScreenshareStream(
303 : groupCall,
304 : currentScreenshareStream,
305 : remoteScreensharingStream,
306 : );
307 : } else if (currentScreenshareStream != null &&
308 : remoteScreensharingStream == null) {
309 0 : await _removeScreenshareStream(groupCall, currentScreenshareStream);
310 : }
311 : }
312 :
313 0 : onGroupCallFeedsChanged.add(groupCall);
314 : }
315 :
316 0 : WrappedMediaStream? _getUserMediaStreamByParticipantId(String participantId) {
317 0 : final stream = _userMediaStreams
318 0 : .where((stream) => stream.participant.id == participantId);
319 0 : if (stream.isNotEmpty) {
320 0 : return stream.first;
321 : }
322 : return null;
323 : }
324 :
325 2 : void _onActiveSpeakerLoop(GroupCallSession groupCall) async {
326 : CallParticipant? nextActiveSpeaker;
327 : // idc about screen sharing atm.
328 : final userMediaStreamsCopyList =
329 4 : List<WrappedMediaStream>.from(_userMediaStreams);
330 4 : for (final stream in userMediaStreamsCopyList) {
331 6 : if (stream.participant.isLocal && stream.pc == null) {
332 : continue;
333 : }
334 :
335 0 : final List<StatsReport> statsReport = await stream.pc!.getStats();
336 : statsReport
337 0 : .removeWhere((element) => !element.values.containsKey('audioLevel'));
338 :
339 : // https://www.w3.org/TR/webrtc-stats/#summary
340 : final otherPartyAudioLevel = statsReport
341 0 : .singleWhereOrNull(
342 0 : (element) =>
343 0 : element.type == 'inbound-rtp' &&
344 0 : element.values['kind'] == 'audio',
345 : )
346 0 : ?.values['audioLevel'];
347 : if (otherPartyAudioLevel != null) {
348 0 : _audioLevelsMap[stream.participant] = otherPartyAudioLevel;
349 : }
350 :
351 : // https://www.w3.org/TR/webrtc-stats/#dom-rtcstatstype-media-source
352 : // firefox does not seem to have this though. Works on chrome and android
353 : final ownAudioLevel = statsReport
354 0 : .singleWhereOrNull(
355 0 : (element) =>
356 0 : element.type == 'media-source' &&
357 0 : element.values['kind'] == 'audio',
358 : )
359 0 : ?.values['audioLevel'];
360 0 : if (groupCall.localParticipant != null &&
361 : ownAudioLevel != null &&
362 0 : _audioLevelsMap[groupCall.localParticipant] != ownAudioLevel) {
363 0 : _audioLevelsMap[groupCall.localParticipant!] = ownAudioLevel;
364 : }
365 : }
366 :
367 : double maxAudioLevel = double.negativeInfinity;
368 : // TODO: we probably want a threshold here?
369 4 : _audioLevelsMap.forEach((key, value) {
370 0 : if (value > maxAudioLevel) {
371 : nextActiveSpeaker = key;
372 : maxAudioLevel = value;
373 : }
374 : });
375 :
376 0 : if (nextActiveSpeaker != null && _activeSpeaker != nextActiveSpeaker) {
377 0 : _activeSpeaker = nextActiveSpeaker;
378 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.activeSpeakerChanged);
379 : }
380 2 : _activeSpeakerLoopTimeout?.cancel();
381 4 : _activeSpeakerLoopTimeout = Timer(
382 : CallConstants.activeSpeakerInterval,
383 0 : () => _onActiveSpeakerLoop(groupCall),
384 : );
385 : }
386 :
387 0 : WrappedMediaStream? _getScreenshareStreamByParticipantId(
388 : String participantId,
389 : ) {
390 0 : final stream = _screenshareStreams
391 0 : .where((stream) => stream.participant.id == participantId);
392 0 : if (stream.isNotEmpty) {
393 0 : return stream.first;
394 : }
395 : return null;
396 : }
397 :
398 0 : void _addScreenshareStream(
399 : GroupCallSession groupCall,
400 : WrappedMediaStream stream,
401 : ) {
402 0 : _screenshareStreams.add(stream);
403 0 : onStreamAdd.add(stream);
404 0 : groupCall.onGroupCallEvent
405 0 : .add(GroupCallStateChange.screenshareStreamsChanged);
406 : }
407 :
408 0 : Future<void> _replaceScreenshareStream(
409 : GroupCallSession groupCall,
410 : WrappedMediaStream existingStream,
411 : WrappedMediaStream replacementStream,
412 : ) async {
413 0 : final streamIndex = _screenshareStreams.indexWhere(
414 0 : (stream) => stream.participant.id == existingStream.participant.id,
415 : );
416 :
417 0 : if (streamIndex == -1) {
418 0 : throw MatrixSDKVoipException(
419 : 'Couldn\'t find screenshare stream to replace',
420 : );
421 : }
422 :
423 0 : _screenshareStreams.replaceRange(streamIndex, 1, [replacementStream]);
424 :
425 0 : await existingStream.dispose();
426 0 : groupCall.onGroupCallEvent
427 0 : .add(GroupCallStateChange.screenshareStreamsChanged);
428 : }
429 :
430 0 : Future<void> _removeScreenshareStream(
431 : GroupCallSession groupCall,
432 : WrappedMediaStream stream,
433 : ) async {
434 0 : final streamIndex = _screenshareStreams
435 0 : .indexWhere((stream) => stream.participant.id == stream.participant.id);
436 :
437 0 : if (streamIndex == -1) {
438 0 : throw MatrixSDKVoipException(
439 : 'Couldn\'t find screenshare stream to remove',
440 : );
441 : }
442 :
443 0 : _screenshareStreams.removeWhere(
444 0 : (element) => element.participant.id == stream.participant.id,
445 : );
446 :
447 0 : onStreamRemoved.add(stream);
448 :
449 0 : if (stream.isLocal()) {
450 0 : await stopMediaStream(stream.stream);
451 : }
452 :
453 0 : groupCall.onGroupCallEvent
454 0 : .add(GroupCallStateChange.screenshareStreamsChanged);
455 : }
456 :
457 0 : Future<void> _onCallStateChanged(CallSession call, CallState state) async {
458 0 : final audioMuted = localUserMediaStream?.isAudioMuted() ?? true;
459 0 : if (call.localUserMediaStream != null &&
460 0 : call.isMicrophoneMuted != audioMuted) {
461 0 : await call.setMicrophoneMuted(audioMuted);
462 : }
463 :
464 0 : final videoMuted = localUserMediaStream?.isVideoMuted() ?? true;
465 :
466 0 : if (call.localUserMediaStream != null &&
467 0 : call.isLocalVideoMuted != videoMuted) {
468 0 : await call.setLocalVideoMuted(videoMuted);
469 : }
470 : }
471 :
472 0 : Future<void> _onCallHangup(
473 : GroupCallSession groupCall,
474 : CallSession call,
475 : ) async {
476 0 : if (call.hangupReason == CallErrorCode.replaced) {
477 : return;
478 : }
479 0 : await _onStreamsChanged(groupCall, call);
480 0 : await _removeCall(groupCall, call, call.hangupReason!);
481 : }
482 :
483 2 : Future<void> _addUserMediaStream(
484 : GroupCallSession groupCall,
485 : WrappedMediaStream stream,
486 : ) async {
487 4 : _userMediaStreams.add(stream);
488 4 : onStreamAdd.add(stream);
489 2 : groupCall.onGroupCallEvent
490 2 : .add(GroupCallStateChange.userMediaStreamsChanged);
491 : }
492 :
493 0 : Future<void> _replaceUserMediaStream(
494 : GroupCallSession groupCall,
495 : WrappedMediaStream existingStream,
496 : WrappedMediaStream replacementStream,
497 : ) async {
498 0 : final streamIndex = _userMediaStreams.indexWhere(
499 0 : (stream) => stream.participant.id == existingStream.participant.id,
500 : );
501 :
502 0 : if (streamIndex == -1) {
503 0 : throw MatrixSDKVoipException(
504 : 'Couldn\'t find user media stream to replace',
505 : );
506 : }
507 :
508 0 : _userMediaStreams.replaceRange(streamIndex, 1, [replacementStream]);
509 :
510 0 : await existingStream.dispose();
511 0 : groupCall.onGroupCallEvent
512 0 : .add(GroupCallStateChange.userMediaStreamsChanged);
513 : }
514 :
515 0 : Future<void> _removeUserMediaStream(
516 : GroupCallSession groupCall,
517 : WrappedMediaStream stream,
518 : ) async {
519 0 : final streamIndex = _userMediaStreams.indexWhere(
520 0 : (element) => element.participant.id == stream.participant.id,
521 : );
522 :
523 0 : if (streamIndex == -1) {
524 0 : throw MatrixSDKVoipException(
525 : 'Couldn\'t find user media stream to remove',
526 : );
527 : }
528 :
529 0 : _userMediaStreams.removeWhere(
530 0 : (element) => element.participant.id == stream.participant.id,
531 : );
532 0 : _audioLevelsMap.remove(stream.participant);
533 0 : onStreamRemoved.add(stream);
534 :
535 0 : if (stream.isLocal()) {
536 0 : await stopMediaStream(stream.stream);
537 : }
538 :
539 0 : groupCall.onGroupCallEvent
540 0 : .add(GroupCallStateChange.userMediaStreamsChanged);
541 :
542 0 : if (_activeSpeaker == stream.participant && _userMediaStreams.isNotEmpty) {
543 0 : _activeSpeaker = _userMediaStreams[0].participant;
544 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.activeSpeakerChanged);
545 : }
546 : }
547 :
548 0 : @override
549 : bool get e2eeEnabled => false;
550 :
551 0 : @override
552 0 : CallParticipant? get activeSpeaker => _activeSpeaker;
553 :
554 2 : @override
555 2 : WrappedMediaStream? get localUserMediaStream => _localUserMediaStream;
556 :
557 2 : @override
558 2 : WrappedMediaStream? get localScreenshareStream => _localScreenshareStream;
559 :
560 0 : @override
561 : List<WrappedMediaStream> get userMediaStreams =>
562 0 : List.unmodifiable(_userMediaStreams);
563 :
564 0 : @override
565 : List<WrappedMediaStream> get screenShareStreams =>
566 0 : List.unmodifiable(_screenshareStreams);
567 :
568 0 : @override
569 : Future<void> updateMediaDeviceForCalls() async {
570 0 : for (final call in _callSessions) {
571 0 : await call.updateMediaDeviceForCall();
572 : }
573 : }
574 :
575 : /// Initializes the local user media stream.
576 : /// The media stream must be prepared before the group call enters.
577 : /// if you allow the user to configure their camera and such ahead of time,
578 : /// you can pass that `stream` on to this function.
579 : /// This allows you to configure the camera before joining the call without
580 : /// having to reopen the stream and possibly losing settings.
581 2 : @override
582 : Future<WrappedMediaStream?> initLocalStream(
583 : GroupCallSession groupCall, {
584 : WrappedMediaStream? stream,
585 : }) async {
586 4 : if (groupCall.state != GroupCallState.localCallFeedUninitialized) {
587 0 : throw MatrixSDKVoipException(
588 0 : 'Cannot initialize local call feed in the ${groupCall.state} state.',
589 : );
590 : }
591 :
592 2 : groupCall.setState(GroupCallState.initializingLocalCallFeed);
593 :
594 : WrappedMediaStream localWrappedMediaStream;
595 :
596 : if (stream == null) {
597 : MediaStream stream;
598 :
599 : try {
600 2 : stream = await _getUserMedia(groupCall, CallType.kVideo);
601 : } catch (error) {
602 0 : groupCall.setState(GroupCallState.localCallFeedUninitialized);
603 : rethrow;
604 : }
605 :
606 2 : localWrappedMediaStream = WrappedMediaStream(
607 : stream: stream,
608 2 : participant: groupCall.localParticipant!,
609 2 : room: groupCall.room,
610 2 : client: groupCall.client,
611 : purpose: SDPStreamMetadataPurpose.Usermedia,
612 4 : audioMuted: stream.getAudioTracks().isEmpty,
613 4 : videoMuted: stream.getVideoTracks().isEmpty,
614 : isGroupCall: true,
615 2 : voip: groupCall.voip,
616 : );
617 : } else {
618 : localWrappedMediaStream = stream;
619 : }
620 :
621 2 : _localUserMediaStream = localWrappedMediaStream;
622 2 : await _addUserMediaStream(groupCall, localWrappedMediaStream);
623 :
624 2 : groupCall.setState(GroupCallState.localCallFeedInitialized);
625 :
626 2 : _activeSpeaker = null;
627 :
628 : return localWrappedMediaStream;
629 : }
630 :
631 0 : @override
632 : Future<void> setDeviceMuted(
633 : GroupCallSession groupCall,
634 : bool muted,
635 : MediaInputKind kind,
636 : ) async {
637 0 : if (!await hasMediaDevice(groupCall.voip.delegate, kind)) {
638 : return;
639 : }
640 :
641 0 : if (localUserMediaStream != null) {
642 : switch (kind) {
643 0 : case MediaInputKind.audioinput:
644 0 : localUserMediaStream!.setAudioMuted(muted);
645 0 : setTracksEnabled(
646 0 : localUserMediaStream!.stream!.getAudioTracks(),
647 : !muted,
648 : );
649 0 : for (final call in _callSessions) {
650 0 : await call.setMicrophoneMuted(muted);
651 : }
652 : break;
653 0 : case MediaInputKind.videoinput:
654 0 : localUserMediaStream!.setVideoMuted(muted);
655 0 : setTracksEnabled(
656 0 : localUserMediaStream!.stream!.getVideoTracks(),
657 : !muted,
658 : );
659 0 : for (final call in _callSessions) {
660 0 : await call.setLocalVideoMuted(muted);
661 : }
662 : break;
663 : }
664 : }
665 :
666 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.localMuteStateChanged);
667 : return;
668 : }
669 :
670 2 : void _onIncomingCallInMeshSetup(
671 : GroupCallSession groupCall,
672 : CallSession newCall,
673 : ) {
674 : // The incoming calls may be for another room, which we will ignore.
675 10 : if (newCall.room.id != groupCall.room.id) return;
676 :
677 4 : if (newCall.state != CallState.kRinging) {
678 4 : Logs().v(
679 : '[_onIncomingCallInMeshSetup] Incoming call no longer in ringing state. Ignoring.',
680 : );
681 : return;
682 : }
683 :
684 0 : if (newCall.groupCallId == null ||
685 0 : newCall.groupCallId != groupCall.groupCallId) {
686 0 : Logs().v(
687 0 : '[_onIncomingCallInMeshSetup] Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call',
688 : );
689 : return;
690 : }
691 :
692 0 : final existingCall = _getCallForParticipant(
693 : groupCall,
694 0 : CallParticipant(
695 0 : groupCall.voip,
696 0 : userId: newCall.remoteUserId!,
697 0 : deviceId: newCall.remoteDeviceId,
698 : ),
699 : );
700 :
701 : // if it's an existing call, `_registerListenersForCall` will be called in
702 : // `_replaceCall` that is used in `_onIncomingCallStart`.
703 : if (existingCall != null) return;
704 :
705 0 : Logs().v(
706 0 : '[_onIncomingCallInMeshSetup] GroupCallSession: incoming call from: ${newCall.remoteUserId}${newCall.remoteDeviceId}${newCall.remotePartyId}',
707 : );
708 :
709 0 : _registerListenersBeforeCallAdd(newCall);
710 : }
711 :
712 2 : Future<void> _onIncomingCallInMeshStart(
713 : GroupCallSession groupCall,
714 : CallSession newCall,
715 : ) async {
716 : // The incoming calls may be for another room, which we will ignore.
717 10 : if (newCall.room.id != groupCall.room.id) {
718 : return;
719 : }
720 :
721 4 : if (newCall.state != CallState.kRinging) {
722 4 : Logs().v(
723 : '[_onIncomingCallInMeshStart] Incoming call no longer in ringing state. Ignoring.',
724 : );
725 : return;
726 : }
727 :
728 0 : if (newCall.groupCallId == null ||
729 0 : newCall.groupCallId != groupCall.groupCallId) {
730 0 : Logs().v(
731 0 : '[_onIncomingCallInMeshStart] Incoming call with groupCallId ${newCall.groupCallId} ignored because it doesn\'t match the current group call',
732 : );
733 0 : await newCall.reject();
734 : return;
735 : }
736 :
737 0 : final existingCall = _getCallForParticipant(
738 : groupCall,
739 0 : CallParticipant(
740 0 : groupCall.voip,
741 0 : userId: newCall.remoteUserId!,
742 0 : deviceId: newCall.remoteDeviceId,
743 : ),
744 : );
745 :
746 0 : if (existingCall != null && existingCall.callId == newCall.callId) {
747 : return;
748 : }
749 :
750 0 : Logs().v(
751 0 : '[_onIncomingCallInMeshStart] GroupCallSession: incoming call from: ${newCall.remoteUserId}${newCall.remoteDeviceId}${newCall.remotePartyId}',
752 : );
753 :
754 : // Check if the user calling has an existing call and use this call instead.
755 : if (existingCall != null) {
756 0 : await _replaceCall(groupCall, existingCall, newCall);
757 : } else {
758 0 : await _addCall(groupCall, newCall);
759 : }
760 :
761 0 : await newCall.answerWithStreams(_getLocalStreams());
762 : }
763 :
764 0 : @override
765 : Future<void> setScreensharingEnabled(
766 : GroupCallSession groupCall,
767 : bool enabled,
768 : String desktopCapturerSourceId,
769 : ) async {
770 0 : if (enabled == (localScreenshareStream != null)) {
771 : return;
772 : }
773 :
774 : if (enabled) {
775 : try {
776 0 : Logs().v('Asking for screensharing permissions...');
777 0 : final stream = await _getDisplayMedia(groupCall);
778 0 : for (final track in stream.getTracks()) {
779 : // screen sharing should only have 1 video track anyway, so this only
780 : // fires once
781 0 : track.onEnded = () async {
782 0 : await setScreensharingEnabled(groupCall, false, '');
783 : };
784 : }
785 0 : Logs().v(
786 : 'Screensharing permissions granted. Setting screensharing enabled on all calls',
787 : );
788 0 : _localScreenshareStream = WrappedMediaStream(
789 : stream: stream,
790 0 : participant: groupCall.localParticipant!,
791 0 : room: groupCall.room,
792 0 : client: groupCall.client,
793 : purpose: SDPStreamMetadataPurpose.Screenshare,
794 0 : audioMuted: stream.getAudioTracks().isEmpty,
795 0 : videoMuted: stream.getVideoTracks().isEmpty,
796 : isGroupCall: true,
797 0 : voip: groupCall.voip,
798 : );
799 :
800 0 : _addScreenshareStream(groupCall, localScreenshareStream!);
801 :
802 0 : groupCall.onGroupCallEvent
803 0 : .add(GroupCallStateChange.localScreenshareStateChanged);
804 0 : for (final call in _callSessions) {
805 0 : await call.addLocalStream(
806 0 : await localScreenshareStream!.stream!.clone(),
807 0 : localScreenshareStream!.purpose,
808 : );
809 : }
810 :
811 0 : await groupCall.sendMemberStateEvent();
812 :
813 : return;
814 : } catch (e, s) {
815 0 : Logs().e('[VOIP] Enabling screensharing error', e, s);
816 0 : groupCall.onGroupCallEvent.add(GroupCallStateChange.error);
817 : return;
818 : }
819 : } else {
820 0 : for (final call in _callSessions) {
821 0 : await call.removeLocalStream(call.localScreenSharingStream!);
822 : }
823 0 : await stopMediaStream(localScreenshareStream?.stream);
824 0 : await _removeScreenshareStream(groupCall, localScreenshareStream!);
825 0 : _localScreenshareStream = null;
826 :
827 0 : await groupCall.sendMemberStateEvent();
828 :
829 0 : groupCall.onGroupCallEvent
830 0 : .add(GroupCallStateChange.localMuteStateChanged);
831 : return;
832 : }
833 : }
834 :
835 0 : @override
836 : Future<void> dispose(GroupCallSession groupCall) async {
837 0 : if (localUserMediaStream != null) {
838 0 : await _removeUserMediaStream(groupCall, localUserMediaStream!);
839 0 : _localUserMediaStream = null;
840 : }
841 :
842 0 : if (localScreenshareStream != null) {
843 0 : await stopMediaStream(localScreenshareStream!.stream);
844 0 : await _removeScreenshareStream(groupCall, localScreenshareStream!);
845 0 : _localScreenshareStream = null;
846 : }
847 :
848 : // removeCall removes it from `_callSessions` later.
849 0 : final callsCopy = _callSessions.toList();
850 :
851 0 : for (final call in callsCopy) {
852 0 : await _removeCall(groupCall, call, CallErrorCode.userHangup);
853 : }
854 :
855 0 : _activeSpeaker = null;
856 0 : _activeSpeakerLoopTimeout?.cancel();
857 0 : await _callSetupSubscription?.cancel();
858 0 : await _callStartSubscription?.cancel();
859 : }
860 :
861 0 : @override
862 : bool get isLocalVideoMuted {
863 0 : if (localUserMediaStream != null) {
864 0 : return localUserMediaStream!.isVideoMuted();
865 : }
866 :
867 : return true;
868 : }
869 :
870 0 : @override
871 : bool get isMicrophoneMuted {
872 0 : if (localUserMediaStream != null) {
873 0 : return localUserMediaStream!.isAudioMuted();
874 : }
875 :
876 : return true;
877 : }
878 :
879 2 : @override
880 : Future<void> setupP2PCallsWithExistingMembers(
881 : GroupCallSession groupCall,
882 : ) async {
883 4 : for (final call in _callSessions) {
884 2 : _onIncomingCallInMeshSetup(groupCall, call);
885 2 : await _onIncomingCallInMeshStart(groupCall, call);
886 : }
887 :
888 10 : _callSetupSubscription = groupCall.voip.onIncomingCallSetup.stream.listen(
889 0 : (newCall) => _onIncomingCallInMeshSetup(groupCall, newCall),
890 : );
891 :
892 10 : _callStartSubscription = groupCall.voip.onIncomingCallStart.stream.listen(
893 0 : (newCall) => _onIncomingCallInMeshStart(groupCall, newCall),
894 : );
895 :
896 2 : _onActiveSpeakerLoop(groupCall);
897 : }
898 :
899 2 : @override
900 : Future<void> setupP2PCallWithNewMember(
901 : GroupCallSession groupCall,
902 : CallParticipant rp,
903 : CallMembership mem,
904 : ) async {
905 2 : final existingCall = _getCallForParticipant(groupCall, rp);
906 : if (existingCall != null) {
907 0 : if (existingCall.remoteSessionId != mem.membershipId) {
908 0 : await existingCall.hangup(reason: CallErrorCode.unknownError);
909 : } else {
910 0 : Logs().e(
911 0 : '[VOIP] onMemberStateChanged Not updating _participants list, already have a ongoing call with ${rp.id}',
912 : );
913 : return;
914 : }
915 : }
916 :
917 : // Only initiate a call with a participant who has a id that is lexicographically
918 : // less than your own. Otherwise, that user will call you.
919 10 : if (groupCall.localParticipant!.id.compareTo(rp.id) > 0) {
920 8 : Logs().i('[VOIP] Waiting for ${rp.id} to send call invite.');
921 : return;
922 : }
923 :
924 2 : final opts = CallOptions(
925 2 : callId: genCallID(),
926 2 : room: groupCall.room,
927 2 : voip: groupCall.voip,
928 : dir: CallDirection.kOutgoing,
929 4 : localPartyId: groupCall.voip.currentSessionId,
930 2 : groupCallId: groupCall.groupCallId,
931 : type: CallType.kVideo,
932 4 : iceServers: await groupCall.voip.getIceServers(),
933 : );
934 4 : final newCall = groupCall.voip.createNewCall(opts);
935 :
936 : /// both invitee userId and deviceId are set here because there can be
937 : /// multiple devices from same user in a call, so we specifiy who the
938 : /// invite is for
939 : ///
940 : /// MOVE TO CREATENEWCALL?
941 4 : newCall.remoteUserId = mem.userId;
942 4 : newCall.remoteDeviceId = mem.deviceId;
943 : // party id set to when answered
944 4 : newCall.remoteSessionId = mem.membershipId;
945 :
946 2 : _registerListenersBeforeCallAdd(newCall);
947 :
948 2 : await newCall.placeCallWithStreams(
949 2 : _getLocalStreams(),
950 2 : requestScreenSharing: mem.feeds?.any(
951 0 : (element) =>
952 0 : element['purpose'] == SDPStreamMetadataPurpose.Screenshare,
953 : ) ??
954 : false,
955 : );
956 :
957 2 : await _addCall(groupCall, newCall);
958 : }
959 :
960 2 : @override
961 : List<Map<String, String>>? getCurrentFeeds() {
962 2 : return _getLocalStreams()
963 2 : .map(
964 4 : (feed) => ({
965 2 : 'purpose': feed.purpose,
966 : }),
967 : )
968 2 : .toList();
969 : }
970 :
971 0 : @override
972 : bool operator ==(Object other) =>
973 0 : identical(this, other) || (other is MeshBackend && type == other.type);
974 0 : @override
975 0 : int get hashCode => type.hashCode;
976 :
977 : /// get everything is livekit specific mesh calls shouldn't be affected by these
978 0 : @override
979 : Future<void> onCallEncryption(
980 : GroupCallSession groupCall,
981 : String userId,
982 : String deviceId,
983 : Map<String, dynamic> content,
984 : ) async {
985 : return;
986 : }
987 :
988 0 : @override
989 : Future<void> onCallEncryptionKeyRequest(
990 : GroupCallSession groupCall,
991 : String userId,
992 : String deviceId,
993 : Map<String, dynamic> content,
994 : ) async {
995 : return;
996 : }
997 :
998 0 : @override
999 : Future<void> onLeftParticipant(
1000 : GroupCallSession groupCall,
1001 : List<CallParticipant> anyLeft,
1002 : ) async {
1003 : return;
1004 : }
1005 :
1006 0 : @override
1007 : Future<void> onNewParticipant(
1008 : GroupCallSession groupCall,
1009 : List<CallParticipant> anyJoined,
1010 : ) async {
1011 : return;
1012 : }
1013 :
1014 0 : @override
1015 : Future<void> requestEncrytionKey(
1016 : GroupCallSession groupCall,
1017 : List<CallParticipant> remoteParticipants,
1018 : ) async {
1019 : return;
1020 : }
1021 :
1022 0 : @override
1023 : Future<void> preShareKey(GroupCallSession groupCall) async {
1024 : return;
1025 : }
1026 : }
|
