Add screensharing support

Signed-off-by: Mario Danic <mario@lovelyhq.com>

app/src/main/java/com/nextcloud/talk/controllers/CallController.java

@@ -1280,8 +1280,10 @@ public class CallController extends BaseController {
     }
 
     private void processMessage(NCSignalingMessage ncSignalingMessage) {
-        if (ncSignalingMessage.getRoomType().equals("video")) {
-            MagicPeerConnectionWrapper magicPeerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(ncSignalingMessage.getFrom(), false);
+        if (ncSignalingMessage.getRoomType().equals("video") || ncSignalingMessage.getRoomType().equals("screen")) {
+            MagicPeerConnectionWrapper magicPeerConnectionWrapper =
+                    getPeerConnectionWrapperForSessionIdAndType(ncSignalingMessage.getFrom(),
+                            ncSignalingMessage.getRoomType(),false);
 
             String type = null;
             if (ncSignalingMessage.getPayload() != null && ncSignalingMessage.getPayload().getType() != null) {
@@ -1292,6 +1294,9 @@ public class CallController extends BaseController {
 
             if (type != null) {
                 switch (type) {
+                    case "unshareScreen":
+                        endPeerConnection(ncSignalingMessage.getFrom(), true);
+                        break;
                     case "offer":
                     case "answer":
                         magicPeerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick());
@@ -1344,7 +1349,7 @@ public class CallController extends BaseController {
         }
 
         for (int i = 0; i < magicPeerConnectionWrapperList.size(); i++) {
-            endPeerConnection(magicPeerConnectionWrapperList.get(i).getSessionId());
+            endPeerConnection(magicPeerConnectionWrapperList.get(i).getSessionId(), false);
 
         }
 
@@ -1500,11 +1505,11 @@ public class CallController extends BaseController {
         hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient.hasMCU();
 
         for (String sessionId : newSessions) {
-            alwaysGetPeerConnectionWrapperForSessionId(sessionId, hasMCU && sessionId.equals(webSocketClient.getSessionId()));
+            getPeerConnectionWrapperForSessionIdAndType(sessionId, "video",hasMCU && sessionId.equals(webSocketClient.getSessionId()));
         }
 
         for (String sessionId : oldSesssions) {
-            endPeerConnection(sessionId);
+            endPeerConnection(sessionId, false);
         }
     }
 
@@ -1545,20 +1550,35 @@ public class CallController extends BaseController {
         magicPeerConnectionWrapperList.remove(magicPeerConnectionWrapper);
     }
 
-    private MagicPeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean publisher) {
+    private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId, String type) {
+        for (int i = 0; i < magicPeerConnectionWrapperList.size(); i++) {
+            if (magicPeerConnectionWrapperList.get(i).getSessionId().equals(sessionId) && magicPeerConnectionWrapperList.get(i).getVideoStreamType().equals(type)) {
+                return magicPeerConnectionWrapperList.get(i);
+            }
+        }
+
+        return null;
+    }
+
+    private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionIdAndType(String sessionId, String type, boolean publisher) {
         MagicPeerConnectionWrapper magicPeerConnectionWrapper;
-        if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
+        if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId, type)) != null) {
             return magicPeerConnectionWrapper;
         } else {
             if (hasMCU && publisher) {
                 magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
-                        iceServers, sdpConstraintsForMCU, sessionId, callSession, localMediaStream, true, true);
+                        iceServers, sdpConstraintsForMCU, sessionId, callSession, localMediaStream, true, true, type);
             } else if (hasMCU) {
                 magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
-                        iceServers, sdpConstraints, sessionId, callSession, null, false, true);
+                        iceServers, sdpConstraints, sessionId, callSession, null, false, true, type);
             } else {
-                magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
-                        iceServers, sdpConstraints, sessionId, callSession, localMediaStream, false, false);
+                if (!type.equals("screen")) {
+                    magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
+                            iceServers, sdpConstraints, sessionId, callSession, localMediaStream, false, false, type);
+                } else {
+                    magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
+                            iceServers, sdpConstraints, sessionId, callSession, null, false, false, type);
+                }
             }
 
             magicPeerConnectionWrapperList.add(magicPeerConnectionWrapper);
@@ -1566,21 +1586,38 @@ public class CallController extends BaseController {
         }
     }
 
-    private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) {
+    private List<MagicPeerConnectionWrapper> getPeerConnectionWrapperListForSessionId(String sessionId) {
+        List<MagicPeerConnectionWrapper> internalList = new ArrayList<>();
         for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) {
             if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) {
-                return magicPeerConnectionWrapper;
+                internalList.add(magicPeerConnectionWrapper);
             }
         }
-        return null;
+
+        return internalList;
     }
 
-    private void endPeerConnection(String sessionId) {
+    private void endPeerConnection(String sessionId, boolean justScreen) {
+        List<MagicPeerConnectionWrapper> magicPeerConnectionWrappers;
         MagicPeerConnectionWrapper magicPeerConnectionWrapper;
-        if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null && getActivity()
+        if (!(magicPeerConnectionWrappers = getPeerConnectionWrapperListForSessionId(sessionId)).isEmpty() && getActivity()
                 != null) {
-            getActivity().runOnUiThread(() -> removeMediaStream(sessionId));
-            deleteMagicPeerConnection(magicPeerConnectionWrapper);
+            for (int i = 0; i < magicPeerConnectionWrappers.size(); i++) {
+                magicPeerConnectionWrapper = magicPeerConnectionWrappers.get(i);
+                if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) {
+                    if (magicPeerConnectionWrapper.getVideoStreamType().equals("screen")) {
+                        MagicPeerConnectionWrapper finalMagicPeerConnectionWrapper = magicPeerConnectionWrapper;
+                        getActivity().runOnUiThread(() -> removeMediaStream(sessionId + "+" +
+                                finalMagicPeerConnectionWrapper.getVideoStreamType()));
+                        deleteMagicPeerConnection(magicPeerConnectionWrapper);
+                    } else if (!justScreen) {
+                        MagicPeerConnectionWrapper finalMagicPeerConnectionWrapper = magicPeerConnectionWrapper;
+                        getActivity().runOnUiThread(() -> removeMediaStream(sessionId + "+" +
+                                finalMagicPeerConnectionWrapper.getVideoStreamType()));
+                        deleteMagicPeerConnection(magicPeerConnectionWrapper);
+                    }
+                }
+            }
         }
     }
 
@@ -1613,7 +1650,7 @@ public class CallController extends BaseController {
     public void onMessageEvent(PeerConnectionEvent peerConnectionEvent) {
         if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent.PeerConnectionEventType
                 .PEER_CLOSED)) {
-            endPeerConnection(peerConnectionEvent.getSessionId());
+            endPeerConnection(peerConnectionEvent.getSessionId(), false);
         } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
                 .PeerConnectionEventType.SENSOR_FAR) ||
                 peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
@@ -1630,14 +1667,14 @@ public class CallController extends BaseController {
             }
         } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
                 .PeerConnectionEventType.NICK_CHANGE)) {
-            gotNick(peerConnectionEvent.getSessionId(), peerConnectionEvent.getNick(), true);
+            gotNick(peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(), peerConnectionEvent.getNick(), true);
         } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
                 .PeerConnectionEventType.VIDEO_CHANGE) && !isVoiceOnlyCall) {
-            gotAudioOrVideoChange(true, peerConnectionEvent.getSessionId(),
+            gotAudioOrVideoChange(true, peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(),
                     peerConnectionEvent.getChangeValue());
         } else if (peerConnectionEvent.getPeerConnectionEventType().equals(PeerConnectionEvent
                 .PeerConnectionEventType.AUDIO_CHANGE)) {
-            gotAudioOrVideoChange(false, peerConnectionEvent.getSessionId(),
+            gotAudioOrVideoChange(false, peerConnectionEvent.getSessionId() + "+" + peerConnectionEvent.getVideoStreamType(),
                     peerConnectionEvent.getChangeValue());
         }
     }
@@ -1668,9 +1705,9 @@ public class CallController extends BaseController {
         if (mediaStreamEvent.getMediaStream() != null) {
             setupVideoStreamForLayout(mediaStreamEvent.getMediaStream(), mediaStreamEvent.getSession(),
                     mediaStreamEvent.getMediaStream().videoTracks != null
-                            && mediaStreamEvent.getMediaStream().videoTracks.size() > 0);
+                            && mediaStreamEvent.getMediaStream().videoTracks.size() > 0, mediaStreamEvent.getVideoStreamType());
         } else {
-            setupVideoStreamForLayout(null, mediaStreamEvent.getSession(), false);
+            setupVideoStreamForLayout(null, mediaStreamEvent.getSession(), false, mediaStreamEvent.getVideoStreamType());
         }
     }
 
@@ -1681,7 +1718,7 @@ public class CallController extends BaseController {
         ncMessageWrapper.setSessionId(callSession);
         NCSignalingMessage ncSignalingMessage = new NCSignalingMessage();
         ncSignalingMessage.setTo(sessionDescriptionSend.getPeerId());
-        ncSignalingMessage.setRoomType("video");
+        ncSignalingMessage.setRoomType(sessionDescriptionSend.getVideoStreamType());
         ncSignalingMessage.setType(sessionDescriptionSend.getType());
         NCMessagePayload ncMessagePayload = new NCMessagePayload();
         ncMessagePayload.setType(sessionDescriptionSend.getType());
@@ -1768,7 +1805,7 @@ public class CallController extends BaseController {
 
     private void setupAvatarForSession(String session) {
         if (remoteRenderersLayout != null) {
-            RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session);
+            RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session + "+video");
             if (relativeLayout != null) {
                 ImageView avatarImageView = relativeLayout.findViewById(R.id.avatarImageView);
 
@@ -1791,14 +1828,14 @@ public class CallController extends BaseController {
         }
     }
 
-    private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, String session, boolean enable) {
+    private void setupVideoStreamForLayout(@Nullable MediaStream mediaStream, String session, boolean enable, String videoStreamType) {
         boolean isInitialLayoutSetupForPeer = false;
         if (remoteRenderersLayout.findViewWithTag(session) == null) {
-            setupNewPeerLayout(session);
+            setupNewPeerLayout(session, videoStreamType);
             isInitialLayoutSetupForPeer = true;
         }
 
-        RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session);
+        RelativeLayout relativeLayout = remoteRenderersLayout.findViewWithTag(session + "+" + videoStreamType);
         SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id.surface_view);
         ImageView imageView = relativeLayout.findViewById(R.id.avatarImageView);
 
@@ -1850,13 +1887,13 @@ public class CallController extends BaseController {
         }
     }
 
-    private void setupNewPeerLayout(String session) {
-        if (remoteRenderersLayout.findViewWithTag(session) == null && getActivity() != null) {
+    private void setupNewPeerLayout(String session, String type) {
+        if (remoteRenderersLayout.findViewWithTag(session + "+" + type) == null && getActivity() != null) {
             getActivity().runOnUiThread(() -> {
                 RelativeLayout relativeLayout = (RelativeLayout)
                         getActivity().getLayoutInflater().inflate(R.layout.call_item, remoteRenderersLayout,
                                 false);
-                relativeLayout.setTag(session);
+                relativeLayout.setTag(session + "+" + type);
                 SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id
                         .surface_view);
 
@@ -1869,12 +1906,14 @@ public class CallController extends BaseController {
                 surfaceViewRenderer.setOnClickListener(videoOnClickListener);
                 remoteRenderersLayout.addView(relativeLayout);
                 if (hasExternalSignalingServer) {
-                    gotNick(session, webSocketClient.getDisplayNameForSession(session), false);
+                    gotNick(session + "+" + type, webSocketClient.getDisplayNameForSession(session), false);
                 } else {
-                    gotNick(session, getPeerConnectionWrapperForSessionId(session).getNick(), false);
+                    gotNick(session + "+" + type, getPeerConnectionWrapperForSessionIdAndType(session, "video", false).getNick(), false);
                 }
 
-                setupAvatarForSession(session);
+                if (type.equals("video")) {
+                    setupAvatarForSession(session);
+                }
 
                 callControls.setZ(100.0f);
             });

app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java

@@ -29,9 +29,11 @@ import lombok.Data;
 public class MediaStreamEvent {
     private final MediaStream mediaStream;
     private final String session;
+    private final String videoStreamType;
 
-    public MediaStreamEvent(@Nullable MediaStream mediaStream, String session) {
+    public MediaStreamEvent(@Nullable MediaStream mediaStream, String session, String videoStreamType) {
         this.mediaStream = mediaStream;
         this.session = session;
+        this.videoStreamType = videoStreamType;
     }
 }

app/src/main/java/com/nextcloud/talk/events/PeerConnectionEvent.java

@@ -29,13 +29,15 @@ public class PeerConnectionEvent {
     private final String sessionId;
     private final String nick;
     private final Boolean changeValue;
+    private final String videoStreamType;
 
     public PeerConnectionEvent(PeerConnectionEventType peerConnectionEventType, @Nullable String sessionId,
-                               @Nullable String nick, Boolean changeValue) {
+                               @Nullable String nick, Boolean changeValue, @Nullable String videoStreamType) {
         this.peerConnectionEventType = peerConnectionEventType;
         this.nick = nick;
         this.changeValue = changeValue;
         this.sessionId = sessionId;
+        this.videoStreamType = videoStreamType;
     }
 
     public enum PeerConnectionEventType {

app/src/main/java/com/nextcloud/talk/events/SessionDescriptionSendEvent.java

@@ -35,12 +35,14 @@ public class SessionDescriptionSendEvent {
     private final String type;
     @Nullable
     private final NCIceCandidate ncIceCandidate;
+    private final String videoStreamType;
 
     public SessionDescriptionSendEvent(@Nullable SessionDescription sessionDescription, String peerId, String type,
-                                       @Nullable NCIceCandidate ncIceCandidate) {
+                                       @Nullable NCIceCandidate ncIceCandidate, @Nullable String videoStreamType) {
         this.sessionDescription = sessionDescription;
         this.peerId = peerId;
         this.type = type;
         this.ncIceCandidate = ncIceCandidate;
+        this.videoStreamType = videoStreamType;
     }
 }

app/src/main/java/com/nextcloud/talk/webrtc/MagicAudioManager.java

@@ -176,7 +176,7 @@ public class MagicAudioManager {
                 setAudioDeviceInternal(MagicAudioManager.AudioDevice.EARPIECE);
 
                 EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                        .SENSOR_NEAR, null, null, null));
+                        .SENSOR_NEAR, null, null, null, null));
 
             } else {
                 // Sensor reports that a "handset is removed from a person's ear", or
@@ -184,7 +184,7 @@ public class MagicAudioManager {
                 setAudioDeviceInternal(MagicAudioManager.AudioDevice.SPEAKER_PHONE);
 
                 EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                        .SENSOR_FAR, null, null, null));
+                        .SENSOR_FAR, null, null, null, null));
             }
         }
     }

app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java

@@ -71,14 +71,16 @@ public class MagicPeerConnectionWrapper {
 
     private MediaStream localMediaStream;
     private boolean isMCUPublisher;
+    private String videoStreamType;
 
     public MagicPeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory,
                                       List<PeerConnection.IceServer> iceServerList,
                                       MediaConstraints mediaConstraints,
                                       String sessionId, String localSession, @Nullable MediaStream mediaStream,
-                                      boolean isMCUPublisher, boolean hasMCU) {
+                                      boolean isMCUPublisher, boolean hasMCU, String videoStreamType) {
 
         this.localMediaStream = mediaStream;
+        this.videoStreamType = videoStreamType;
 
         this.sessionId = sessionId;
         this.mediaConstraints = mediaConstraints;
@@ -114,6 +116,10 @@ public class MagicPeerConnectionWrapper {
         }
     }
 
+    public String getVideoStreamType() {
+        return videoStreamType;
+    }
+
     public void removePeerConnection() {
         if (magicDataChannel != null) {
             magicDataChannel.dispose();
@@ -243,32 +249,32 @@ public class MagicPeerConnectionWrapper {
                         if (!internalNick.equals(nick)) {
                             setNick(nick);
                             EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                                    .NICK_CHANGE, sessionId, getNick(), null));
+                                    .NICK_CHANGE, sessionId, getNick(), null, videoStreamType));
                         }
                     } else {
                         if (dataChannelMessage.getPayload() != null) {
                             HashMap<String, String> payloadHashMap = (HashMap<String, String>) dataChannelMessage.getPayload();
                             EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                                    .NICK_CHANGE, payloadHashMap.get("userid"), payloadHashMap.get("name"), null));
+                                    .NICK_CHANGE, payloadHashMap.get("userid"), payloadHashMap.get("name"), null, videoStreamType));
                         }
                     }
 
                 } else if ("audioOn".equals(dataChannelMessage.getType())) {
                     remoteAudioOn = true;
                     EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                            .AUDIO_CHANGE, sessionId, null, remoteAudioOn));
+                            .AUDIO_CHANGE, sessionId, null, remoteAudioOn, videoStreamType));
                 } else if ("audioOff".equals(dataChannelMessage.getType())) {
                     remoteAudioOn = false;
                     EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                            .AUDIO_CHANGE, sessionId, null, remoteAudioOn));
+                            .AUDIO_CHANGE, sessionId, null, remoteAudioOn, videoStreamType));
                 } else if ("videoOn".equals(dataChannelMessage.getType())) {
                     remoteVideoOn = true;
                     EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                            .VIDEO_CHANGE, sessionId, null, remoteVideoOn));
+                            .VIDEO_CHANGE, sessionId, null, remoteVideoOn, videoStreamType));
                 } else if ("videoOff".equals(dataChannelMessage.getType())) {
                     remoteVideoOn = false;
                     EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                            .VIDEO_CHANGE, sessionId, null, remoteVideoOn));
+                            .VIDEO_CHANGE, sessionId, null, remoteVideoOn, videoStreamType));
                 }
             } catch (IOException e) {
                 Log.d(TAG, "Failed to parse data channel message");
@@ -290,7 +296,7 @@ public class MagicPeerConnectionWrapper {
                         .PEER_CONNECTED, sessionId, null, null));*/
 
                 if (!isMCUPublisher) {
-                    EventBus.getDefault().post(new MediaStreamEvent(remoteMediaStream, sessionId));
+                    EventBus.getDefault().post(new MediaStreamEvent(remoteMediaStream, sessionId, videoStreamType));
                 }
 
                 if (hasInitiated) {
@@ -299,7 +305,7 @@ public class MagicPeerConnectionWrapper {
 
             } else if (iceConnectionState.equals(PeerConnection.IceConnectionState.CLOSED)) {
                 EventBus.getDefault().post(new PeerConnectionEvent(PeerConnectionEvent.PeerConnectionEventType
-                        .PEER_CLOSED, sessionId, null, null));
+                        .PEER_CLOSED, sessionId, null, null, videoStreamType));
             }
         }
 
@@ -320,7 +326,7 @@ public class MagicPeerConnectionWrapper {
             ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex);
             ncIceCandidate.setCandidate(iceCandidate.sdp);
             EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId,
-                    "candidate", ncIceCandidate));
+                    "candidate", ncIceCandidate, videoStreamType));
         }
 
         @Override
@@ -336,7 +342,7 @@ public class MagicPeerConnectionWrapper {
         @Override
         public void onRemoveStream(MediaStream mediaStream) {
             if (!isMCUPublisher) {
-                EventBus.getDefault().post(new MediaStreamEvent(null, sessionId));
+                EventBus.getDefault().post(new MediaStreamEvent(null, sessionId, videoStreamType));
             }
         }
 
@@ -383,7 +389,7 @@ public class MagicPeerConnectionWrapper {
 
 
             EventBus.getDefault().post(new SessionDescriptionSendEvent(sessionDescriptionWithPreferredCodec, sessionId,
-                    sessionDescription.type.canonicalForm().toLowerCase(), null));
+                    sessionDescription.type.canonicalForm().toLowerCase(), null, videoStreamType));
 
             if (peerConnection != null) {
                 peerConnection.setLocalDescription(magicSdpObserver, sessionDescriptionWithPreferredCodec);

app/src/main/java/com/nextcloud/talk/webrtc/MagicWebRTCUtils.java

@@ -129,7 +129,7 @@ public class MagicWebRTCUtils {
         }
         Log.d(TAG, "Change media description from: " + lines[mLineIndex] + " to " + newMLine);
         lines[mLineIndex] = newMLine;
-        return joinString(Arrays.asList(lines), "\r\n", true /* delimiterAtEnd */);
+        return joinString(Arrays.asList(lines), "\r\n", true);
     }
 
     /**