Progress

Signed-off-by: Mario Danic <mario@lovelyhq.com>

app/src/main/java/com/nextcloud/talk/activities/CallActivity.java

@@ -34,6 +34,8 @@ import android.util.TypedValue;
 import android.view.View;
 import android.view.Window;
 import android.view.WindowManager;
+import android.widget.GridLayout;
+import android.widget.RelativeLayout;
 
 import com.bluelinelabs.logansquare.LoganSquare;
 import com.nextcloud.talk.R;
@@ -52,7 +54,7 @@ import com.nextcloud.talk.events.MediaStreamEvent;
 import com.nextcloud.talk.events.SessionDescriptionSendEvent;
 import com.nextcloud.talk.persistence.entities.UserEntity;
 import com.nextcloud.talk.webrtc.MagicAudioManager;
-import com.nextcloud.talk.webrtc.PeerConnectionWrapper;
+import com.nextcloud.talk.webrtc.MagicPeerConnectionWrapper;
 
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.greenrobot.eventbus.EventBus;
@@ -104,8 +106,8 @@ public class CallActivity extends AppCompatActivity {
     @BindView(R.id.pip_video_view)
     SurfaceViewRenderer pipVideoView;
 
-    @BindView(R.id.fullscreen_video_view)
-    SurfaceViewRenderer fullScreenVideoView;
+    @BindView(R.id.videos_grid_view)
+    GridLayout videosGrid;
 
     @Inject
     NcApi ncApi;
@@ -125,7 +127,9 @@ public class CallActivity extends AppCompatActivity {
     VideoCapturer videoCapturer;
     VideoRenderer localRenderer;
     VideoRenderer remoteRenderer;
+    HashMap<String, VideoRenderer> videoRendererHashMap = new HashMap<>();
     PeerConnection localPeer;
+    EglBase rootEglBase;
     boolean leavingCall = false;
     BooleanSupplier booleanSupplier = () -> leavingCall;
     Disposable signalingDisposable;
@@ -138,7 +142,7 @@ public class CallActivity extends AppCompatActivity {
     private MediaStream localMediaStream;
 
     private String credentials;
-    private List<PeerConnectionWrapper> peerConnectionWrapperList = new ArrayList<>();
+    private List<MagicPeerConnectionWrapper> magicPeerConnectionWrapperList = new ArrayList<>();
 
     private static int getSystemUiVisibility() {
         int flags = View.SYSTEM_UI_FLAG_HIDE_NAVIGATION | View.SYSTEM_UI_FLAG_FULLSCREEN;
@@ -224,16 +228,11 @@ public class CallActivity extends AppCompatActivity {
 
     public void initViews() {
         pipVideoView.setMirror(true);
-        fullScreenVideoView.setMirror(false);
-        EglBase rootEglBase = EglBase.create();
+        rootEglBase = EglBase.create();
         pipVideoView.init(rootEglBase.getEglBaseContext(), null);
         pipVideoView.setZOrderMediaOverlay(true);
-        fullScreenVideoView.init(rootEglBase.getEglBaseContext(), null);
-        fullScreenVideoView.setZOrderMediaOverlay(true);
-        fullScreenVideoView.setEnableHardwareScaler(true);
         pipVideoView.setEnableHardwareScaler(true);
         pipVideoView.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT);
-        fullScreenVideoView.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL);
     }
 
     public void start() {
@@ -435,7 +434,7 @@ public class CallActivity extends AppCompatActivity {
             NCSignalingMessage ncSignalingMessage = LoganSquare.parse(signaling.getMessageWrapper().toString(),
                     NCSignalingMessage.class);
             if (ncSignalingMessage.getRoomType().equals("video")) {
-                PeerConnectionWrapper peerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId
+                MagicPeerConnectionWrapper magicPeerConnectionWrapper = alwaysGetPeerConnectionWrapperForSessionId
                         (ncSignalingMessage.getFrom(), ncSignalingMessage.getFrom().equals(callSession));
 
                 String type = null;
@@ -450,8 +449,8 @@ public class CallActivity extends AppCompatActivity {
                     switch (type) {
                         case "offer":
                         case "answer":
-                            peerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick());
-                            peerConnectionWrapper.getPeerConnection().setRemoteDescription(peerConnectionWrapper
+                            magicPeerConnectionWrapper.setNick(ncSignalingMessage.getPayload().getNick());
+                            magicPeerConnectionWrapper.getPeerConnection().setRemoteDescription(magicPeerConnectionWrapper
                                     .getMagicSdpObserver(), new SessionDescription(SessionDescription.Type.fromCanonicalForm(type),
                                     ncSignalingMessage.getPayload().getSdp()));
                             break;
@@ -459,11 +458,11 @@ public class CallActivity extends AppCompatActivity {
                             NCIceCandidate ncIceCandidate = ncSignalingMessage.getPayload().getIceCandidate();
                             IceCandidate iceCandidate = new IceCandidate(ncIceCandidate.getSdpMid(),
                                     ncIceCandidate.getSdpMLineIndex(), ncIceCandidate.getCandidate());
-                            peerConnectionWrapper.addCandidate(iceCandidate);
+                            magicPeerConnectionWrapper.addCandidate(iceCandidate);
                             break;
                         case "endOfCandidates":
-                            peerConnectionWrapper.drainIceCandidates();
-                            peerConnectionWrapper.sendLocalCandidates();
+                            magicPeerConnectionWrapper.drainIceCandidates();
+                            magicPeerConnectionWrapper.sendLocalCandidates();
                             break;
                         default:
                             break;
@@ -496,9 +495,9 @@ public class CallActivity extends AppCompatActivity {
             }
         }
 
-        for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
-            if (!peerConnectionWrapper.getSessionId().equals(callSession)) {
-                oldSesssions.add(peerConnectionWrapper.getSessionId());
+        for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) {
+            if (!magicPeerConnectionWrapper.getSessionId().equals(callSession)) {
+                oldSesssions.add(magicPeerConnectionWrapper.getSessionId());
             }
         }
 
@@ -513,12 +512,12 @@ public class CallActivity extends AppCompatActivity {
             return;
         }
 
-        PeerConnectionWrapper peerConnectionWrapper;
+        MagicPeerConnectionWrapper magicPeerConnectionWrapper;
 
         for (String sessionId : newSessions) {
             if (getPeerConnectionWrapperForSessionId(sessionId) == null) {
                 if (sessionId.compareTo(callSession) < 0) {
-                    PeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId,
+                    MagicPeerConnectionWrapper connectionWrapper = alwaysGetPeerConnectionWrapperForSessionId(sessionId,
                             false);
                     if (connectionWrapper.getPeerConnection() != null) {
                         connectionWrapper.getPeerConnection().createOffer(connectionWrapper.getMagicSdpObserver(),
@@ -532,33 +531,33 @@ public class CallActivity extends AppCompatActivity {
         }
 
         for (String sessionId : leftSessions) {
-            if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
-                if (peerConnectionWrapper.getPeerConnection() != null) {
-                    peerConnectionWrapper.getPeerConnection().close();
+            if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
+                if (magicPeerConnectionWrapper.getPeerConnection() != null) {
+                    magicPeerConnectionWrapper.getPeerConnection().close();
                 }
-                peerConnectionWrapperList.remove(peerConnectionWrapper);
+                magicPeerConnectionWrapperList.remove(magicPeerConnectionWrapper);
             }
         }
     }
 
 
-    private PeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) {
-        PeerConnectionWrapper peerConnectionWrapper;
-        if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
-            return peerConnectionWrapper;
+    private MagicPeerConnectionWrapper alwaysGetPeerConnectionWrapperForSessionId(String sessionId, boolean isLocalPeer) {
+        MagicPeerConnectionWrapper magicPeerConnectionWrapper;
+        if ((magicPeerConnectionWrapper = getPeerConnectionWrapperForSessionId(sessionId)) != null) {
+            return magicPeerConnectionWrapper;
         } else {
-            peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory,
+            magicPeerConnectionWrapper = new MagicPeerConnectionWrapper(peerConnectionFactory,
                     iceServers, sdpConstraints, sessionId);
-            peerConnectionWrapper.getPeerConnection().addStream(localMediaStream);
-            peerConnectionWrapperList.add(peerConnectionWrapper);
-            return peerConnectionWrapper;
+            magicPeerConnectionWrapper.getPeerConnection().addStream(localMediaStream);
+            magicPeerConnectionWrapperList.add(magicPeerConnectionWrapper);
+            return magicPeerConnectionWrapper;
         }
     }
 
-    private PeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) {
-        for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
-            if (peerConnectionWrapper.getSessionId().equals(sessionId)) {
-                return peerConnectionWrapper;
+    private MagicPeerConnectionWrapper getPeerConnectionWrapperForSessionId(String sessionId) {
+        for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) {
+            if (magicPeerConnectionWrapper.getSessionId().equals(sessionId)) {
+                return magicPeerConnectionWrapper;
             }
         }
         return null;
@@ -570,9 +569,9 @@ public class CallActivity extends AppCompatActivity {
 
         dispose(null);
 
-        for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
-            if (peerConnectionWrapper.getPeerConnection() != null) {
-                peerConnectionWrapper.getPeerConnection().close();
+        for (MagicPeerConnectionWrapper magicPeerConnectionWrapper : magicPeerConnectionWrapperList) {
+            if (magicPeerConnectionWrapper.getPeerConnection() != null) {
+                magicPeerConnectionWrapper.getPeerConnection().close();
             }
         }
 
@@ -581,7 +580,6 @@ public class CallActivity extends AppCompatActivity {
         }
 
         pipVideoView.release();
-        fullScreenVideoView.release();
 
         String credentials = ApiHelper.getCredentials(userEntity.getUsername(), userEntity.getToken());
         ncApi.leaveCall(credentials, ApiHelper.getUrlForCall(userEntity.getBaseUrl(), roomToken))
@@ -610,7 +608,7 @@ public class CallActivity extends AppCompatActivity {
                 });
     }
 
-    private void gotRemoteStream(MediaStream stream) {
+    private void gotRemoteStream(MediaStream stream, String session) {
         //we have remote video stream. add to the renderer.
         if (stream.videoTracks.size() < 2 && stream.audioTracks.size() < 2) {
             if (stream.videoTracks.size() == 1) {
@@ -621,8 +619,26 @@ public class CallActivity extends AppCompatActivity {
                     public void run() {
                         if (stream.videoTracks.size() == 1) {
                             try {
-                                remoteRenderer = new VideoRenderer(fullScreenVideoView);
+                                RelativeLayout relativeLayout = (RelativeLayout)
+                                        getLayoutInflater().inflate(R.layout.surface_renderer, videosGrid,
+                                        false);
+                                relativeLayout.setTag(session);
+                                SurfaceViewRenderer surfaceViewRenderer = relativeLayout.findViewById(R.id
+                                        .surface_view);
+                                surfaceViewRenderer.setMirror(false);
+                                surfaceViewRenderer.init(rootEglBase.getEglBaseContext(), null);
+                                surfaceViewRenderer.setZOrderMediaOverlay(true);
+                                surfaceViewRenderer.setEnableHardwareScaler(true);
+                                surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL);
+                                VideoRenderer remoteRenderer = new VideoRenderer(surfaceViewRenderer);
+                                videoRendererHashMap.put(session, remoteRenderer);
                                 videoTrack.addRenderer(remoteRenderer);
+                                videosGrid.addView(relativeLayout);
+
+                                GridLayout.LayoutParams param = new GridLayout.LayoutParams(GridLayout.spec(
+                                        GridLayout.UNDEFINED,GridLayout.FILL,1f),
+                                        GridLayout.spec(GridLayout.UNDEFINED,GridLayout.FILL,1f));
+                                relativeLayout.setLayoutParams(param);
                             } catch (Exception e) {
                                 e.printStackTrace();
                             }
@@ -676,7 +692,7 @@ public class CallActivity extends AppCompatActivity {
 
     @Subscribe(threadMode = ThreadMode.BACKGROUND)
     public void onMessageEvent(MediaStreamEvent mediaStreamEvent) {
-        gotRemoteStream(mediaStreamEvent.getMediaStream());
+        gotRemoteStream(mediaStreamEvent.getMediaStream(), mediaStreamEvent.getSession());
     }
 
     @Subscribe(threadMode = ThreadMode.BACKGROUND)

app/src/main/java/com/nextcloud/talk/api/models/json/signaling/DataChannelMessage.java

@@ -33,4 +33,11 @@ public class DataChannelMessage {
 
     @JsonField(name = "payload")
     String payload;
+
+    public DataChannelMessage(String type) {
+        this.type = type;
+    }
+
+    public DataChannelMessage() {
+    }
 }

app/src/main/java/com/nextcloud/talk/events/MediaStreamEvent.java

@@ -27,8 +27,10 @@ import lombok.Data;
 @Data
 public class MediaStreamEvent {
     private final MediaStream mediaStream;
+    private final String session;
 
-    public MediaStreamEvent(MediaStream mediaStream) {
+    public MediaStreamEvent(MediaStream mediaStream, String session) {
         this.mediaStream = mediaStream;
+        this.session = session;
     }
 }

app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionObserver.java

@@ -20,8 +20,6 @@
 
 package com.nextcloud.talk.webrtc;
 
-import android.util.Log;
-
 import org.webrtc.DataChannel;
 import org.webrtc.IceCandidate;
 import org.webrtc.MediaStream;
@@ -35,7 +33,6 @@ public class MagicPeerConnectionObserver implements PeerConnection.Observer {
 
     @Override
     public void onSignalingChange(PeerConnection.SignalingState signalingState) {
-        Log.d("MARIO", signalingState.name());
     }
 
     @Override

app/src/main/java/com/nextcloud/talk/webrtc/PeerConnectionWrapper.java → app/src/main/java/com/nextcloud/talk/webrtc/MagicPeerConnectionWrapper.java

@@ -22,6 +22,7 @@ package com.nextcloud.talk.webrtc;
 
 import android.util.Log;
 
+import com.bluelinelabs.logansquare.LoganSquare;
 import com.nextcloud.talk.api.models.json.signaling.DataChannelMessage;
 import com.nextcloud.talk.api.models.json.signaling.NCIceCandidate;
 import com.nextcloud.talk.events.MediaStreamEvent;
@@ -36,12 +37,13 @@ import org.webrtc.PeerConnection;
 import org.webrtc.PeerConnectionFactory;
 import org.webrtc.SessionDescription;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
-public class PeerConnectionWrapper {
-    private static String TAG = "PeerConnectionWrapper";
+public class MagicPeerConnectionWrapper {
+    private static String TAG = "MagicPeerConnectionWrapper";
     private static PeerConnection peerConnection;
     List<IceCandidate> iceCandidates = new ArrayList<>();
     List<PeerConnection.IceServer> iceServers;
@@ -49,39 +51,21 @@ public class PeerConnectionWrapper {
     private String sessionId;
     private String nick;
     private MediaConstraints mediaConstraints;
-    private DataChannel dataChannel;
+    private DataChannel magicDataChannel;
     private MagicSdpObserver magicSdpObserver;
     private MagicPeerConnectionObserver magicPeerConnectionObserver;
 
-    public PeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory,
-                                 List<PeerConnection.IceServer> iceServerList,
-                                 MediaConstraints mediaConstraints,
-                                 String sessionId) {
+    public MagicPeerConnectionWrapper(PeerConnectionFactory peerConnectionFactory,
+                                      List<PeerConnection.IceServer> iceServerList,
+                                      MediaConstraints mediaConstraints,
+                                      String sessionId) {
 
         this.iceServers = iceServerList;
 
         magicPeerConnectionObserver = new MagicPeerConnectionObserver() {
-            @Override
-            public void onIceConnectionReceivingChange(boolean b) {
-
-            }
-
-            @Override
-            public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
-            }
-
             @Override
             public void onAddStream(MediaStream mediaStream) {
-                EventBus.getDefault().post(new MediaStreamEvent(mediaStream));
-            }
-
-            @Override
-            public void onSignalingChange(PeerConnection.SignalingState signalingState) {
-            }
-
-
-            @Override
-            public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) {
+                EventBus.getDefault().post(new MediaStreamEvent(mediaStream, sessionId));
             }
 
             @Override
@@ -90,12 +74,15 @@ public class PeerConnectionWrapper {
                 ncIceCandidate.setSdpMid(iceCandidate.sdpMid);
                 ncIceCandidate.setSdpMLineIndex(iceCandidate.sdpMLineIndex);
                 ncIceCandidate.setCandidate(iceCandidate.sdp);
-                if (peerConnection.getRemoteDescription() == null) {
+                /*if (peerConnection.getRemoteDescription() == null) {
                     localCandidates.add(ncIceCandidate);
                 } else {
                     EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId,
                             "candidate", ncIceCandidate));
-                }
+                }*/
+                EventBus.getDefault().post(new SessionDescriptionSendEvent(null, sessionId,
+                        "candidate", ncIceCandidate));
+
             }
 
         };
@@ -103,6 +90,11 @@ public class PeerConnectionWrapper {
         peerConnection = peerConnectionFactory.createPeerConnection(iceServerList, mediaConstraints,
                 magicPeerConnectionObserver);
 
+        DataChannel.Init init = new DataChannel.Init();
+        init.negotiated = false;
+        magicDataChannel = peerConnection.createDataChannel("status", init);
+        magicDataChannel.registerObserver(new MagicDataChannelObserver());
+
         this.sessionId = sessionId;
         this.mediaConstraints = mediaConstraints;
 
@@ -120,6 +112,7 @@ public class PeerConnectionWrapper {
             @Override
             public void onCreateSuccess(SessionDescription sessionDescription) {
                 super.onCreateSuccess(sessionDescription);
+
                 peerConnection.setLocalDescription(magicSdpObserver, sessionDescription);
             }
 
@@ -181,8 +174,13 @@ public class PeerConnectionWrapper {
     }
 
     private void sendChannelData(DataChannelMessage dataChannelMessage) {
-        ByteBuffer buffer = ByteBuffer.wrap(dataChannelMessage.toString().getBytes());
-        dataChannel.send(new DataChannel.Buffer(buffer, false));
+        ByteBuffer buffer = null;
+        try {
+            buffer = ByteBuffer.wrap(LoganSquare.serialize(dataChannelMessage).getBytes());
+            magicDataChannel.send(new DataChannel.Buffer(buffer, false));
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
     }
 
 
@@ -191,7 +189,7 @@ public class PeerConnectionWrapper {
     }
 
     public static void setPeerConnection(PeerConnection peerConnection) {
-        PeerConnectionWrapper.peerConnection = peerConnection;
+        MagicPeerConnectionWrapper.peerConnection = peerConnection;
     }
 
     public String getSessionId() {
@@ -209,4 +207,35 @@ public class PeerConnectionWrapper {
     public void setNick(String nick) {
         this.nick = nick;
     }
+
+    private class MagicDataChannelObserver implements DataChannel.Observer {
+
+        @Override
+        public void onBufferedAmountChange(long l) {
+
+        }
+
+        @Override
+        public void onStateChange() {
+            if (magicDataChannel.state().equals(DataChannel.State.OPEN) &&
+                    magicDataChannel.label().equals("status")) {
+                sendChannelData(new DataChannelMessage("videoOn"));
+                sendChannelData(new DataChannelMessage("audioOn"));
+            }
+
+        }
+
+        @Override
+        public void onMessage(DataChannel.Buffer buffer) {
+            if (buffer.binary) {
+                Log.d(TAG, "Received binary msg over " + TAG + " " + sessionId);
+                return;
+            }
+            ByteBuffer data = buffer.data;
+            final byte[] bytes = new byte[data.capacity()];
+            data.get(bytes);
+            String strData = new String(bytes);
+            Log.d(TAG, "Got msg: " + strData + " over " + TAG + " " + sessionId);
+        }
+    }
 }

app/src/main/res/layout/activity_call.xml

@@ -26,6 +26,15 @@
                 android:fitsSystemWindows="true"
                 tools:context=".activities.CallActivity">
 
+    <GridLayout
+        android:layout_width="match_parent"
+        android:layout_height="wrap_content"
+        android:id="@+id/videos_grid_view"
+        android:columnCount="4"
+        android:rowCount="2">
+
+    </GridLayout>
+
     <org.webrtc.SurfaceViewRenderer
         android:id="@+id/pip_video_view"
         android:layout_width="120dp"
@@ -34,9 +43,4 @@
         android:layout_alignParentEnd="true"
         android:layout_margin="16dp"/>
 
-    <org.webrtc.SurfaceViewRenderer
-        android:id="@+id/fullscreen_video_view"
-        android:layout_width="match_parent"
-        android:layout_height="match_parent"/>
-
 </RelativeLayout>

app/src/main/res/layout/surface_renderer.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+  ~ Nextcloud Talk application
+  ~
+  ~ @author Mario Danic
+  ~ Copyright (C) 2017 Mario Danic <mario@lovelyhq.com>
+  ~
+  ~ This program is free software: you can redistribute it and/or modify
+  ~ it under the terms of the GNU General Public License as published by
+  ~ the Free Software Foundation, either version 3 of the License, or
+  ~ at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program.  If not, see <http://www.gnu.org/licenses/>.
+  -->
+
+<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
+                                android:id="@+id/relative_layout"
+                                android:layout_width="match_parent"
+                                android:layout_height="match_parent"
+                                android:orientation="vertical">
+
+    <org.webrtc.SurfaceViewRenderer
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:id="@+id/surface_view"/>
+
+</RelativeLayout>