CallActivity.java 132 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111
  1. /*
  2. * Nextcloud Talk application
  3. *
  4. * @author Mario Danic
  5. * @author Tim Krüger
  6. * @author Marcel Hibbe
  7. * Copyright (C) 2022 Marcel Hibbe <dev@mhibbe.de>
  8. * Copyright (C) 2022 Tim Krüger <t@timkrueger.me>
  9. * Copyright (C) 2017-2018 Mario Danic <mario@lovelyhq.com>
  10. *
  11. * This program is free software: you can redistribute it and/or modify
  12. * it under the terms of the GNU General Public License as published by
  13. * the Free Software Foundation, either version 3 of the License, or
  14. * at your option) any later version.
  15. *
  16. * This program is distributed in the hope that it will be useful,
  17. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  19. * GNU General Public License for more details.
  20. *
  21. * You should have received a copy of the GNU General Public License
  22. * along with this program. If not, see <http://www.gnu.org/licenses/>.
  23. */
  24. package com.nextcloud.talk.activities;
  25. import android.Manifest;
  26. import android.animation.Animator;
  27. import android.animation.AnimatorListenerAdapter;
  28. import android.annotation.SuppressLint;
  29. import android.app.PendingIntent;
  30. import android.app.RemoteAction;
  31. import android.content.BroadcastReceiver;
  32. import android.content.Context;
  33. import android.content.Intent;
  34. import android.content.IntentFilter;
  35. import android.content.pm.PackageManager;
  36. import android.content.res.Configuration;
  37. import android.graphics.Color;
  38. import android.graphics.drawable.Icon;
  39. import android.media.AudioAttributes;
  40. import android.media.MediaPlayer;
  41. import android.net.Uri;
  42. import android.os.Build;
  43. import android.os.Bundle;
  44. import android.os.Handler;
  45. import android.os.Looper;
  46. import android.text.TextUtils;
  47. import android.util.DisplayMetrics;
  48. import android.util.Log;
  49. import android.view.MotionEvent;
  50. import android.view.View;
  51. import android.view.ViewGroup;
  52. import android.view.ViewTreeObserver;
  53. import android.widget.FrameLayout;
  54. import android.widget.RelativeLayout;
  55. import android.widget.Toast;
  56. import com.bluelinelabs.logansquare.LoganSquare;
  57. import com.google.android.material.dialog.MaterialAlertDialogBuilder;
  58. import com.nextcloud.talk.R;
  59. import com.nextcloud.talk.adapters.ParticipantDisplayItem;
  60. import com.nextcloud.talk.adapters.ParticipantsAdapter;
  61. import com.nextcloud.talk.api.NcApi;
  62. import com.nextcloud.talk.application.NextcloudTalkApplication;
  63. import com.nextcloud.talk.call.CallParticipant;
  64. import com.nextcloud.talk.call.CallParticipantList;
  65. import com.nextcloud.talk.call.CallParticipantModel;
  66. import com.nextcloud.talk.data.user.model.User;
  67. import com.nextcloud.talk.databinding.CallActivityBinding;
  68. import com.nextcloud.talk.events.ConfigurationChangeEvent;
  69. import com.nextcloud.talk.events.NetworkEvent;
  70. import com.nextcloud.talk.events.ProximitySensorEvent;
  71. import com.nextcloud.talk.events.WebSocketCommunicationEvent;
  72. import com.nextcloud.talk.models.ExternalSignalingServer;
  73. import com.nextcloud.talk.models.json.capabilities.CapabilitiesOverall;
  74. import com.nextcloud.talk.models.json.conversations.Conversation;
  75. import com.nextcloud.talk.models.json.conversations.RoomOverall;
  76. import com.nextcloud.talk.models.json.conversations.RoomsOverall;
  77. import com.nextcloud.talk.models.json.generic.GenericOverall;
  78. import com.nextcloud.talk.models.json.participants.Participant;
  79. import com.nextcloud.talk.models.json.signaling.DataChannelMessage;
  80. import com.nextcloud.talk.models.json.signaling.NCMessagePayload;
  81. import com.nextcloud.talk.models.json.signaling.NCSignalingMessage;
  82. import com.nextcloud.talk.models.json.signaling.Signaling;
  83. import com.nextcloud.talk.models.json.signaling.SignalingOverall;
  84. import com.nextcloud.talk.models.json.signaling.settings.IceServer;
  85. import com.nextcloud.talk.models.json.signaling.settings.SignalingSettingsOverall;
  86. import com.nextcloud.talk.raisehand.viewmodel.RaiseHandViewModel;
  87. import com.nextcloud.talk.signaling.SignalingMessageReceiver;
  88. import com.nextcloud.talk.signaling.SignalingMessageSender;
  89. import com.nextcloud.talk.ui.dialog.AudioOutputDialog;
  90. import com.nextcloud.talk.ui.dialog.MoreCallActionsDialog;
  91. import com.nextcloud.talk.users.UserManager;
  92. import com.nextcloud.talk.utils.ApiUtils;
  93. import com.nextcloud.talk.utils.DisplayUtils;
  94. import com.nextcloud.talk.utils.NotificationUtils;
  95. import com.nextcloud.talk.utils.VibrationUtils;
  96. import com.nextcloud.talk.utils.animations.PulseAnimation;
  97. import com.nextcloud.talk.utils.database.user.CapabilitiesUtilNew;
  98. import com.nextcloud.talk.utils.permissions.PlatformPermissionUtil;
  99. import com.nextcloud.talk.utils.power.PowerManagerUtils;
  100. import com.nextcloud.talk.utils.singletons.ApplicationWideCurrentRoomHolder;
  101. import com.nextcloud.talk.viewmodels.CallRecordingViewModel;
  102. import com.nextcloud.talk.webrtc.MagicWebRTCUtils;
  103. import com.nextcloud.talk.webrtc.PeerConnectionWrapper;
  104. import com.nextcloud.talk.webrtc.WebRtcAudioManager;
  105. import com.nextcloud.talk.webrtc.WebSocketConnectionHelper;
  106. import com.nextcloud.talk.webrtc.WebSocketInstance;
  107. import com.wooplr.spotlight.SpotlightView;
  108. import org.apache.commons.lang3.StringEscapeUtils;
  109. import org.greenrobot.eventbus.Subscribe;
  110. import org.greenrobot.eventbus.ThreadMode;
  111. import org.webrtc.AudioSource;
  112. import org.webrtc.AudioTrack;
  113. import org.webrtc.Camera1Enumerator;
  114. import org.webrtc.Camera2Enumerator;
  115. import org.webrtc.CameraEnumerator;
  116. import org.webrtc.CameraVideoCapturer;
  117. import org.webrtc.DefaultVideoDecoderFactory;
  118. import org.webrtc.DefaultVideoEncoderFactory;
  119. import org.webrtc.EglBase;
  120. import org.webrtc.Logging;
  121. import org.webrtc.MediaConstraints;
  122. import org.webrtc.MediaStream;
  123. import org.webrtc.PeerConnection;
  124. import org.webrtc.PeerConnectionFactory;
  125. import org.webrtc.RendererCommon;
  126. import org.webrtc.SurfaceTextureHelper;
  127. import org.webrtc.VideoCapturer;
  128. import org.webrtc.VideoSource;
  129. import org.webrtc.VideoTrack;
  130. import java.io.IOException;
  131. import java.util.ArrayList;
  132. import java.util.Collection;
  133. import java.util.HashMap;
  134. import java.util.List;
  135. import java.util.Map;
  136. import java.util.Objects;
  137. import java.util.Set;
  138. import java.util.concurrent.TimeUnit;
  139. import java.util.concurrent.atomic.AtomicInteger;
  140. import javax.inject.Inject;
  141. import androidx.activity.result.ActivityResultLauncher;
  142. import androidx.activity.result.contract.ActivityResultContracts;
  143. import androidx.annotation.DrawableRes;
  144. import androidx.annotation.NonNull;
  145. import androidx.annotation.Nullable;
  146. import androidx.annotation.RequiresApi;
  147. import androidx.appcompat.app.AlertDialog;
  148. import androidx.appcompat.app.AppCompatActivity;
  149. import androidx.core.content.ContextCompat;
  150. import androidx.core.graphics.drawable.DrawableCompat;
  151. import androidx.lifecycle.ViewModelProvider;
  152. import autodagger.AutoInjector;
  153. import io.reactivex.Observable;
  154. import io.reactivex.Observer;
  155. import io.reactivex.android.schedulers.AndroidSchedulers;
  156. import io.reactivex.disposables.Disposable;
  157. import io.reactivex.schedulers.Schedulers;
  158. import me.zhanghai.android.effortlesspermissions.AfterPermissionDenied;
  159. import me.zhanghai.android.effortlesspermissions.EffortlessPermissions;
  160. import me.zhanghai.android.effortlesspermissions.OpenAppDetailsDialogFragment;
  161. import okhttp3.Cache;
  162. import pub.devrel.easypermissions.AfterPermissionGranted;
  163. import static android.app.PendingIntent.FLAG_IMMUTABLE;
  164. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_CALL_VOICE_ONLY;
  165. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_CALL_WITHOUT_NOTIFICATION;
  166. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_CONVERSATION_NAME;
  167. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_CONVERSATION_PASSWORD;
  168. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_FROM_NOTIFICATION_START_CALL;
  169. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_IS_BREAKOUT_ROOM;
  170. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_IS_MODERATOR;
  171. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_MODIFIED_BASE_URL;
  172. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_PARTICIPANT_PERMISSION_CAN_PUBLISH_AUDIO;
  173. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_PARTICIPANT_PERMISSION_CAN_PUBLISH_VIDEO;
  174. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_RECORDING_STATE;
  175. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_ROOM_ID;
  176. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_ROOM_TOKEN;
  177. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_SWITCH_TO_ROOM_AND_START_CALL;
  178. import static com.nextcloud.talk.utils.bundle.BundleKeys.KEY_USER_ENTITY;
  179. @AutoInjector(NextcloudTalkApplication.class)
  180. public class CallActivity extends CallBaseActivity {
  181. public static boolean active = false;
  182. public static final String VIDEO_STREAM_TYPE_SCREEN = "screen";
  183. public static final String VIDEO_STREAM_TYPE_VIDEO = "video";
  184. @Inject
  185. NcApi ncApi;
  186. @Inject
  187. UserManager userManager;
  188. @Inject
  189. Cache cache;
  190. @Inject
  191. PlatformPermissionUtil permissionUtil;
  192. @Inject
  193. ViewModelProvider.Factory viewModelFactory;
  194. public static final String TAG = "CallActivity";
  195. public WebRtcAudioManager audioManager;
  196. public CallRecordingViewModel callRecordingViewModel;
  197. public RaiseHandViewModel raiseHandViewModel;
  198. private static final String[] PERMISSIONS_CALL = {
  199. Manifest.permission.CAMERA,
  200. Manifest.permission.RECORD_AUDIO
  201. };
  202. private static final String[] PERMISSIONS_CAMERA = {
  203. Manifest.permission.CAMERA
  204. };
  205. private static final String[] PERMISSIONS_MICROPHONE = {
  206. Manifest.permission.RECORD_AUDIO
  207. };
  208. private static final String MICROPHONE_PIP_INTENT_NAME = "microphone_pip_intent";
  209. private static final String MICROPHONE_PIP_INTENT_EXTRA_ACTION = "microphone_pip_action";
  210. private static final int MICROPHONE_PIP_REQUEST_MUTE = 1;
  211. private static final int MICROPHONE_PIP_REQUEST_UNMUTE = 2;
  212. private BroadcastReceiver mReceiver;
  213. private PeerConnectionFactory peerConnectionFactory;
  214. private MediaConstraints audioConstraints;
  215. private MediaConstraints videoConstraints;
  216. private MediaConstraints sdpConstraints;
  217. private MediaConstraints sdpConstraintsForMCU;
  218. private VideoSource videoSource;
  219. private VideoTrack localVideoTrack;
  220. private AudioSource audioSource;
  221. private AudioTrack localAudioTrack;
  222. private VideoCapturer videoCapturer;
  223. private EglBase rootEglBase;
  224. private Disposable signalingDisposable;
  225. private List<PeerConnection.IceServer> iceServers;
  226. private CameraEnumerator cameraEnumerator;
  227. private String roomToken;
  228. private User conversationUser;
  229. private String conversationName;
  230. private String callSession;
  231. private MediaStream localStream;
  232. private String credentials;
  233. private List<PeerConnectionWrapper> peerConnectionWrapperList = new ArrayList<>();
  234. private boolean videoOn = false;
  235. private boolean microphoneOn = false;
  236. private boolean isVoiceOnlyCall;
  237. private boolean isCallWithoutNotification;
  238. private boolean isIncomingCallFromNotification;
  239. private Handler callControlHandler = new Handler();
  240. private Handler callInfosHandler = new Handler();
  241. private Handler cameraSwitchHandler = new Handler();
  242. // push to talk
  243. private boolean isPushToTalkActive = false;
  244. private PulseAnimation pulseAnimation;
  245. private String baseUrl;
  246. private String roomId;
  247. private SpotlightView spotlightView;
  248. private InternalSignalingMessageReceiver internalSignalingMessageReceiver = new InternalSignalingMessageReceiver();
  249. private SignalingMessageReceiver signalingMessageReceiver;
  250. private InternalSignalingMessageSender internalSignalingMessageSender = new InternalSignalingMessageSender();
  251. private SignalingMessageSender signalingMessageSender;
  252. private Map<String, OfferAnswerNickProvider> offerAnswerNickProviders = new HashMap<>();
  253. private Map<String, SignalingMessageReceiver.CallParticipantMessageListener> callParticipantMessageListeners =
  254. new HashMap<>();
  255. private PeerConnectionWrapper.PeerConnectionObserver selfPeerConnectionObserver = new CallActivitySelfPeerConnectionObserver();
  256. private Map<String, CallParticipant> callParticipants = new HashMap<>();
  257. private Map<String, ScreenParticipantDisplayItemManager> screenParticipantDisplayItemManagers = new HashMap<>();
  258. private Handler screenParticipantDisplayItemManagersHandler = new Handler(Looper.getMainLooper());
  259. private CallParticipantList.Observer callParticipantListObserver = new CallParticipantList.Observer() {
  260. @Override
  261. public void onCallParticipantsChanged(Collection<Participant> joined, Collection<Participant> updated,
  262. Collection<Participant> left, Collection<Participant> unchanged) {
  263. handleCallParticipantsChanged(joined, updated, left, unchanged);
  264. }
  265. @Override
  266. public void onCallEndedForAll() {
  267. Log.d(TAG, "A moderator ended the call for all.");
  268. hangup(true);
  269. }
  270. };
  271. private CallParticipantList callParticipantList;
  272. private String switchToRoomToken = "";
  273. private boolean isBreakoutRoom = false;
  274. private SignalingMessageReceiver.LocalParticipantMessageListener localParticipantMessageListener =
  275. new SignalingMessageReceiver.LocalParticipantMessageListener() {
  276. @Override
  277. public void onSwitchTo(String token) {
  278. switchToRoomToken = token;
  279. hangup(true);
  280. }
  281. };
  282. private SignalingMessageReceiver.OfferMessageListener offerMessageListener = new SignalingMessageReceiver.OfferMessageListener() {
  283. @Override
  284. public void onOffer(String sessionId, String roomType, String sdp, String nick) {
  285. getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, roomType, false);
  286. }
  287. };
  288. private ExternalSignalingServer externalSignalingServer;
  289. private WebSocketInstance webSocketClient;
  290. private WebSocketConnectionHelper webSocketConnectionHelper;
  291. private boolean hasMCU;
  292. private boolean hasExternalSignalingServer;
  293. private String conversationPassword;
  294. private PowerManagerUtils powerManagerUtils;
  295. private Handler handler;
  296. private CallStatus currentCallStatus;
  297. private MediaPlayer mediaPlayer;
  298. private Map<String, ParticipantDisplayItem> participantDisplayItems;
  299. private ParticipantsAdapter participantsAdapter;
  300. private CallActivityBinding binding;
  301. private AudioOutputDialog audioOutputDialog;
  302. private MoreCallActionsDialog moreCallActionsDialog;
  303. private final ActivityResultLauncher<String> requestBluetoothPermissionLauncher =
  304. registerForActivityResult(new ActivityResultContracts.RequestPermission(), isGranted -> {
  305. if (isGranted) {
  306. enableBluetoothManager();
  307. }
  308. });
  309. private boolean canPublishAudioStream;
  310. private boolean canPublishVideoStream;
  311. private boolean isModerator;
  312. @SuppressLint("ClickableViewAccessibility")
  313. @Override
  314. public void onCreate(Bundle savedInstanceState) {
  315. Log.d(TAG, "onCreate");
  316. super.onCreate(savedInstanceState);
  317. NextcloudTalkApplication.Companion.getSharedApplication().getComponentApplication().inject(this);
  318. binding = CallActivityBinding.inflate(getLayoutInflater());
  319. setContentView(binding.getRoot());
  320. hideNavigationIfNoPipAvailable();
  321. Bundle extras = getIntent().getExtras();
  322. roomId = extras.getString(KEY_ROOM_ID, "");
  323. roomToken = extras.getString(KEY_ROOM_TOKEN, "");
  324. conversationUser = extras.getParcelable(KEY_USER_ENTITY);
  325. conversationPassword = extras.getString(KEY_CONVERSATION_PASSWORD, "");
  326. conversationName = extras.getString(KEY_CONVERSATION_NAME, "");
  327. isVoiceOnlyCall = extras.getBoolean(KEY_CALL_VOICE_ONLY, false);
  328. isCallWithoutNotification = extras.getBoolean(KEY_CALL_WITHOUT_NOTIFICATION, false);
  329. canPublishAudioStream = extras.getBoolean(KEY_PARTICIPANT_PERMISSION_CAN_PUBLISH_AUDIO);
  330. canPublishVideoStream = extras.getBoolean(KEY_PARTICIPANT_PERMISSION_CAN_PUBLISH_VIDEO);
  331. isModerator = extras.getBoolean(KEY_IS_MODERATOR, false);
  332. if (extras.containsKey(KEY_FROM_NOTIFICATION_START_CALL)) {
  333. isIncomingCallFromNotification = extras.getBoolean(KEY_FROM_NOTIFICATION_START_CALL);
  334. }
  335. if (extras.containsKey(KEY_IS_BREAKOUT_ROOM)) {
  336. isBreakoutRoom = extras.getBoolean(KEY_IS_BREAKOUT_ROOM);
  337. }
  338. credentials = ApiUtils.getCredentials(conversationUser.getUsername(), conversationUser.getToken());
  339. baseUrl = extras.getString(KEY_MODIFIED_BASE_URL, "");
  340. if (TextUtils.isEmpty(baseUrl)) {
  341. baseUrl = conversationUser.getBaseUrl();
  342. }
  343. powerManagerUtils = new PowerManagerUtils();
  344. if ("resume".equalsIgnoreCase(extras.getString("state", ""))) {
  345. setCallState(CallStatus.IN_CONVERSATION);
  346. } else {
  347. setCallState(CallStatus.CONNECTING);
  348. }
  349. raiseHandViewModel = new ViewModelProvider(this, viewModelFactory).get((RaiseHandViewModel.class));
  350. raiseHandViewModel.setData(roomToken, isBreakoutRoom);
  351. raiseHandViewModel.getViewState().observe(this, viewState -> {
  352. if (viewState instanceof RaiseHandViewModel.RaisedHandState) {
  353. binding.lowerHandButton.setVisibility(View.VISIBLE);
  354. } else if (viewState instanceof RaiseHandViewModel.LoweredHandState) {
  355. binding.lowerHandButton.setVisibility(View.GONE);
  356. }
  357. // TODO: build&send raiseHand message (if not possible in RaiseHandViewModel, just do it here..)
  358. // if (isConnectionEstablished() && peerConnectionWrapperList != null) {
  359. // if (!hasMCU) {
  360. // for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
  361. // peerConnectionWrapper.raiseHand(...);
  362. // }
  363. // } else {
  364. // for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
  365. // if (peerConnectionWrapper.getSessionId().equals(webSocketClient.getSessionId())) {
  366. // peerConnectionWrapper.raiseHand(...);
  367. // break;
  368. // }
  369. // }
  370. // }
  371. // }
  372. });
  373. callRecordingViewModel = new ViewModelProvider(this, viewModelFactory).get((CallRecordingViewModel.class));
  374. callRecordingViewModel.setData(roomToken);
  375. callRecordingViewModel.setRecordingState(extras.getInt(KEY_RECORDING_STATE));
  376. callRecordingViewModel.getViewState().observe(this, viewState -> {
  377. if (viewState instanceof CallRecordingViewModel.RecordingStartedState) {
  378. binding.callRecordingIndicator.setVisibility(View.VISIBLE);
  379. if (((CallRecordingViewModel.RecordingStartedState) viewState).getShowStartedInfo()) {
  380. VibrationUtils.INSTANCE.vibrateShort(context);
  381. Toast.makeText(context, context.getResources().getString(R.string.record_active_info), Toast.LENGTH_LONG).show();
  382. }
  383. } else if (viewState instanceof CallRecordingViewModel.RecordingConfirmStopState) {
  384. MaterialAlertDialogBuilder dialogBuilder = new MaterialAlertDialogBuilder(this)
  385. .setTitle(R.string.record_stop_confirm_title)
  386. .setMessage(R.string.record_stop_confirm_message)
  387. .setPositiveButton(R.string.record_stop_description,
  388. (dialog, which) -> callRecordingViewModel.stopRecording())
  389. .setNegativeButton(R.string.nc_common_dismiss,
  390. (dialog, which) -> callRecordingViewModel.dismissStopRecording());
  391. viewThemeUtils.dialog.colorMaterialAlertDialogBackground(this, dialogBuilder);
  392. AlertDialog dialog = dialogBuilder.show();
  393. viewThemeUtils.platform.colorTextButtons(
  394. dialog.getButton(AlertDialog.BUTTON_POSITIVE),
  395. dialog.getButton(AlertDialog.BUTTON_NEGATIVE)
  396. );
  397. } else if (viewState instanceof CallRecordingViewModel.RecordingErrorState) {
  398. Toast.makeText(context, context.getResources().getString(R.string.nc_common_error_sorry),
  399. Toast.LENGTH_LONG).show();
  400. } else {
  401. binding.callRecordingIndicator.setVisibility(View.GONE);
  402. }
  403. });
  404. initClickListeners();
  405. binding.microphoneButton.setOnTouchListener(new MicrophoneButtonTouchListener());
  406. pulseAnimation = PulseAnimation.create().with(binding.microphoneButton)
  407. .setDuration(310)
  408. .setRepeatCount(PulseAnimation.INFINITE)
  409. .setRepeatMode(PulseAnimation.REVERSE);
  410. if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
  411. requestBluetoothPermission();
  412. }
  413. basicInitialization();
  414. callParticipants = new HashMap<>();
  415. participantDisplayItems = new HashMap<>();
  416. initViews();
  417. if (!isConnectionEstablished()) {
  418. initiateCall();
  419. }
  420. updateSelfVideoViewPosition();
  421. }
  422. @Override
  423. public void onStart() {
  424. super.onStart();
  425. active = true;
  426. initFeaturesVisibility();
  427. try {
  428. cache.evictAll();
  429. } catch (IOException e) {
  430. Log.e(TAG, "Failed to evict cache");
  431. }
  432. }
  433. @Override
  434. public void onStop() {
  435. super.onStop();
  436. active = false;
  437. }
  438. @RequiresApi(api = Build.VERSION_CODES.S)
  439. private void requestBluetoothPermission() {
  440. if (ContextCompat.checkSelfPermission(
  441. getContext(), Manifest.permission.BLUETOOTH_CONNECT) ==
  442. PackageManager.PERMISSION_DENIED) {
  443. requestBluetoothPermissionLauncher.launch(Manifest.permission.BLUETOOTH_CONNECT);
  444. }
  445. }
  446. private void enableBluetoothManager() {
  447. if (audioManager != null) {
  448. audioManager.startBluetoothManager();
  449. }
  450. }
  451. private void initFeaturesVisibility() {
  452. if (isAllowedToStartOrStopRecording() || isAllowedToRaiseHand()) {
  453. binding.moreCallActions.setVisibility(View.VISIBLE);
  454. } else {
  455. binding.moreCallActions.setVisibility(View.GONE);
  456. }
  457. }
  458. private void initClickListeners() {
  459. binding.pictureInPictureButton.setOnClickListener(l -> enterPipMode());
  460. binding.audioOutputButton.setOnClickListener(v -> {
  461. audioOutputDialog = new AudioOutputDialog(this);
  462. audioOutputDialog.show();
  463. });
  464. binding.moreCallActions.setOnClickListener(v -> {
  465. moreCallActionsDialog = new MoreCallActionsDialog(this);
  466. moreCallActionsDialog.show();
  467. });
  468. if (canPublishAudioStream) {
  469. binding.microphoneButton.setOnClickListener(l -> onMicrophoneClick());
  470. binding.microphoneButton.setOnLongClickListener(l -> {
  471. if (!microphoneOn) {
  472. callControlHandler.removeCallbacksAndMessages(null);
  473. callInfosHandler.removeCallbacksAndMessages(null);
  474. cameraSwitchHandler.removeCallbacksAndMessages(null);
  475. isPushToTalkActive = true;
  476. binding.callControls.setVisibility(View.VISIBLE);
  477. if (!isVoiceOnlyCall) {
  478. binding.switchSelfVideoButton.setVisibility(View.VISIBLE);
  479. }
  480. }
  481. onMicrophoneClick();
  482. return true;
  483. });
  484. } else {
  485. binding.microphoneButton.setOnClickListener(
  486. l -> Toast.makeText(context,
  487. R.string.nc_not_allowed_to_activate_audio,
  488. Toast.LENGTH_SHORT
  489. ).show()
  490. );
  491. }
  492. if (canPublishVideoStream) {
  493. binding.cameraButton.setOnClickListener(l -> onCameraClick());
  494. } else {
  495. binding.cameraButton.setOnClickListener(
  496. l -> Toast.makeText(context,
  497. R.string.nc_not_allowed_to_activate_video,
  498. Toast.LENGTH_SHORT
  499. ).show()
  500. );
  501. }
  502. binding.hangupButton.setOnClickListener(l -> {
  503. hangup(true);
  504. });
  505. binding.switchSelfVideoButton.setOnClickListener(l -> switchCamera());
  506. binding.gridview.setOnItemClickListener((parent, view, position, id) -> animateCallControls(true, 0));
  507. binding.callStates.callStateRelativeLayout.setOnClickListener(l -> {
  508. if (currentCallStatus == CallStatus.CALLING_TIMEOUT) {
  509. setCallState(CallStatus.RECONNECTING);
  510. hangupNetworkCalls(false);
  511. }
  512. });
  513. binding.callRecordingIndicator.setOnClickListener(l -> {
  514. if (isAllowedToStartOrStopRecording()) {
  515. callRecordingViewModel.clickRecordButton();
  516. } else {
  517. Toast.makeText(context, context.getResources().getString(R.string.record_active_info), Toast.LENGTH_LONG).show();
  518. }
  519. });
  520. binding.lowerHandButton.setOnClickListener(l -> {
  521. raiseHandViewModel.lowerHand();
  522. });
  523. }
  524. private void createCameraEnumerator() {
  525. boolean camera2EnumeratorIsSupported = false;
  526. try {
  527. camera2EnumeratorIsSupported = Camera2Enumerator.isSupported(this);
  528. } catch (final Throwable t) {
  529. Log.w(TAG, "Camera2Enumerator threw an error", t);
  530. }
  531. if (camera2EnumeratorIsSupported) {
  532. cameraEnumerator = new Camera2Enumerator(this);
  533. } else {
  534. cameraEnumerator = new Camera1Enumerator(MagicWebRTCUtils.shouldEnableVideoHardwareAcceleration());
  535. }
  536. }
  537. private void basicInitialization() {
  538. rootEglBase = EglBase.create();
  539. createCameraEnumerator();
  540. //Create a new PeerConnectionFactory instance.
  541. PeerConnectionFactory.Options options = new PeerConnectionFactory.Options();
  542. DefaultVideoEncoderFactory defaultVideoEncoderFactory = new DefaultVideoEncoderFactory(
  543. rootEglBase.getEglBaseContext(), true, true);
  544. DefaultVideoDecoderFactory defaultVideoDecoderFactory = new DefaultVideoDecoderFactory(
  545. rootEglBase.getEglBaseContext());
  546. peerConnectionFactory = PeerConnectionFactory.builder()
  547. .setOptions(options)
  548. .setVideoEncoderFactory(defaultVideoEncoderFactory)
  549. .setVideoDecoderFactory(defaultVideoDecoderFactory)
  550. .createPeerConnectionFactory();
  551. //Create MediaConstraints - Will be useful for specifying video and audio constraints.
  552. audioConstraints = new MediaConstraints();
  553. videoConstraints = new MediaConstraints();
  554. localStream = peerConnectionFactory.createLocalMediaStream("NCMS");
  555. // Create and audio manager that will take care of audio routing,
  556. // audio modes, audio device enumeration etc.
  557. audioManager = WebRtcAudioManager.create(getApplicationContext(), isVoiceOnlyCall);
  558. // Store existing audio settings and change audio mode to
  559. // MODE_IN_COMMUNICATION for best possible VoIP performance.
  560. Log.d(TAG, "Starting the audio manager...");
  561. audioManager.start(this::onAudioManagerDevicesChanged);
  562. if (isVoiceOnlyCall) {
  563. setAudioOutputChannel(WebRtcAudioManager.AudioDevice.EARPIECE);
  564. } else {
  565. setAudioOutputChannel(WebRtcAudioManager.AudioDevice.SPEAKER_PHONE);
  566. }
  567. iceServers = new ArrayList<>();
  568. //create sdpConstraints
  569. sdpConstraints = new MediaConstraints();
  570. sdpConstraintsForMCU = new MediaConstraints();
  571. sdpConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true"));
  572. String offerToReceiveVideoString = "true";
  573. if (isVoiceOnlyCall) {
  574. offerToReceiveVideoString = "false";
  575. }
  576. sdpConstraints.mandatory.add(
  577. new MediaConstraints.KeyValuePair("OfferToReceiveVideo", offerToReceiveVideoString));
  578. sdpConstraintsForMCU.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "false"));
  579. sdpConstraintsForMCU.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "false"));
  580. sdpConstraintsForMCU.optional.add(new MediaConstraints.KeyValuePair("internalSctpDataChannels", "true"));
  581. sdpConstraintsForMCU.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true"));
  582. sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("internalSctpDataChannels", "true"));
  583. sdpConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true"));
  584. if (!isVoiceOnlyCall) {
  585. cameraInitialization();
  586. }
  587. microphoneInitialization();
  588. }
  589. public void setAudioOutputChannel(WebRtcAudioManager.AudioDevice selectedAudioDevice) {
  590. if (audioManager != null) {
  591. audioManager.selectAudioDevice(selectedAudioDevice);
  592. updateAudioOutputButton(audioManager.getCurrentAudioDevice());
  593. }
  594. }
  595. private void updateAudioOutputButton(WebRtcAudioManager.AudioDevice activeAudioDevice) {
  596. switch (activeAudioDevice) {
  597. case BLUETOOTH:
  598. binding.audioOutputButton.setImageResource(R.drawable.ic_baseline_bluetooth_audio_24);
  599. break;
  600. case SPEAKER_PHONE:
  601. binding.audioOutputButton.setImageResource(R.drawable.ic_volume_up_white_24dp);
  602. break;
  603. case EARPIECE:
  604. binding.audioOutputButton.setImageResource(R.drawable.ic_baseline_phone_in_talk_24);
  605. break;
  606. case WIRED_HEADSET:
  607. binding.audioOutputButton.setImageResource(R.drawable.ic_baseline_headset_mic_24);
  608. break;
  609. default:
  610. Log.e(TAG, "Icon for audio output not available");
  611. break;
  612. }
  613. DrawableCompat.setTint(binding.audioOutputButton.getDrawable(), Color.WHITE);
  614. }
  615. private void handleFromNotification() {
  616. int apiVersion = ApiUtils.getConversationApiVersion(conversationUser, new int[]{ApiUtils.APIv4, 1});
  617. ncApi.getRooms(credentials, ApiUtils.getUrlForRooms(apiVersion, baseUrl), Boolean.FALSE)
  618. .retry(3)
  619. .subscribeOn(Schedulers.io())
  620. .observeOn(AndroidSchedulers.mainThread())
  621. .subscribe(new Observer<RoomsOverall>() {
  622. @Override
  623. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  624. // unused atm
  625. }
  626. @Override
  627. public void onNext(@io.reactivex.annotations.NonNull RoomsOverall roomsOverall) {
  628. for (Conversation conversation : roomsOverall.getOcs().getData()) {
  629. if (roomId.equals(conversation.getRoomId())) {
  630. roomToken = conversation.getToken();
  631. break;
  632. }
  633. }
  634. checkDevicePermissions();
  635. }
  636. @Override
  637. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  638. // unused atm
  639. }
  640. @Override
  641. public void onComplete() {
  642. // unused atm
  643. }
  644. });
  645. }
  646. @SuppressLint("ClickableViewAccessibility")
  647. private void initViews() {
  648. Log.d(TAG, "initViews");
  649. binding.callInfosLinearLayout.setVisibility(View.VISIBLE);
  650. binding.selfVideoViewWrapper.setVisibility(View.VISIBLE);
  651. if (!isPipModePossible()) {
  652. binding.pictureInPictureButton.setVisibility(View.GONE);
  653. }
  654. if (isVoiceOnlyCall) {
  655. binding.switchSelfVideoButton.setVisibility(View.GONE);
  656. binding.cameraButton.setVisibility(View.GONE);
  657. binding.selfVideoRenderer.setVisibility(View.GONE);
  658. RelativeLayout.LayoutParams params = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT,
  659. ViewGroup.LayoutParams.WRAP_CONTENT);
  660. params.addRule(RelativeLayout.BELOW, R.id.callInfosLinearLayout);
  661. int callControlsHeight = Math.round(getApplicationContext().getResources().getDimension(R.dimen.call_controls_height));
  662. params.setMargins(0, 0, 0, callControlsHeight);
  663. binding.gridview.setLayoutParams(params);
  664. } else {
  665. RelativeLayout.LayoutParams params = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT,
  666. ViewGroup.LayoutParams.WRAP_CONTENT);
  667. params.setMargins(0, 0, 0, 0);
  668. binding.gridview.setLayoutParams(params);
  669. if (cameraEnumerator.getDeviceNames().length < 2) {
  670. binding.switchSelfVideoButton.setVisibility(View.GONE);
  671. }
  672. initSelfVideoView();
  673. }
  674. binding.gridview.setOnTouchListener(new View.OnTouchListener() {
  675. public boolean onTouch(View v, MotionEvent me) {
  676. int action = me.getActionMasked();
  677. if (action == MotionEvent.ACTION_DOWN) {
  678. animateCallControls(true, 0);
  679. }
  680. return false;
  681. }
  682. });
  683. binding.conversationRelativeLayout.setOnTouchListener(new View.OnTouchListener() {
  684. public boolean onTouch(View v, MotionEvent me) {
  685. int action = me.getActionMasked();
  686. if (action == MotionEvent.ACTION_DOWN) {
  687. animateCallControls(true, 0);
  688. }
  689. return false;
  690. }
  691. });
  692. animateCallControls(true, 0);
  693. initGridAdapter();
  694. }
  695. @SuppressLint("ClickableViewAccessibility")
  696. private void initSelfVideoView() {
  697. try {
  698. binding.selfVideoRenderer.init(rootEglBase.getEglBaseContext(), null);
  699. } catch (IllegalStateException e) {
  700. Log.d(TAG, "selfVideoRenderer already initialized", e);
  701. }
  702. binding.selfVideoRenderer.setZOrderMediaOverlay(true);
  703. // disabled because it causes some devices to crash
  704. binding.selfVideoRenderer.setEnableHardwareScaler(false);
  705. binding.selfVideoRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FIT);
  706. binding.selfVideoRenderer.setOnTouchListener(new SelfVideoTouchListener());
  707. }
  708. private void initGridAdapter() {
  709. Log.d(TAG, "initGridAdapter");
  710. int columns;
  711. int participantsInGrid = participantDisplayItems.size();
  712. if (getResources() != null
  713. && getResources().getConfiguration().orientation == Configuration.ORIENTATION_PORTRAIT) {
  714. if (participantsInGrid > 2) {
  715. columns = 2;
  716. } else {
  717. columns = 1;
  718. }
  719. } else {
  720. if (participantsInGrid > 2) {
  721. columns = 3;
  722. } else if (participantsInGrid > 1) {
  723. columns = 2;
  724. } else {
  725. columns = 1;
  726. }
  727. }
  728. binding.gridview.setNumColumns(columns);
  729. binding.conversationRelativeLayout
  730. .getViewTreeObserver()
  731. .addOnGlobalLayoutListener(new ViewTreeObserver.OnGlobalLayoutListener() {
  732. @Override
  733. public void onGlobalLayout() {
  734. binding.conversationRelativeLayout.getViewTreeObserver().removeOnGlobalLayoutListener(this);
  735. int height = binding.conversationRelativeLayout.getMeasuredHeight();
  736. binding.gridview.setMinimumHeight(height);
  737. }
  738. });
  739. binding
  740. .callInfosLinearLayout
  741. .getViewTreeObserver()
  742. .addOnGlobalLayoutListener(new ViewTreeObserver.OnGlobalLayoutListener() {
  743. @Override
  744. public void onGlobalLayout() {
  745. binding.callInfosLinearLayout.getViewTreeObserver().removeOnGlobalLayoutListener(this);
  746. }
  747. });
  748. if (participantsAdapter != null) {
  749. participantsAdapter.destroy();
  750. }
  751. participantsAdapter = new ParticipantsAdapter(
  752. this,
  753. participantDisplayItems,
  754. binding.conversationRelativeLayout,
  755. binding.callInfosLinearLayout,
  756. columns,
  757. isVoiceOnlyCall);
  758. binding.gridview.setAdapter(participantsAdapter);
  759. if (isInPipMode) {
  760. updateUiForPipMode();
  761. }
  762. }
  763. private void checkDevicePermissions() {
  764. if (isVoiceOnlyCall) {
  765. onMicrophoneClick();
  766. } else {
  767. if (EffortlessPermissions.hasPermissions(this, PERMISSIONS_CALL)) {
  768. onPermissionsGranted();
  769. } else {
  770. requestPermissions(PERMISSIONS_CALL, 100);
  771. }
  772. }
  773. }
  774. private boolean isConnectionEstablished() {
  775. return (currentCallStatus == CallStatus.JOINED || currentCallStatus == CallStatus.IN_CONVERSATION);
  776. }
  777. @AfterPermissionGranted(100)
  778. private void onPermissionsGranted() {
  779. if (EffortlessPermissions.hasPermissions(this, PERMISSIONS_CALL)) {
  780. if (!videoOn && !isVoiceOnlyCall) {
  781. onCameraClick();
  782. }
  783. if (!microphoneOn) {
  784. onMicrophoneClick();
  785. }
  786. if (!isVoiceOnlyCall) {
  787. if (cameraEnumerator.getDeviceNames().length == 0) {
  788. binding.cameraButton.setVisibility(View.GONE);
  789. }
  790. if (cameraEnumerator.getDeviceNames().length > 1) {
  791. binding.switchSelfVideoButton.setVisibility(View.VISIBLE);
  792. }
  793. }
  794. if (!isConnectionEstablished()) {
  795. fetchSignalingSettings();
  796. }
  797. } else if (EffortlessPermissions.somePermissionPermanentlyDenied(this, PERMISSIONS_CALL)) {
  798. checkIfSomeAreApproved();
  799. }
  800. }
  801. private void checkIfSomeAreApproved() {
  802. if (!isVoiceOnlyCall) {
  803. if (cameraEnumerator.getDeviceNames().length == 0) {
  804. binding.cameraButton.setVisibility(View.GONE);
  805. }
  806. if (cameraEnumerator.getDeviceNames().length > 1) {
  807. binding.switchSelfVideoButton.setVisibility(View.VISIBLE);
  808. }
  809. if (EffortlessPermissions.hasPermissions(this, PERMISSIONS_CAMERA) && canPublishVideoStream) {
  810. if (!videoOn) {
  811. onCameraClick();
  812. }
  813. } else {
  814. binding.cameraButton.setImageResource(R.drawable.ic_videocam_off_white_24px);
  815. binding.cameraButton.setAlpha(0.7f);
  816. binding.switchSelfVideoButton.setVisibility(View.GONE);
  817. }
  818. }
  819. if (EffortlessPermissions.hasPermissions(this, PERMISSIONS_MICROPHONE) && canPublishAudioStream) {
  820. if (!microphoneOn) {
  821. onMicrophoneClick();
  822. }
  823. } else {
  824. binding.microphoneButton.setImageResource(R.drawable.ic_mic_off_white_24px);
  825. }
  826. if (!isConnectionEstablished()) {
  827. fetchSignalingSettings();
  828. }
  829. }
  830. @AfterPermissionDenied(100)
  831. private void onPermissionsDenied() {
  832. if (!isVoiceOnlyCall) {
  833. if (cameraEnumerator.getDeviceNames().length == 0) {
  834. binding.cameraButton.setVisibility(View.GONE);
  835. } else if (cameraEnumerator.getDeviceNames().length == 1) {
  836. binding.switchSelfVideoButton.setVisibility(View.GONE);
  837. }
  838. }
  839. if ((EffortlessPermissions.hasPermissions(this, PERMISSIONS_CAMERA) ||
  840. EffortlessPermissions.hasPermissions(this, PERMISSIONS_MICROPHONE))) {
  841. checkIfSomeAreApproved();
  842. } else if (!isConnectionEstablished()) {
  843. fetchSignalingSettings();
  844. }
  845. }
  846. private void onAudioManagerDevicesChanged(
  847. final WebRtcAudioManager.AudioDevice currentDevice,
  848. final Set<WebRtcAudioManager.AudioDevice> availableDevices) {
  849. Log.d(TAG, "onAudioManagerDevicesChanged: " + availableDevices + ", "
  850. + "currentDevice: " + currentDevice);
  851. final boolean shouldDisableProximityLock = (currentDevice == WebRtcAudioManager.AudioDevice.WIRED_HEADSET
  852. || currentDevice == WebRtcAudioManager.AudioDevice.SPEAKER_PHONE
  853. || currentDevice == WebRtcAudioManager.AudioDevice.BLUETOOTH);
  854. if (shouldDisableProximityLock) {
  855. powerManagerUtils.updatePhoneState(PowerManagerUtils.PhoneState.WITHOUT_PROXIMITY_SENSOR_LOCK);
  856. } else {
  857. powerManagerUtils.updatePhoneState(PowerManagerUtils.PhoneState.WITH_PROXIMITY_SENSOR_LOCK);
  858. }
  859. if (audioOutputDialog != null) {
  860. audioOutputDialog.updateOutputDeviceList();
  861. }
  862. updateAudioOutputButton(currentDevice);
  863. }
  864. private void cameraInitialization() {
  865. videoCapturer = createCameraCapturer(cameraEnumerator);
  866. //Create a VideoSource instance
  867. if (videoCapturer != null) {
  868. SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("CaptureThread",
  869. rootEglBase.getEglBaseContext());
  870. videoSource = peerConnectionFactory.createVideoSource(false);
  871. videoCapturer.initialize(surfaceTextureHelper, getApplicationContext(), videoSource.getCapturerObserver());
  872. }
  873. localVideoTrack = peerConnectionFactory.createVideoTrack("NCv0", videoSource);
  874. localStream.addTrack(localVideoTrack);
  875. localVideoTrack.setEnabled(false);
  876. localVideoTrack.addSink(binding.selfVideoRenderer);
  877. }
  878. private void microphoneInitialization() {
  879. //create an AudioSource instance
  880. audioSource = peerConnectionFactory.createAudioSource(audioConstraints);
  881. localAudioTrack = peerConnectionFactory.createAudioTrack("NCa0", audioSource);
  882. localAudioTrack.setEnabled(false);
  883. localStream.addTrack(localAudioTrack);
  884. }
  885. private VideoCapturer createCameraCapturer(CameraEnumerator enumerator) {
  886. final String[] deviceNames = enumerator.getDeviceNames();
  887. // First, try to find front facing camera
  888. Logging.d(TAG, "Looking for front facing cameras.");
  889. for (String deviceName : deviceNames) {
  890. if (enumerator.isFrontFacing(deviceName)) {
  891. Logging.d(TAG, "Creating front facing camera capturer.");
  892. VideoCapturer videoCapturer = enumerator.createCapturer(deviceName, null);
  893. if (videoCapturer != null) {
  894. binding.selfVideoRenderer.setMirror(true);
  895. return videoCapturer;
  896. }
  897. }
  898. }
  899. // Front facing camera not found, try something else
  900. Logging.d(TAG, "Looking for other cameras.");
  901. for (String deviceName : deviceNames) {
  902. if (!enumerator.isFrontFacing(deviceName)) {
  903. Logging.d(TAG, "Creating other camera capturer.");
  904. VideoCapturer videoCapturer = enumerator.createCapturer(deviceName, null);
  905. if (videoCapturer != null) {
  906. binding.selfVideoRenderer.setMirror(false);
  907. return videoCapturer;
  908. }
  909. }
  910. }
  911. return null;
  912. }
  913. public void onMicrophoneClick() {
  914. if (!canPublishAudioStream) {
  915. microphoneOn = false;
  916. binding.microphoneButton.setImageResource(R.drawable.ic_mic_off_white_24px);
  917. toggleMedia(false, false);
  918. }
  919. if (isVoiceOnlyCall && !isConnectionEstablished()) {
  920. fetchSignalingSettings();
  921. }
  922. if (!canPublishAudioStream) {
  923. // In the case no audio stream will be published it's not needed to check microphone permissions
  924. return;
  925. }
  926. if (EffortlessPermissions.hasPermissions(this, PERMISSIONS_MICROPHONE)) {
  927. if (!appPreferences.getPushToTalkIntroShown()) {
  928. int primary = viewThemeUtils.getScheme(binding.audioOutputButton.getContext()).getPrimary();
  929. spotlightView = new SpotlightView.Builder(this)
  930. .introAnimationDuration(300)
  931. .enableRevealAnimation(true)
  932. .performClick(false)
  933. .fadeinTextDuration(400)
  934. .headingTvColor(primary)
  935. .headingTvSize(20)
  936. .headingTvText(getResources().getString(R.string.nc_push_to_talk))
  937. .subHeadingTvColor(getResources().getColor(R.color.bg_default))
  938. .subHeadingTvSize(16)
  939. .subHeadingTvText(getResources().getString(R.string.nc_push_to_talk_desc))
  940. .maskColor(Color.parseColor("#dc000000"))
  941. .target(binding.microphoneButton)
  942. .lineAnimDuration(400)
  943. .lineAndArcColor(primary)
  944. .enableDismissAfterShown(true)
  945. .dismissOnBackPress(true)
  946. .usageId("pushToTalk")
  947. .show();
  948. appPreferences.setPushToTalkIntroShown(true);
  949. }
  950. if (!isPushToTalkActive) {
  951. microphoneOn = !microphoneOn;
  952. if (microphoneOn) {
  953. binding.microphoneButton.setImageResource(R.drawable.ic_mic_white_24px);
  954. updatePictureInPictureActions(R.drawable.ic_mic_white_24px,
  955. getResources().getString(R.string.nc_pip_microphone_mute),
  956. MICROPHONE_PIP_REQUEST_MUTE);
  957. } else {
  958. binding.microphoneButton.setImageResource(R.drawable.ic_mic_off_white_24px);
  959. updatePictureInPictureActions(R.drawable.ic_mic_off_white_24px,
  960. getResources().getString(R.string.nc_pip_microphone_unmute),
  961. MICROPHONE_PIP_REQUEST_UNMUTE);
  962. }
  963. toggleMedia(microphoneOn, false);
  964. } else {
  965. binding.microphoneButton.setImageResource(R.drawable.ic_mic_white_24px);
  966. pulseAnimation.start();
  967. toggleMedia(true, false);
  968. }
  969. } else if (EffortlessPermissions.somePermissionPermanentlyDenied(this, PERMISSIONS_MICROPHONE)) {
  970. // Microphone permission is permanently denied so we cannot request it normally.
  971. OpenAppDetailsDialogFragment.show(
  972. R.string.nc_microphone_permission_permanently_denied,
  973. R.string.nc_permissions_settings, (AppCompatActivity) this);
  974. } else {
  975. requestPermissions(PERMISSIONS_MICROPHONE, 100);
  976. }
  977. }
  978. public void onCameraClick() {
  979. if (!canPublishVideoStream) {
  980. videoOn = false;
  981. binding.cameraButton.setImageResource(R.drawable.ic_videocam_off_white_24px);
  982. binding.switchSelfVideoButton.setVisibility(View.GONE);
  983. return;
  984. }
  985. if (EffortlessPermissions.hasPermissions(this, PERMISSIONS_CAMERA)) {
  986. videoOn = !videoOn;
  987. if (videoOn) {
  988. binding.cameraButton.setImageResource(R.drawable.ic_videocam_white_24px);
  989. if (cameraEnumerator.getDeviceNames().length > 1) {
  990. binding.switchSelfVideoButton.setVisibility(View.VISIBLE);
  991. }
  992. } else {
  993. binding.cameraButton.setImageResource(R.drawable.ic_videocam_off_white_24px);
  994. binding.switchSelfVideoButton.setVisibility(View.GONE);
  995. }
  996. toggleMedia(videoOn, true);
  997. } else if (EffortlessPermissions.somePermissionPermanentlyDenied(this, PERMISSIONS_CAMERA)) {
  998. // Camera permission is permanently denied so we cannot request it normally.
  999. OpenAppDetailsDialogFragment.show(
  1000. R.string.nc_camera_permission_permanently_denied,
  1001. R.string.nc_permissions_settings, (AppCompatActivity) this);
  1002. } else {
  1003. requestPermissions(PERMISSIONS_CAMERA, 100);
  1004. }
  1005. }
  1006. public void switchCamera() {
  1007. CameraVideoCapturer cameraVideoCapturer = (CameraVideoCapturer) videoCapturer;
  1008. if (cameraVideoCapturer != null) {
  1009. cameraVideoCapturer.switchCamera(new CameraVideoCapturer.CameraSwitchHandler() {
  1010. @Override
  1011. public void onCameraSwitchDone(boolean currentCameraIsFront) {
  1012. binding.selfVideoRenderer.setMirror(currentCameraIsFront);
  1013. }
  1014. @Override
  1015. public void onCameraSwitchError(String s) {
  1016. }
  1017. });
  1018. }
  1019. }
  1020. private void toggleMedia(boolean enable, boolean video) {
  1021. String message;
  1022. if (video) {
  1023. message = "videoOff";
  1024. if (enable) {
  1025. binding.cameraButton.setAlpha(1.0f);
  1026. message = "videoOn";
  1027. startVideoCapture();
  1028. } else {
  1029. binding.cameraButton.setAlpha(0.7f);
  1030. if (videoCapturer != null) {
  1031. try {
  1032. videoCapturer.stopCapture();
  1033. } catch (InterruptedException e) {
  1034. Log.d(TAG, "Failed to stop capturing video while sensor is near the ear");
  1035. }
  1036. }
  1037. }
  1038. if (localStream != null && localStream.videoTracks.size() > 0) {
  1039. localStream.videoTracks.get(0).setEnabled(enable);
  1040. }
  1041. if (enable) {
  1042. binding.selfVideoRenderer.setVisibility(View.VISIBLE);
  1043. } else {
  1044. binding.selfVideoRenderer.setVisibility(View.INVISIBLE);
  1045. }
  1046. } else {
  1047. message = "audioOff";
  1048. if (enable) {
  1049. message = "audioOn";
  1050. binding.microphoneButton.setAlpha(1.0f);
  1051. } else {
  1052. binding.microphoneButton.setAlpha(0.7f);
  1053. }
  1054. if (localStream != null && localStream.audioTracks.size() > 0) {
  1055. localStream.audioTracks.get(0).setEnabled(enable);
  1056. }
  1057. }
  1058. if (isConnectionEstablished() && peerConnectionWrapperList != null) {
  1059. if (!hasMCU) {
  1060. for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
  1061. peerConnectionWrapper.sendChannelData(new DataChannelMessage(message));
  1062. }
  1063. } else {
  1064. for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
  1065. if (peerConnectionWrapper.getSessionId().equals(webSocketClient.getSessionId())) {
  1066. peerConnectionWrapper.sendChannelData(new DataChannelMessage(message));
  1067. break;
  1068. }
  1069. }
  1070. }
  1071. }
  1072. }
  1073. public void clickRaiseOrLowerHandButton() {
  1074. raiseHandViewModel.clickHandButton();
  1075. }
  1076. private void animateCallControls(boolean show, long startDelay) {
  1077. if (isVoiceOnlyCall) {
  1078. if (spotlightView != null && spotlightView.getVisibility() != View.GONE) {
  1079. spotlightView.setVisibility(View.GONE);
  1080. }
  1081. } else if (!isPushToTalkActive) {
  1082. float alpha;
  1083. long duration;
  1084. if (show) {
  1085. callControlHandler.removeCallbacksAndMessages(null);
  1086. callInfosHandler.removeCallbacksAndMessages(null);
  1087. cameraSwitchHandler.removeCallbacksAndMessages(null);
  1088. alpha = 1.0f;
  1089. duration = 1000;
  1090. if (binding.callControls.getVisibility() != View.VISIBLE) {
  1091. binding.callControls.setAlpha(0.0f);
  1092. binding.callControls.setVisibility(View.VISIBLE);
  1093. binding.callInfosLinearLayout.setAlpha(0.0f);
  1094. binding.callInfosLinearLayout.setVisibility(View.VISIBLE);
  1095. binding.switchSelfVideoButton.setAlpha(0.0f);
  1096. if (videoOn) {
  1097. binding.switchSelfVideoButton.setVisibility(View.VISIBLE);
  1098. }
  1099. } else {
  1100. callControlHandler.postDelayed(() -> animateCallControls(false, 0), 5000);
  1101. return;
  1102. }
  1103. } else {
  1104. alpha = 0.0f;
  1105. duration = 1000;
  1106. }
  1107. binding.callControls.setEnabled(false);
  1108. binding.callControls.animate()
  1109. .translationY(0)
  1110. .alpha(alpha)
  1111. .setDuration(duration)
  1112. .setStartDelay(startDelay)
  1113. .setListener(new AnimatorListenerAdapter() {
  1114. @Override
  1115. public void onAnimationEnd(Animator animation) {
  1116. super.onAnimationEnd(animation);
  1117. if (!show) {
  1118. binding.callControls.setVisibility(View.GONE);
  1119. if (spotlightView != null && spotlightView.getVisibility() != View.GONE) {
  1120. spotlightView.setVisibility(View.GONE);
  1121. }
  1122. } else {
  1123. callControlHandler.postDelayed(new Runnable() {
  1124. @Override
  1125. public void run() {
  1126. if (!isPushToTalkActive) {
  1127. animateCallControls(false, 0);
  1128. }
  1129. }
  1130. }, 7500);
  1131. }
  1132. binding.callControls.setEnabled(true);
  1133. }
  1134. });
  1135. binding.callInfosLinearLayout.setEnabled(false);
  1136. binding.callInfosLinearLayout.animate()
  1137. .translationY(0)
  1138. .alpha(alpha)
  1139. .setDuration(duration)
  1140. .setStartDelay(startDelay)
  1141. .setListener(new AnimatorListenerAdapter() {
  1142. @Override
  1143. public void onAnimationEnd(Animator animation) {
  1144. super.onAnimationEnd(animation);
  1145. if (!show) {
  1146. binding.callInfosLinearLayout.setVisibility(View.GONE);
  1147. } else {
  1148. callInfosHandler.postDelayed(new Runnable() {
  1149. @Override
  1150. public void run() {
  1151. if (!isPushToTalkActive) {
  1152. animateCallControls(false, 0);
  1153. }
  1154. }
  1155. }, 7500);
  1156. }
  1157. binding.callInfosLinearLayout.setEnabled(true);
  1158. }
  1159. });
  1160. binding.switchSelfVideoButton.setEnabled(false);
  1161. binding.switchSelfVideoButton.animate()
  1162. .translationY(0)
  1163. .alpha(alpha)
  1164. .setDuration(duration)
  1165. .setStartDelay(startDelay)
  1166. .setListener(new AnimatorListenerAdapter() {
  1167. @Override
  1168. public void onAnimationEnd(Animator animation) {
  1169. super.onAnimationEnd(animation);
  1170. if (!show) {
  1171. binding.switchSelfVideoButton.setVisibility(View.GONE);
  1172. }
  1173. binding.switchSelfVideoButton.setEnabled(true);
  1174. }
  1175. });
  1176. }
  1177. }
  1178. @Override
  1179. public void onDestroy() {
  1180. if (signalingMessageReceiver != null) {
  1181. signalingMessageReceiver.removeListener(localParticipantMessageListener);
  1182. signalingMessageReceiver.removeListener(offerMessageListener);
  1183. }
  1184. if (localStream != null) {
  1185. localStream.dispose();
  1186. localStream = null;
  1187. Log.d(TAG, "Disposed localStream");
  1188. } else {
  1189. Log.d(TAG, "localStream is null");
  1190. }
  1191. if (currentCallStatus != CallStatus.LEAVING) {
  1192. hangup(true);
  1193. }
  1194. powerManagerUtils.updatePhoneState(PowerManagerUtils.PhoneState.IDLE);
  1195. super.onDestroy();
  1196. }
  1197. private void fetchSignalingSettings() {
  1198. Log.d(TAG, "fetchSignalingSettings");
  1199. int apiVersion = ApiUtils.getSignalingApiVersion(conversationUser, new int[]{ApiUtils.APIv3, 2, 1});
  1200. ncApi.getSignalingSettings(credentials, ApiUtils.getUrlForSignalingSettings(apiVersion, baseUrl))
  1201. .subscribeOn(Schedulers.io())
  1202. .retry(3)
  1203. .observeOn(AndroidSchedulers.mainThread())
  1204. .subscribe(new Observer<SignalingSettingsOverall>() {
  1205. @Override
  1206. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  1207. // unused atm
  1208. }
  1209. @Override
  1210. public void onNext(@io.reactivex.annotations.NonNull SignalingSettingsOverall signalingSettingsOverall) {
  1211. if (signalingSettingsOverall.getOcs() != null
  1212. && signalingSettingsOverall.getOcs().getSettings() != null) {
  1213. externalSignalingServer = new ExternalSignalingServer();
  1214. if (!TextUtils.isEmpty(
  1215. signalingSettingsOverall.getOcs().getSettings().getExternalSignalingServer()) &&
  1216. !TextUtils.isEmpty(
  1217. signalingSettingsOverall.getOcs().getSettings().getExternalSignalingTicket())) {
  1218. externalSignalingServer = new ExternalSignalingServer();
  1219. externalSignalingServer.setExternalSignalingServer(
  1220. signalingSettingsOverall.getOcs().getSettings().getExternalSignalingServer());
  1221. externalSignalingServer.setExternalSignalingTicket(
  1222. signalingSettingsOverall.getOcs().getSettings().getExternalSignalingTicket());
  1223. hasExternalSignalingServer = true;
  1224. } else {
  1225. hasExternalSignalingServer = false;
  1226. }
  1227. Log.d(TAG, " hasExternalSignalingServer: " + hasExternalSignalingServer);
  1228. if (!"?".equals(conversationUser.getUserId()) && conversationUser.getId() != null) {
  1229. Log.d(TAG, "Update externalSignalingServer for: " + conversationUser.getId() +
  1230. " / " + conversationUser.getUserId());
  1231. userManager.updateExternalSignalingServer(conversationUser.getId(), externalSignalingServer)
  1232. .subscribeOn(Schedulers.io())
  1233. .subscribe();
  1234. } else {
  1235. conversationUser.setExternalSignalingServer(externalSignalingServer);
  1236. }
  1237. if (signalingSettingsOverall.getOcs().getSettings().getStunServers() != null) {
  1238. List<IceServer> stunServers =
  1239. signalingSettingsOverall.getOcs().getSettings().getStunServers();
  1240. if (apiVersion == ApiUtils.APIv3) {
  1241. for (IceServer stunServer : stunServers) {
  1242. if (stunServer.getUrls() != null) {
  1243. for (String url : stunServer.getUrls()) {
  1244. Log.d(TAG, " STUN server url: " + url);
  1245. iceServers.add(new PeerConnection.IceServer(url));
  1246. }
  1247. }
  1248. }
  1249. } else {
  1250. if (signalingSettingsOverall.getOcs().getSettings().getStunServers() != null) {
  1251. for (IceServer stunServer : stunServers) {
  1252. Log.d(TAG, " STUN server url: " + stunServer.getUrl());
  1253. iceServers.add(new PeerConnection.IceServer(stunServer.getUrl()));
  1254. }
  1255. }
  1256. }
  1257. }
  1258. if (signalingSettingsOverall.getOcs().getSettings().getTurnServers() != null) {
  1259. List<IceServer> turnServers =
  1260. signalingSettingsOverall.getOcs().getSettings().getTurnServers();
  1261. for (IceServer turnServer : turnServers) {
  1262. if (turnServer.getUrls() != null) {
  1263. for (String url : turnServer.getUrls()) {
  1264. Log.d(TAG, " TURN server url: " + url);
  1265. iceServers.add(new PeerConnection.IceServer(
  1266. url, turnServer.getUsername(), turnServer.getCredential()
  1267. ));
  1268. }
  1269. }
  1270. }
  1271. }
  1272. }
  1273. checkCapabilities();
  1274. }
  1275. @Override
  1276. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  1277. Log.e(TAG, e.getMessage(), e);
  1278. }
  1279. @Override
  1280. public void onComplete() {
  1281. // unused atm
  1282. }
  1283. });
  1284. }
  1285. private void checkCapabilities() {
  1286. ncApi.getCapabilities(credentials, ApiUtils.getUrlForCapabilities(baseUrl))
  1287. .retry(3)
  1288. .subscribeOn(Schedulers.io())
  1289. .observeOn(AndroidSchedulers.mainThread())
  1290. .subscribe(new Observer<CapabilitiesOverall>() {
  1291. @Override
  1292. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  1293. // unused atm
  1294. }
  1295. @Override
  1296. public void onNext(@io.reactivex.annotations.NonNull CapabilitiesOverall capabilitiesOverall) {
  1297. // FIXME check for compatible Call API version
  1298. if (hasExternalSignalingServer) {
  1299. setupAndInitiateWebSocketsConnection();
  1300. } else {
  1301. signalingMessageReceiver = internalSignalingMessageReceiver;
  1302. signalingMessageReceiver.addListener(localParticipantMessageListener);
  1303. signalingMessageReceiver.addListener(offerMessageListener);
  1304. signalingMessageSender = internalSignalingMessageSender;
  1305. joinRoomAndCall();
  1306. }
  1307. }
  1308. @Override
  1309. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  1310. // unused atm
  1311. }
  1312. @Override
  1313. public void onComplete() {
  1314. // unused atm
  1315. }
  1316. });
  1317. }
  1318. private void joinRoomAndCall() {
  1319. callSession = ApplicationWideCurrentRoomHolder.getInstance().getSession();
  1320. int apiVersion = ApiUtils.getConversationApiVersion(conversationUser, new int[]{ApiUtils.APIv4, 1});
  1321. Log.d(TAG, "joinRoomAndCall");
  1322. Log.d(TAG, " baseUrl= " + baseUrl);
  1323. Log.d(TAG, " roomToken= " + roomToken);
  1324. Log.d(TAG, " callSession= " + callSession);
  1325. String url = ApiUtils.getUrlForParticipantsActive(apiVersion, baseUrl, roomToken);
  1326. Log.d(TAG, " url= " + url);
  1327. if (TextUtils.isEmpty(callSession)) {
  1328. ncApi.joinRoom(credentials, url, conversationPassword)
  1329. .subscribeOn(Schedulers.io())
  1330. .observeOn(AndroidSchedulers.mainThread())
  1331. .retry(3)
  1332. .subscribe(new Observer<RoomOverall>() {
  1333. @Override
  1334. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  1335. // unused atm
  1336. }
  1337. @Override
  1338. public void onNext(@io.reactivex.annotations.NonNull RoomOverall roomOverall) {
  1339. Conversation conversation = roomOverall.getOcs().getData();
  1340. callRecordingViewModel.setRecordingState(conversation.getCallRecording());
  1341. callSession = conversation.getSessionId();
  1342. Log.d(TAG, " new callSession by joinRoom= " + callSession);
  1343. ApplicationWideCurrentRoomHolder.getInstance().setSession(callSession);
  1344. ApplicationWideCurrentRoomHolder.getInstance().setCurrentRoomId(roomId);
  1345. ApplicationWideCurrentRoomHolder.getInstance().setCurrentRoomToken(roomToken);
  1346. ApplicationWideCurrentRoomHolder.getInstance().setUserInRoom(conversationUser);
  1347. callOrJoinRoomViaWebSocket();
  1348. }
  1349. @Override
  1350. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  1351. Log.e(TAG, "joinRoom onError", e);
  1352. }
  1353. @Override
  1354. public void onComplete() {
  1355. Log.d(TAG, "joinRoom onComplete");
  1356. }
  1357. });
  1358. } else {
  1359. // we are in a room and start a call -> same session needs to be used
  1360. callOrJoinRoomViaWebSocket();
  1361. }
  1362. }
  1363. private void callOrJoinRoomViaWebSocket() {
  1364. if (hasExternalSignalingServer) {
  1365. webSocketClient.joinRoomWithRoomTokenAndSession(roomToken, callSession);
  1366. } else {
  1367. performCall();
  1368. }
  1369. }
  1370. private void performCall() {
  1371. int inCallFlag = Participant.InCallFlags.IN_CALL;
  1372. if (canPublishAudioStream) {
  1373. inCallFlag += Participant.InCallFlags.WITH_AUDIO;
  1374. }
  1375. if (!isVoiceOnlyCall && canPublishVideoStream) {
  1376. inCallFlag += Participant.InCallFlags.WITH_VIDEO;
  1377. }
  1378. callParticipantList = new CallParticipantList(signalingMessageReceiver);
  1379. callParticipantList.addObserver(callParticipantListObserver);
  1380. int apiVersion = ApiUtils.getCallApiVersion(conversationUser, new int[]{ApiUtils.APIv4, 1});
  1381. ncApi.joinCall(
  1382. credentials,
  1383. ApiUtils.getUrlForCall(apiVersion, baseUrl, roomToken),
  1384. inCallFlag,
  1385. isCallWithoutNotification)
  1386. .subscribeOn(Schedulers.io())
  1387. .retry(3)
  1388. .observeOn(AndroidSchedulers.mainThread())
  1389. .subscribe(new Observer<GenericOverall>() {
  1390. @Override
  1391. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  1392. // unused atm
  1393. }
  1394. @Override
  1395. public void onNext(@io.reactivex.annotations.NonNull GenericOverall genericOverall) {
  1396. if (currentCallStatus != CallStatus.LEAVING) {
  1397. if (currentCallStatus != CallStatus.IN_CONVERSATION) {
  1398. setCallState(CallStatus.JOINED);
  1399. }
  1400. ApplicationWideCurrentRoomHolder.getInstance().setInCall(true);
  1401. ApplicationWideCurrentRoomHolder.getInstance().setDialing(false);
  1402. if (!TextUtils.isEmpty(roomToken)) {
  1403. NotificationUtils.INSTANCE.cancelExistingNotificationsForRoom(getApplicationContext(),
  1404. conversationUser,
  1405. roomToken);
  1406. }
  1407. if (!hasExternalSignalingServer) {
  1408. int apiVersion = ApiUtils.getSignalingApiVersion(conversationUser,
  1409. new int[]{ApiUtils.APIv3, 2, 1});
  1410. AtomicInteger delayOnError = new AtomicInteger(0);
  1411. ncApi.pullSignalingMessages(credentials,
  1412. ApiUtils.getUrlForSignaling(apiVersion,
  1413. baseUrl,
  1414. roomToken))
  1415. .subscribeOn(Schedulers.io())
  1416. .observeOn(AndroidSchedulers.mainThread())
  1417. .repeatWhen(observable -> observable)
  1418. .takeWhile(observable -> isConnectionEstablished())
  1419. .doOnNext(value -> delayOnError.set(0))
  1420. .retryWhen(errors -> errors
  1421. .flatMap(error -> {
  1422. if (!isConnectionEstablished()) {
  1423. return Observable.error(error);
  1424. }
  1425. if (delayOnError.get() == 0) {
  1426. delayOnError.set(1);
  1427. } else if (delayOnError.get() < 16) {
  1428. delayOnError.set(delayOnError.get() * 2);
  1429. }
  1430. return Observable.timer(delayOnError.get(), TimeUnit.SECONDS);
  1431. })
  1432. )
  1433. .subscribe(new Observer<SignalingOverall>() {
  1434. @Override
  1435. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  1436. signalingDisposable = d;
  1437. }
  1438. @Override
  1439. public void onNext(
  1440. @io.reactivex.annotations.NonNull
  1441. SignalingOverall signalingOverall) {
  1442. receivedSignalingMessages(signalingOverall.getOcs().getSignalings());
  1443. }
  1444. @Override
  1445. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  1446. dispose(signalingDisposable);
  1447. }
  1448. @Override
  1449. public void onComplete() {
  1450. dispose(signalingDisposable);
  1451. }
  1452. });
  1453. }
  1454. }
  1455. }
  1456. @Override
  1457. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  1458. // unused atm
  1459. }
  1460. @Override
  1461. public void onComplete() {
  1462. // unused atm
  1463. }
  1464. });
  1465. }
  1466. private void setupAndInitiateWebSocketsConnection() {
  1467. if (webSocketConnectionHelper == null) {
  1468. webSocketConnectionHelper = new WebSocketConnectionHelper();
  1469. }
  1470. if (webSocketClient == null) {
  1471. webSocketClient = WebSocketConnectionHelper.getExternalSignalingInstanceForServer(
  1472. externalSignalingServer.getExternalSignalingServer(),
  1473. conversationUser, externalSignalingServer.getExternalSignalingTicket(),
  1474. TextUtils.isEmpty(credentials));
  1475. // Although setupAndInitiateWebSocketsConnection could be called several times the web socket is
  1476. // initialized just once, so the message receiver is also initialized just once.
  1477. signalingMessageReceiver = webSocketClient.getSignalingMessageReceiver();
  1478. signalingMessageReceiver.addListener(localParticipantMessageListener);
  1479. signalingMessageReceiver.addListener(offerMessageListener);
  1480. signalingMessageSender = webSocketClient.getSignalingMessageSender();
  1481. } else {
  1482. if (webSocketClient.isConnected() && currentCallStatus == CallStatus.PUBLISHER_FAILED) {
  1483. webSocketClient.restartWebSocket();
  1484. }
  1485. }
  1486. joinRoomAndCall();
  1487. }
  1488. private void initiateCall() {
  1489. if (!TextUtils.isEmpty(roomToken)) {
  1490. checkDevicePermissions();
  1491. } else {
  1492. handleFromNotification();
  1493. }
  1494. }
  1495. @Subscribe(threadMode = ThreadMode.BACKGROUND)
  1496. public void onMessageEvent(WebSocketCommunicationEvent webSocketCommunicationEvent) {
  1497. if (currentCallStatus == CallStatus.LEAVING) {
  1498. return;
  1499. }
  1500. if (webSocketCommunicationEvent.getHashMap() != null) {
  1501. switch (webSocketCommunicationEvent.getType()) {
  1502. case "hello":
  1503. Log.d(TAG, "onMessageEvent 'hello'");
  1504. if (!webSocketCommunicationEvent.getHashMap().containsKey("oldResumeId")) {
  1505. if (currentCallStatus == CallStatus.RECONNECTING) {
  1506. hangup(false);
  1507. } else {
  1508. setCallState(CallStatus.RECONNECTING);
  1509. runOnUiThread(this::initiateCall);
  1510. }
  1511. }
  1512. break;
  1513. case "roomJoined":
  1514. Log.d(TAG, "onMessageEvent 'roomJoined'");
  1515. startSendingNick();
  1516. if (webSocketCommunicationEvent.getHashMap().get("roomToken").equals(roomToken)) {
  1517. performCall();
  1518. }
  1519. break;
  1520. case "recordingStatus":
  1521. Log.d(TAG, "onMessageEvent 'recordingStatus'");
  1522. if (webSocketCommunicationEvent.getHashMap().containsKey(KEY_RECORDING_STATE)) {
  1523. String recordingStateString =
  1524. webSocketCommunicationEvent.getHashMap().get(KEY_RECORDING_STATE);
  1525. if (recordingStateString != null) {
  1526. runOnUiThread(() -> {
  1527. callRecordingViewModel.setRecordingState(Integer.parseInt(recordingStateString));
  1528. });
  1529. }
  1530. }
  1531. break;
  1532. }
  1533. }
  1534. }
  1535. private void dispose(@Nullable Disposable disposable) {
  1536. if (disposable != null && !disposable.isDisposed()) {
  1537. disposable.dispose();
  1538. } else if (disposable == null) {
  1539. if (signalingDisposable != null && !signalingDisposable.isDisposed()) {
  1540. signalingDisposable.dispose();
  1541. signalingDisposable = null;
  1542. }
  1543. }
  1544. }
  1545. private void receivedSignalingMessages(@Nullable List<Signaling> signalingList) {
  1546. if (signalingList != null) {
  1547. for (Signaling signaling : signalingList) {
  1548. try {
  1549. receivedSignalingMessage(signaling);
  1550. } catch (IOException e) {
  1551. Log.e(TAG, "Failed to process received signaling message", e);
  1552. }
  1553. }
  1554. }
  1555. }
  1556. private void receivedSignalingMessage(Signaling signaling) throws IOException {
  1557. String messageType = signaling.getType();
  1558. if (!isConnectionEstablished() && currentCallStatus != CallStatus.CONNECTING) {
  1559. return;
  1560. }
  1561. if ("usersInRoom".equals(messageType)) {
  1562. internalSignalingMessageReceiver.process((List<Map<String, Object>>) signaling.getMessageWrapper());
  1563. } else if ("message".equals(messageType)) {
  1564. NCSignalingMessage ncSignalingMessage = LoganSquare.parse(signaling.getMessageWrapper().toString(),
  1565. NCSignalingMessage.class);
  1566. internalSignalingMessageReceiver.process(ncSignalingMessage);
  1567. } else {
  1568. Log.e(TAG, "unexpected message type when receiving signaling message");
  1569. }
  1570. }
  1571. private void hangup(boolean shutDownView) {
  1572. Log.d(TAG, "hangup! shutDownView=" + shutDownView);
  1573. if (shutDownView) {
  1574. setCallState(CallStatus.LEAVING);
  1575. }
  1576. stopCallingSound();
  1577. dispose(null);
  1578. if (shutDownView) {
  1579. if (videoCapturer != null) {
  1580. try {
  1581. videoCapturer.stopCapture();
  1582. } catch (InterruptedException e) {
  1583. Log.e(TAG, "Failed to stop capturing while hanging up");
  1584. }
  1585. videoCapturer.dispose();
  1586. videoCapturer = null;
  1587. }
  1588. binding.selfVideoRenderer.release();
  1589. if (audioSource != null) {
  1590. audioSource.dispose();
  1591. audioSource = null;
  1592. }
  1593. runOnUiThread(() -> {
  1594. if (audioManager != null) {
  1595. audioManager.stop();
  1596. audioManager = null;
  1597. }
  1598. });
  1599. if (videoSource != null) {
  1600. videoSource = null;
  1601. }
  1602. if (peerConnectionFactory != null) {
  1603. peerConnectionFactory = null;
  1604. }
  1605. localAudioTrack = null;
  1606. localVideoTrack = null;
  1607. if (TextUtils.isEmpty(credentials) && hasExternalSignalingServer) {
  1608. WebSocketConnectionHelper.deleteExternalSignalingInstanceForUserEntity(-1);
  1609. }
  1610. }
  1611. List<String> peerConnectionIdsToEnd = new ArrayList<String>(peerConnectionWrapperList.size());
  1612. for (PeerConnectionWrapper wrapper : peerConnectionWrapperList) {
  1613. peerConnectionIdsToEnd.add(wrapper.getSessionId());
  1614. }
  1615. for (String sessionId : peerConnectionIdsToEnd) {
  1616. endPeerConnection(sessionId, "video");
  1617. endPeerConnection(sessionId, "screen");
  1618. }
  1619. List<String> callParticipantIdsToEnd = new ArrayList<String>(peerConnectionWrapperList.size());
  1620. for (CallParticipant callParticipant : callParticipants.values()) {
  1621. callParticipantIdsToEnd.add(callParticipant.getCallParticipantModel().getSessionId());
  1622. }
  1623. for (String sessionId : callParticipantIdsToEnd) {
  1624. removeCallParticipant(sessionId);
  1625. }
  1626. hangupNetworkCalls(shutDownView);
  1627. ApplicationWideCurrentRoomHolder.getInstance().setInCall(false);
  1628. }
  1629. private void hangupNetworkCalls(boolean shutDownView) {
  1630. Log.d(TAG, "hangupNetworkCalls. shutDownView=" + shutDownView);
  1631. int apiVersion = ApiUtils.getCallApiVersion(conversationUser, new int[]{ApiUtils.APIv4, 1});
  1632. if (callParticipantList != null) {
  1633. callParticipantList.removeObserver(callParticipantListObserver);
  1634. callParticipantList.destroy();
  1635. }
  1636. ncApi.leaveCall(credentials, ApiUtils.getUrlForCall(apiVersion, baseUrl, roomToken))
  1637. .subscribeOn(Schedulers.io())
  1638. .observeOn(AndroidSchedulers.mainThread())
  1639. .subscribe(new Observer<GenericOverall>() {
  1640. @Override
  1641. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  1642. // unused atm
  1643. }
  1644. @Override
  1645. public void onNext(@io.reactivex.annotations.NonNull GenericOverall genericOverall) {
  1646. if (!switchToRoomToken.isEmpty()) {
  1647. Intent intent = new Intent(context, MainActivity.class);
  1648. intent.addFlags(Intent.FLAG_ACTIVITY_SINGLE_TOP | Intent.FLAG_ACTIVITY_NEW_TASK);
  1649. Bundle bundle = new Bundle();
  1650. bundle.putBoolean(KEY_SWITCH_TO_ROOM_AND_START_CALL, true);
  1651. bundle.putString(KEY_ROOM_TOKEN, switchToRoomToken);
  1652. bundle.putParcelable(KEY_USER_ENTITY, conversationUser);
  1653. bundle.putBoolean(KEY_CALL_VOICE_ONLY, isVoiceOnlyCall);
  1654. intent.putExtras(bundle);
  1655. startActivity(intent);
  1656. if (isBreakoutRoom) {
  1657. Toast.makeText(context, context.getResources().getString(R.string.switch_to_main_room),
  1658. Toast.LENGTH_LONG).show();
  1659. } else {
  1660. Toast.makeText(context, context.getResources().getString(R.string.switch_to_breakout_room),
  1661. Toast.LENGTH_LONG).show();
  1662. }
  1663. finish();
  1664. } else if (shutDownView) {
  1665. finish();
  1666. } else if (currentCallStatus == CallStatus.RECONNECTING
  1667. || currentCallStatus == CallStatus.PUBLISHER_FAILED) {
  1668. initiateCall();
  1669. }
  1670. }
  1671. @Override
  1672. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  1673. // unused atm
  1674. }
  1675. @Override
  1676. public void onComplete() {
  1677. // unused atm
  1678. }
  1679. });
  1680. }
  1681. private void startVideoCapture() {
  1682. if (videoCapturer != null) {
  1683. videoCapturer.startCapture(1280, 720, 30);
  1684. }
  1685. }
  1686. private void handleCallParticipantsChanged(Collection<Participant> joined, Collection<Participant> updated,
  1687. Collection<Participant> left, Collection<Participant> unchanged) {
  1688. Log.d(TAG, "handleCallParticipantsChanged");
  1689. hasMCU = hasExternalSignalingServer && webSocketClient != null && webSocketClient.hasMCU();
  1690. Log.d(TAG, " hasMCU is " + hasMCU);
  1691. // The signaling session is the same as the Nextcloud session only when the MCU is not used.
  1692. String currentSessionId = callSession;
  1693. if (hasMCU) {
  1694. currentSessionId = webSocketClient.getSessionId();
  1695. }
  1696. Log.d(TAG, " currentSessionId is " + currentSessionId);
  1697. List<Participant> participantsInCall = new ArrayList<>();
  1698. participantsInCall.addAll(joined);
  1699. participantsInCall.addAll(updated);
  1700. participantsInCall.addAll(unchanged);
  1701. boolean isSelfInCall = false;
  1702. Participant selfParticipant = null;
  1703. for (Participant participant : participantsInCall) {
  1704. long inCallFlag = participant.getInCall();
  1705. if (!participant.getSessionId().equals(currentSessionId)) {
  1706. Log.d(TAG, " inCallFlag of participant "
  1707. + participant.getSessionId().substring(0, 4)
  1708. + " : "
  1709. + inCallFlag);
  1710. } else {
  1711. Log.d(TAG, " inCallFlag of currentSessionId: " + inCallFlag);
  1712. isSelfInCall = inCallFlag != 0;
  1713. selfParticipant = participant;
  1714. }
  1715. }
  1716. if (!isSelfInCall && currentCallStatus != CallStatus.LEAVING && ApplicationWideCurrentRoomHolder.getInstance().isInCall()) {
  1717. Log.d(TAG, "Most probably a moderator ended the call for all.");
  1718. hangup(true);
  1719. return;
  1720. }
  1721. if (!isSelfInCall) {
  1722. Log.d(TAG, "Self not in call, disconnecting from all other sessions");
  1723. for (Participant participant : participantsInCall) {
  1724. String sessionId = participant.getSessionId();
  1725. Log.d(TAG, " session that will be removed is: " + sessionId);
  1726. endPeerConnection(sessionId, "video");
  1727. endPeerConnection(sessionId, "screen");
  1728. removeCallParticipant(sessionId);
  1729. }
  1730. return;
  1731. }
  1732. if (currentCallStatus == CallStatus.LEAVING) {
  1733. return;
  1734. }
  1735. if (hasMCU) {
  1736. // Ensure that own publishing peer is set up.
  1737. getOrCreatePeerConnectionWrapperForSessionIdAndType(webSocketClient.getSessionId(), VIDEO_STREAM_TYPE_VIDEO, true);
  1738. }
  1739. boolean selfJoined = false;
  1740. boolean selfParticipantHasAudioOrVideo = participantInCallFlagsHaveAudioOrVideo(selfParticipant);
  1741. for (Participant participant : joined) {
  1742. String sessionId = participant.getSessionId();
  1743. if (sessionId == null) {
  1744. Log.w(TAG, "Null sessionId for call participant, this should not happen: " + participant);
  1745. continue;
  1746. }
  1747. if (sessionId.equals(currentSessionId)) {
  1748. selfJoined = true;
  1749. continue;
  1750. }
  1751. Log.d(TAG, " newSession joined: " + sessionId);
  1752. CallParticipant callParticipant = addCallParticipant(sessionId);
  1753. String userId = participant.getUserId();
  1754. if (userId != null) {
  1755. callParticipants.get(sessionId).setUserId(userId);
  1756. }
  1757. String nick;
  1758. if (hasExternalSignalingServer) {
  1759. nick = webSocketClient.getDisplayNameForSession(sessionId);
  1760. } else {
  1761. nick = offerAnswerNickProviders.get(sessionId) != null ? offerAnswerNickProviders.get(sessionId).getNick() : "";
  1762. }
  1763. callParticipants.get(sessionId).setNick(nick);
  1764. boolean participantHasAudioOrVideo = participantInCallFlagsHaveAudioOrVideo(participant);
  1765. // FIXME Without MCU, PeerConnectionWrapper only sends an offer if the local session ID is higher than the
  1766. // remote session ID. However, if the other participant does not have audio nor video that participant
  1767. // will not send an offer, so no connection is actually established when the remote participant has a
  1768. // higher session ID but is not publishing media.
  1769. if ((hasMCU && participantHasAudioOrVideo) ||
  1770. (!hasMCU && selfParticipantHasAudioOrVideo && (!participantHasAudioOrVideo || sessionId.compareTo(currentSessionId) < 0))) {
  1771. getOrCreatePeerConnectionWrapperForSessionIdAndType(sessionId, VIDEO_STREAM_TYPE_VIDEO, false);
  1772. }
  1773. }
  1774. boolean othersInCall = selfJoined ? joined.size() > 1 : joined.size() > 0;
  1775. if (othersInCall && currentCallStatus != CallStatus.IN_CONVERSATION) {
  1776. setCallState(CallStatus.IN_CONVERSATION);
  1777. }
  1778. for (Participant participant : left) {
  1779. String sessionId = participant.getSessionId();
  1780. Log.d(TAG, " oldSession that will be removed is: " + sessionId);
  1781. endPeerConnection(sessionId, "video");
  1782. endPeerConnection(sessionId, "screen");
  1783. removeCallParticipant(sessionId);
  1784. }
  1785. }
  1786. private boolean participantInCallFlagsHaveAudioOrVideo(Participant participant) {
  1787. if (participant == null) {
  1788. return false;
  1789. }
  1790. return (participant.getInCall() & Participant.InCallFlags.WITH_AUDIO) > 0 ||
  1791. (!isVoiceOnlyCall && (participant.getInCall() & Participant.InCallFlags.WITH_VIDEO) > 0);
  1792. }
  1793. private PeerConnectionWrapper getPeerConnectionWrapperForSessionIdAndType(String sessionId, String type) {
  1794. for (PeerConnectionWrapper wrapper : peerConnectionWrapperList) {
  1795. if (wrapper.getSessionId().equals(sessionId)
  1796. && wrapper.getVideoStreamType().equals(type)) {
  1797. return wrapper;
  1798. }
  1799. }
  1800. return null;
  1801. }
  1802. private PeerConnectionWrapper getOrCreatePeerConnectionWrapperForSessionIdAndType(String sessionId,
  1803. String type,
  1804. boolean publisher) {
  1805. PeerConnectionWrapper peerConnectionWrapper;
  1806. if ((peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(sessionId, type)) != null) {
  1807. return peerConnectionWrapper;
  1808. } else {
  1809. if (peerConnectionFactory == null) {
  1810. Log.e(TAG, "peerConnectionFactory was null in getOrCreatePeerConnectionWrapperForSessionIdAndType.");
  1811. Toast.makeText(context, context.getResources().getString(R.string.nc_common_error_sorry),
  1812. Toast.LENGTH_LONG).show();
  1813. hangup(true);
  1814. return null;
  1815. }
  1816. if (hasMCU && publisher) {
  1817. peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory,
  1818. iceServers,
  1819. sdpConstraintsForMCU,
  1820. sessionId,
  1821. callSession,
  1822. localStream,
  1823. true,
  1824. true,
  1825. type,
  1826. signalingMessageReceiver,
  1827. signalingMessageSender);
  1828. } else if (hasMCU) {
  1829. peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory,
  1830. iceServers,
  1831. sdpConstraints,
  1832. sessionId,
  1833. callSession,
  1834. null,
  1835. false,
  1836. true,
  1837. type,
  1838. signalingMessageReceiver,
  1839. signalingMessageSender);
  1840. } else {
  1841. if (!"screen".equals(type)) {
  1842. peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory,
  1843. iceServers,
  1844. sdpConstraints,
  1845. sessionId,
  1846. callSession,
  1847. localStream,
  1848. false,
  1849. false,
  1850. type,
  1851. signalingMessageReceiver,
  1852. signalingMessageSender);
  1853. } else {
  1854. peerConnectionWrapper = new PeerConnectionWrapper(peerConnectionFactory,
  1855. iceServers,
  1856. sdpConstraints,
  1857. sessionId,
  1858. callSession,
  1859. null,
  1860. false,
  1861. false,
  1862. type,
  1863. signalingMessageReceiver,
  1864. signalingMessageSender);
  1865. }
  1866. }
  1867. peerConnectionWrapperList.add(peerConnectionWrapper);
  1868. if (!publisher) {
  1869. CallParticipant callParticipant = callParticipants.get(sessionId);
  1870. if (callParticipant == null) {
  1871. callParticipant = addCallParticipant(sessionId);
  1872. }
  1873. if ("screen".equals(type)) {
  1874. callParticipant.setScreenPeerConnectionWrapper(peerConnectionWrapper);
  1875. } else {
  1876. callParticipant.setPeerConnectionWrapper(peerConnectionWrapper);
  1877. }
  1878. }
  1879. if (publisher) {
  1880. peerConnectionWrapper.addObserver(selfPeerConnectionObserver);
  1881. startSendingNick();
  1882. }
  1883. return peerConnectionWrapper;
  1884. }
  1885. }
  1886. private CallParticipant addCallParticipant(String sessionId) {
  1887. CallParticipant callParticipant = new CallParticipant(sessionId, signalingMessageReceiver);
  1888. callParticipants.put(sessionId, callParticipant);
  1889. SignalingMessageReceiver.CallParticipantMessageListener callParticipantMessageListener =
  1890. new CallActivityCallParticipantMessageListener(sessionId);
  1891. callParticipantMessageListeners.put(sessionId, callParticipantMessageListener);
  1892. signalingMessageReceiver.addListener(callParticipantMessageListener, sessionId);
  1893. if (!hasExternalSignalingServer) {
  1894. OfferAnswerNickProvider offerAnswerNickProvider = new OfferAnswerNickProvider(sessionId);
  1895. offerAnswerNickProviders.put(sessionId, offerAnswerNickProvider);
  1896. signalingMessageReceiver.addListener(offerAnswerNickProvider.getVideoWebRtcMessageListener(), sessionId, "video");
  1897. signalingMessageReceiver.addListener(offerAnswerNickProvider.getScreenWebRtcMessageListener(), sessionId, "screen");
  1898. }
  1899. final CallParticipantModel callParticipantModel = callParticipant.getCallParticipantModel();
  1900. ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager =
  1901. new ScreenParticipantDisplayItemManager(callParticipantModel);
  1902. screenParticipantDisplayItemManagers.put(sessionId, screenParticipantDisplayItemManager);
  1903. callParticipantModel.addObserver(screenParticipantDisplayItemManager, screenParticipantDisplayItemManagersHandler);
  1904. runOnUiThread(() -> {
  1905. addParticipantDisplayItem(callParticipantModel, "video");
  1906. });
  1907. return callParticipant;
  1908. }
  1909. private void endPeerConnection(String sessionId, String type) {
  1910. PeerConnectionWrapper peerConnectionWrapper = getPeerConnectionWrapperForSessionIdAndType(sessionId, type);
  1911. if (peerConnectionWrapper == null) {
  1912. return;
  1913. }
  1914. if (webSocketClient != null && webSocketClient.getSessionId() != null && webSocketClient.getSessionId().equals(sessionId)) {
  1915. peerConnectionWrapper.removeObserver(selfPeerConnectionObserver);
  1916. }
  1917. CallParticipant callParticipant = callParticipants.get(sessionId);
  1918. if (callParticipant != null) {
  1919. if ("screen".equals(type)) {
  1920. callParticipant.setScreenPeerConnectionWrapper(null);
  1921. } else {
  1922. callParticipant.setPeerConnectionWrapper(null);
  1923. }
  1924. }
  1925. peerConnectionWrapper.removePeerConnection();
  1926. peerConnectionWrapperList.remove(peerConnectionWrapper);
  1927. }
  1928. private void removeCallParticipant(String sessionId) {
  1929. CallParticipant callParticipant = callParticipants.remove(sessionId);
  1930. if (callParticipant == null) {
  1931. return;
  1932. }
  1933. ScreenParticipantDisplayItemManager screenParticipantDisplayItemManager =
  1934. screenParticipantDisplayItemManagers.remove(sessionId);
  1935. callParticipant.getCallParticipantModel().removeObserver(screenParticipantDisplayItemManager);
  1936. callParticipant.destroy();
  1937. SignalingMessageReceiver.CallParticipantMessageListener listener = callParticipantMessageListeners.remove(sessionId);
  1938. signalingMessageReceiver.removeListener(listener);
  1939. OfferAnswerNickProvider offerAnswerNickProvider = offerAnswerNickProviders.remove(sessionId);
  1940. if (offerAnswerNickProvider != null) {
  1941. signalingMessageReceiver.removeListener(offerAnswerNickProvider.getVideoWebRtcMessageListener());
  1942. signalingMessageReceiver.removeListener(offerAnswerNickProvider.getScreenWebRtcMessageListener());
  1943. }
  1944. runOnUiThread(() -> removeParticipantDisplayItem(sessionId, "video"));
  1945. }
  1946. private void removeParticipantDisplayItem(String sessionId, String videoStreamType) {
  1947. Log.d(TAG, "removeParticipantDisplayItem");
  1948. ParticipantDisplayItem participantDisplayItem = participantDisplayItems.remove(sessionId + "-" + videoStreamType);
  1949. if (participantDisplayItem == null) {
  1950. return;
  1951. }
  1952. participantDisplayItem.destroy();
  1953. if (!isDestroyed()) {
  1954. initGridAdapter();
  1955. }
  1956. }
  1957. @Subscribe(threadMode = ThreadMode.MAIN)
  1958. public void onMessageEvent(ConfigurationChangeEvent configurationChangeEvent) {
  1959. powerManagerUtils.setOrientation(Objects.requireNonNull(getResources()).getConfiguration().orientation);
  1960. initGridAdapter();
  1961. updateSelfVideoViewPosition();
  1962. }
  1963. private void updateSelfVideoViewIceConnectionState(PeerConnection.IceConnectionState iceConnectionState) {
  1964. boolean connected = iceConnectionState == PeerConnection.IceConnectionState.CONNECTED ||
  1965. iceConnectionState == PeerConnection.IceConnectionState.COMPLETED;
  1966. // FIXME In voice only calls there is no video view, so the progress bar would appear floating in the middle of
  1967. // nowhere. However, a way to signal that the local participant is not connected to the HPB is still need in
  1968. // that case.
  1969. if (!connected && !isVoiceOnlyCall) {
  1970. binding.selfVideoViewProgressBar.setVisibility(View.VISIBLE);
  1971. } else {
  1972. binding.selfVideoViewProgressBar.setVisibility(View.GONE);
  1973. }
  1974. }
  1975. private void updateSelfVideoViewPosition() {
  1976. Log.d(TAG, "updateSelfVideoViewPosition");
  1977. if (!isInPipMode) {
  1978. FrameLayout.LayoutParams layoutParams =
  1979. (FrameLayout.LayoutParams) binding.selfVideoRenderer.getLayoutParams();
  1980. DisplayMetrics displayMetrics = getApplicationContext().getResources().getDisplayMetrics();
  1981. int screenWidthPx = displayMetrics.widthPixels;
  1982. int screenWidthDp = (int) DisplayUtils.convertPixelToDp(screenWidthPx, getApplicationContext());
  1983. float newXafterRotate = 0;
  1984. float newYafterRotate;
  1985. if (binding.callInfosLinearLayout.getVisibility() == View.VISIBLE) {
  1986. newYafterRotate = 250;
  1987. } else {
  1988. newYafterRotate = 20;
  1989. }
  1990. if (getResources().getConfiguration().orientation == Configuration.ORIENTATION_LANDSCAPE) {
  1991. layoutParams.height = (int) getResources().getDimension(R.dimen.call_self_video_short_side_length);
  1992. layoutParams.width = (int) getResources().getDimension(R.dimen.call_self_video_long_side_length);
  1993. newXafterRotate = (float) (screenWidthDp - getResources().getDimension(R.dimen.call_self_video_short_side_length) * 0.8);
  1994. } else if (getResources().getConfiguration().orientation == Configuration.ORIENTATION_PORTRAIT) {
  1995. layoutParams.height = (int) getResources().getDimension(R.dimen.call_self_video_long_side_length);
  1996. layoutParams.width = (int) getResources().getDimension(R.dimen.call_self_video_short_side_length);
  1997. newXafterRotate = (float) (screenWidthDp - getResources().getDimension(R.dimen.call_self_video_short_side_length) * 0.5);
  1998. }
  1999. binding.selfVideoRenderer.setLayoutParams(layoutParams);
  2000. int newXafterRotatePx = (int) DisplayUtils.convertDpToPixel(newXafterRotate, getApplicationContext());
  2001. binding.selfVideoViewWrapper.setY(newYafterRotate);
  2002. binding.selfVideoViewWrapper.setX(newXafterRotatePx);
  2003. }
  2004. }
  2005. @Subscribe(threadMode = ThreadMode.MAIN)
  2006. public void onMessageEvent(ProximitySensorEvent proximitySensorEvent) {
  2007. if (!isVoiceOnlyCall) {
  2008. boolean enableVideo = proximitySensorEvent.getProximitySensorEventType() ==
  2009. ProximitySensorEvent.ProximitySensorEventType.SENSOR_FAR && videoOn;
  2010. if (EffortlessPermissions.hasPermissions(this, PERMISSIONS_CAMERA) &&
  2011. (currentCallStatus == CallStatus.CONNECTING || isConnectionEstablished()) && videoOn
  2012. && enableVideo != localVideoTrack.enabled()) {
  2013. toggleMedia(enableVideo, true);
  2014. }
  2015. }
  2016. }
  2017. private void startSendingNick() {
  2018. DataChannelMessage dataChannelMessage = new DataChannelMessage();
  2019. dataChannelMessage.setType("nickChanged");
  2020. Map<String, String> nickChangedPayload = new HashMap<>();
  2021. nickChangedPayload.put("userid", conversationUser.getUserId());
  2022. nickChangedPayload.put("name", conversationUser.getDisplayName());
  2023. dataChannelMessage.setPayloadMap(nickChangedPayload);
  2024. for (PeerConnectionWrapper peerConnectionWrapper : peerConnectionWrapperList) {
  2025. if (peerConnectionWrapper.isMCUPublisher()) {
  2026. Observable
  2027. .interval(1, TimeUnit.SECONDS)
  2028. .repeatUntil(() -> (!isConnectionEstablished() || isDestroyed()))
  2029. .observeOn(Schedulers.io())
  2030. .subscribe(new Observer<Long>() {
  2031. @Override
  2032. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  2033. // unused atm
  2034. }
  2035. @Override
  2036. public void onNext(@io.reactivex.annotations.NonNull Long aLong) {
  2037. peerConnectionWrapper.sendChannelData(dataChannelMessage);
  2038. }
  2039. @Override
  2040. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  2041. // unused atm
  2042. }
  2043. @Override
  2044. public void onComplete() {
  2045. // unused atm
  2046. }
  2047. });
  2048. break;
  2049. }
  2050. }
  2051. }
  2052. @Override
  2053. public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions,
  2054. @NonNull int[] grantResults) {
  2055. super.onRequestPermissionsResult(requestCode, permissions, grantResults);
  2056. EffortlessPermissions.onRequestPermissionsResult(requestCode, permissions, grantResults,
  2057. this);
  2058. }
  2059. private void addParticipantDisplayItem(CallParticipantModel callParticipantModel, String videoStreamType) {
  2060. String defaultGuestNick = getResources().getString(R.string.nc_nick_guest);
  2061. ParticipantDisplayItem participantDisplayItem = new ParticipantDisplayItem(baseUrl,
  2062. defaultGuestNick,
  2063. rootEglBase,
  2064. videoStreamType,
  2065. callParticipantModel);
  2066. String sessionId = callParticipantModel.getSessionId();
  2067. participantDisplayItems.put(sessionId + "-" + videoStreamType, participantDisplayItem);
  2068. initGridAdapter();
  2069. }
  2070. private void setCallState(CallStatus callState) {
  2071. if (currentCallStatus == null || currentCallStatus != callState) {
  2072. currentCallStatus = callState;
  2073. if (handler == null) {
  2074. handler = new Handler(Looper.getMainLooper());
  2075. } else {
  2076. handler.removeCallbacksAndMessages(null);
  2077. }
  2078. switch (callState) {
  2079. case CONNECTING:
  2080. handler.post(() -> {
  2081. playCallingSound();
  2082. if (isIncomingCallFromNotification) {
  2083. binding.callStates.callStateTextView.setText(R.string.nc_call_incoming);
  2084. } else {
  2085. binding.callStates.callStateTextView.setText(R.string.nc_call_ringing);
  2086. }
  2087. binding.callConversationNameTextView.setText(conversationName);
  2088. binding.callModeTextView.setText(getDescriptionForCallType());
  2089. if (binding.callStates.callStateRelativeLayout.getVisibility() != View.VISIBLE) {
  2090. binding.callStates.callStateRelativeLayout.setVisibility(View.VISIBLE);
  2091. }
  2092. if (binding.gridview.getVisibility() != View.INVISIBLE) {
  2093. binding.gridview.setVisibility(View.INVISIBLE);
  2094. }
  2095. if (binding.callStates.callStateProgressBar.getVisibility() != View.VISIBLE) {
  2096. binding.callStates.callStateProgressBar.setVisibility(View.VISIBLE);
  2097. }
  2098. if (binding.callStates.errorImageView.getVisibility() != View.GONE) {
  2099. binding.callStates.errorImageView.setVisibility(View.GONE);
  2100. }
  2101. });
  2102. break;
  2103. case CALLING_TIMEOUT:
  2104. handler.post(() -> {
  2105. hangup(false);
  2106. binding.callStates.callStateTextView.setText(R.string.nc_call_timeout);
  2107. binding.callModeTextView.setText(getDescriptionForCallType());
  2108. if (binding.callStates.callStateRelativeLayout.getVisibility() != View.VISIBLE) {
  2109. binding.callStates.callStateRelativeLayout.setVisibility(View.VISIBLE);
  2110. }
  2111. if (binding.callStates.callStateProgressBar.getVisibility() != View.GONE) {
  2112. binding.callStates.callStateProgressBar.setVisibility(View.GONE);
  2113. }
  2114. if (binding.gridview.getVisibility() != View.INVISIBLE) {
  2115. binding.gridview.setVisibility(View.INVISIBLE);
  2116. }
  2117. binding.callStates.errorImageView.setImageResource(R.drawable.ic_av_timer_timer_24dp);
  2118. if (binding.callStates.errorImageView.getVisibility() != View.VISIBLE) {
  2119. binding.callStates.errorImageView.setVisibility(View.VISIBLE);
  2120. }
  2121. });
  2122. break;
  2123. case PUBLISHER_FAILED:
  2124. handler.post(() -> {
  2125. // No calling sound when the publisher failed
  2126. binding.callStates.callStateTextView.setText(R.string.nc_call_reconnecting);
  2127. binding.callModeTextView.setText(getDescriptionForCallType());
  2128. if (binding.callStates.callStateRelativeLayout.getVisibility() != View.VISIBLE) {
  2129. binding.callStates.callStateRelativeLayout.setVisibility(View.VISIBLE);
  2130. }
  2131. if (binding.gridview.getVisibility() != View.INVISIBLE) {
  2132. binding.gridview.setVisibility(View.INVISIBLE);
  2133. }
  2134. if (binding.callStates.callStateProgressBar.getVisibility() != View.VISIBLE) {
  2135. binding.callStates.callStateProgressBar.setVisibility(View.VISIBLE);
  2136. }
  2137. if (binding.callStates.errorImageView.getVisibility() != View.GONE) {
  2138. binding.callStates.errorImageView.setVisibility(View.GONE);
  2139. }
  2140. });
  2141. break;
  2142. case RECONNECTING:
  2143. handler.post(() -> {
  2144. playCallingSound();
  2145. binding.callStates.callStateTextView.setText(R.string.nc_call_reconnecting);
  2146. binding.callModeTextView.setText(getDescriptionForCallType());
  2147. if (binding.callStates.callStateRelativeLayout.getVisibility() != View.VISIBLE) {
  2148. binding.callStates.callStateRelativeLayout.setVisibility(View.VISIBLE);
  2149. }
  2150. if (binding.gridview.getVisibility() != View.INVISIBLE) {
  2151. binding.gridview.setVisibility(View.INVISIBLE);
  2152. }
  2153. if (binding.callStates.callStateProgressBar.getVisibility() != View.VISIBLE) {
  2154. binding.callStates.callStateProgressBar.setVisibility(View.VISIBLE);
  2155. }
  2156. if (binding.callStates.errorImageView.getVisibility() != View.GONE) {
  2157. binding.callStates.errorImageView.setVisibility(View.GONE);
  2158. }
  2159. });
  2160. break;
  2161. case JOINED:
  2162. handler.postDelayed(() -> setCallState(CallStatus.CALLING_TIMEOUT), 45000);
  2163. handler.post(() -> {
  2164. binding.callModeTextView.setText(getDescriptionForCallType());
  2165. if (isIncomingCallFromNotification) {
  2166. binding.callStates.callStateTextView.setText(R.string.nc_call_incoming);
  2167. } else {
  2168. binding.callStates.callStateTextView.setText(R.string.nc_call_ringing);
  2169. }
  2170. if (binding.callStates.callStateRelativeLayout.getVisibility() != View.VISIBLE) {
  2171. binding.callStates.callStateRelativeLayout.setVisibility(View.VISIBLE);
  2172. }
  2173. if (binding.callStates.callStateProgressBar.getVisibility() != View.VISIBLE) {
  2174. binding.callStates.callStateProgressBar.setVisibility(View.VISIBLE);
  2175. }
  2176. if (binding.gridview.getVisibility() != View.INVISIBLE) {
  2177. binding.gridview.setVisibility(View.INVISIBLE);
  2178. }
  2179. if (binding.callStates.errorImageView.getVisibility() != View.GONE) {
  2180. binding.callStates.errorImageView.setVisibility(View.GONE);
  2181. }
  2182. });
  2183. break;
  2184. case IN_CONVERSATION:
  2185. handler.post(() -> {
  2186. stopCallingSound();
  2187. binding.callModeTextView.setText(getDescriptionForCallType());
  2188. if (!isVoiceOnlyCall) {
  2189. binding.callInfosLinearLayout.setVisibility(View.GONE);
  2190. }
  2191. if (!isPushToTalkActive) {
  2192. animateCallControls(false, 5000);
  2193. }
  2194. if (binding.callStates.callStateRelativeLayout.getVisibility() != View.INVISIBLE) {
  2195. binding.callStates.callStateRelativeLayout.setVisibility(View.INVISIBLE);
  2196. }
  2197. if (binding.callStates.callStateProgressBar.getVisibility() != View.GONE) {
  2198. binding.callStates.callStateProgressBar.setVisibility(View.GONE);
  2199. }
  2200. if (binding.gridview.getVisibility() != View.VISIBLE) {
  2201. binding.gridview.setVisibility(View.VISIBLE);
  2202. }
  2203. if (binding.callStates.errorImageView.getVisibility() != View.GONE) {
  2204. binding.callStates.errorImageView.setVisibility(View.GONE);
  2205. }
  2206. });
  2207. break;
  2208. case OFFLINE:
  2209. handler.post(() -> {
  2210. stopCallingSound();
  2211. binding.callStates.callStateTextView.setText(R.string.nc_offline);
  2212. if (binding.callStates.callStateRelativeLayout.getVisibility() != View.VISIBLE) {
  2213. binding.callStates.callStateRelativeLayout.setVisibility(View.VISIBLE);
  2214. }
  2215. if (binding.gridview.getVisibility() != View.INVISIBLE) {
  2216. binding.gridview.setVisibility(View.INVISIBLE);
  2217. }
  2218. if (binding.callStates.callStateProgressBar.getVisibility() != View.GONE) {
  2219. binding.callStates.callStateProgressBar.setVisibility(View.GONE);
  2220. }
  2221. binding.callStates.errorImageView.setImageResource(R.drawable.ic_signal_wifi_off_white_24dp);
  2222. if (binding.callStates.errorImageView.getVisibility() != View.VISIBLE) {
  2223. binding.callStates.errorImageView.setVisibility(View.VISIBLE);
  2224. }
  2225. });
  2226. break;
  2227. case LEAVING:
  2228. handler.post(() -> {
  2229. if (!isDestroyed()) {
  2230. stopCallingSound();
  2231. binding.callModeTextView.setText(getDescriptionForCallType());
  2232. binding.callStates.callStateTextView.setText(R.string.nc_leaving_call);
  2233. binding.callStates.callStateRelativeLayout.setVisibility(View.VISIBLE);
  2234. binding.gridview.setVisibility(View.INVISIBLE);
  2235. binding.callStates.callStateProgressBar.setVisibility(View.VISIBLE);
  2236. binding.callStates.errorImageView.setVisibility(View.GONE);
  2237. }
  2238. });
  2239. break;
  2240. default:
  2241. }
  2242. }
  2243. }
  2244. private String getDescriptionForCallType() {
  2245. String appName = getResources().getString(R.string.nc_app_product_name);
  2246. if (isVoiceOnlyCall) {
  2247. return String.format(getResources().getString(R.string.nc_call_voice), appName);
  2248. } else {
  2249. return String.format(getResources().getString(R.string.nc_call_video), appName);
  2250. }
  2251. }
  2252. private void playCallingSound() {
  2253. stopCallingSound();
  2254. Uri ringtoneUri;
  2255. if (isIncomingCallFromNotification) {
  2256. ringtoneUri = NotificationUtils.INSTANCE.getCallRingtoneUri(getApplicationContext(), appPreferences);
  2257. } else {
  2258. ringtoneUri = Uri.parse("android.resource://" + getApplicationContext().getPackageName() + "/raw" +
  2259. "/tr110_1_kap8_3_freiton1");
  2260. }
  2261. if (ringtoneUri != null) {
  2262. mediaPlayer = new MediaPlayer();
  2263. try {
  2264. mediaPlayer.setDataSource(this, ringtoneUri);
  2265. mediaPlayer.setLooping(true);
  2266. AudioAttributes audioAttributes = new AudioAttributes.Builder().setContentType(
  2267. AudioAttributes.CONTENT_TYPE_SONIFICATION)
  2268. .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION)
  2269. .build();
  2270. mediaPlayer.setAudioAttributes(audioAttributes);
  2271. mediaPlayer.setOnPreparedListener(mp -> mediaPlayer.start());
  2272. mediaPlayer.prepareAsync();
  2273. } catch (IOException e) {
  2274. Log.e(TAG, "Failed to play sound");
  2275. }
  2276. }
  2277. }
  2278. private void stopCallingSound() {
  2279. if (mediaPlayer != null) {
  2280. try {
  2281. if (mediaPlayer.isPlaying()) {
  2282. mediaPlayer.stop();
  2283. }
  2284. } catch (IllegalStateException e) {
  2285. Log.e(TAG, "mediaPlayer was not initialized", e);
  2286. } finally {
  2287. if (mediaPlayer != null) {
  2288. mediaPlayer.release();
  2289. }
  2290. mediaPlayer = null;
  2291. }
  2292. }
  2293. }
  2294. /**
  2295. * Temporary implementation of SignalingMessageReceiver until signaling related code is extracted from
  2296. * CallActivity.
  2297. * <p>
  2298. * All listeners are called in the main thread.
  2299. */
  2300. private static class InternalSignalingMessageReceiver extends SignalingMessageReceiver {
  2301. public void process(List<Map<String, Object>> users) {
  2302. processUsersInRoom(users);
  2303. }
  2304. public void process(NCSignalingMessage message) {
  2305. processSignalingMessage(message);
  2306. }
  2307. }
  2308. private class OfferAnswerNickProvider {
  2309. private class WebRtcMessageListener implements SignalingMessageReceiver.WebRtcMessageListener {
  2310. @Override
  2311. public void onOffer(String sdp, String nick) {
  2312. onOfferOrAnswer(nick);
  2313. }
  2314. @Override
  2315. public void onAnswer(String sdp, String nick) {
  2316. onOfferOrAnswer(nick);
  2317. }
  2318. @Override
  2319. public void onCandidate(String sdpMid, int sdpMLineIndex, String sdp) {
  2320. }
  2321. @Override
  2322. public void onEndOfCandidates() {
  2323. }
  2324. }
  2325. private final WebRtcMessageListener videoWebRtcMessageListener = new WebRtcMessageListener();
  2326. private final WebRtcMessageListener screenWebRtcMessageListener = new WebRtcMessageListener();
  2327. private final String sessionId;
  2328. private String nick;
  2329. private OfferAnswerNickProvider(String sessionId) {
  2330. this.sessionId = sessionId;
  2331. }
  2332. private void onOfferOrAnswer(String nick) {
  2333. this.nick = nick;
  2334. if (callParticipants.get(sessionId) != null) {
  2335. callParticipants.get(sessionId).setNick(nick);
  2336. }
  2337. }
  2338. public WebRtcMessageListener getVideoWebRtcMessageListener() {
  2339. return videoWebRtcMessageListener;
  2340. }
  2341. public WebRtcMessageListener getScreenWebRtcMessageListener() {
  2342. return screenWebRtcMessageListener;
  2343. }
  2344. public String getNick() {
  2345. return nick;
  2346. }
  2347. }
  2348. private class CallActivityCallParticipantMessageListener implements SignalingMessageReceiver.CallParticipantMessageListener {
  2349. private final String sessionId;
  2350. public CallActivityCallParticipantMessageListener(String sessionId) {
  2351. this.sessionId = sessionId;
  2352. }
  2353. @Override
  2354. public void onRaiseHand(boolean state, long timestamp) {
  2355. if (state) {
  2356. CallParticipant participant = callParticipants.get(sessionId);
  2357. if (participant != null) {
  2358. String nick = participant.getCallParticipantModel().getNick();
  2359. runOnUiThread(() -> Toast.makeText(
  2360. context,
  2361. String.format(context.getResources().getString(R.string.nc_call_raised_hand), nick),
  2362. Toast.LENGTH_LONG).show());
  2363. }
  2364. }
  2365. }
  2366. @Override
  2367. public void onUnshareScreen() {
  2368. endPeerConnection(sessionId, "screen");
  2369. }
  2370. }
  2371. private class CallActivitySelfPeerConnectionObserver implements PeerConnectionWrapper.PeerConnectionObserver {
  2372. @Override
  2373. public void onStreamAdded(MediaStream mediaStream) {
  2374. }
  2375. @Override
  2376. public void onStreamRemoved(MediaStream mediaStream) {
  2377. }
  2378. @Override
  2379. public void onIceConnectionStateChanged(PeerConnection.IceConnectionState iceConnectionState) {
  2380. runOnUiThread(() -> {
  2381. updateSelfVideoViewIceConnectionState(iceConnectionState);
  2382. if (iceConnectionState == PeerConnection.IceConnectionState.FAILED) {
  2383. setCallState(CallStatus.PUBLISHER_FAILED);
  2384. webSocketClient.clearResumeId();
  2385. hangup(false);
  2386. }
  2387. });
  2388. }
  2389. }
  2390. private class ScreenParticipantDisplayItemManager implements CallParticipantModel.Observer {
  2391. private final CallParticipantModel callParticipantModel;
  2392. private ScreenParticipantDisplayItemManager(CallParticipantModel callParticipantModel) {
  2393. this.callParticipantModel = callParticipantModel;
  2394. }
  2395. @Override
  2396. public void onChange() {
  2397. String sessionId = callParticipantModel.getSessionId();
  2398. if (callParticipantModel.getScreenIceConnectionState() == null) {
  2399. removeParticipantDisplayItem(sessionId, "screen");
  2400. return;
  2401. }
  2402. boolean hasScreenParticipantDisplayItem = participantDisplayItems.get(sessionId + "-screen") != null;
  2403. if (!hasScreenParticipantDisplayItem) {
  2404. addParticipantDisplayItem(callParticipantModel, "screen");
  2405. }
  2406. }
  2407. }
  2408. private class InternalSignalingMessageSender implements SignalingMessageSender {
  2409. @Override
  2410. public void send(NCSignalingMessage ncSignalingMessage) {
  2411. addLocalParticipantNickIfNeeded(ncSignalingMessage);
  2412. String serializedNcSignalingMessage;
  2413. try {
  2414. serializedNcSignalingMessage = LoganSquare.serialize(ncSignalingMessage);
  2415. } catch (IOException e) {
  2416. Log.e(TAG, "Failed to serialize signaling message", e);
  2417. return;
  2418. }
  2419. // The message wrapper can not be defined in a JSON model to be directly serialized, as sent messages
  2420. // need to be serialized twice; first the signaling message, and then the wrapper as a whole. Received
  2421. // messages, on the other hand, just need to be deserialized once.
  2422. StringBuilder stringBuilder = new StringBuilder();
  2423. stringBuilder.append('{')
  2424. .append("\"fn\":\"")
  2425. .append(StringEscapeUtils.escapeJson(serializedNcSignalingMessage))
  2426. .append('\"')
  2427. .append(',')
  2428. .append("\"sessionId\":")
  2429. .append('\"').append(StringEscapeUtils.escapeJson(callSession)).append('\"')
  2430. .append(',')
  2431. .append("\"ev\":\"message\"")
  2432. .append('}');
  2433. List<String> strings = new ArrayList<>();
  2434. String stringToSend = stringBuilder.toString();
  2435. strings.add(stringToSend);
  2436. int apiVersion = ApiUtils.getSignalingApiVersion(conversationUser, new int[]{ApiUtils.APIv3, 2, 1});
  2437. ncApi.sendSignalingMessages(credentials, ApiUtils.getUrlForSignaling(apiVersion, baseUrl, roomToken),
  2438. strings.toString())
  2439. .retry(3)
  2440. .subscribeOn(Schedulers.io())
  2441. .subscribe(new Observer<SignalingOverall>() {
  2442. @Override
  2443. public void onSubscribe(@io.reactivex.annotations.NonNull Disposable d) {
  2444. }
  2445. @Override
  2446. public void onNext(@io.reactivex.annotations.NonNull SignalingOverall signalingOverall) {
  2447. // When sending messages to the internal signaling server the response has been empty since
  2448. // Talk v2.9.0, so it is not really needed to process it, but there is no harm either in
  2449. // doing that, as technically messages could be returned.
  2450. receivedSignalingMessages(signalingOverall.getOcs().getSignalings());
  2451. }
  2452. @Override
  2453. public void onError(@io.reactivex.annotations.NonNull Throwable e) {
  2454. Log.e(TAG, "", e);
  2455. }
  2456. @Override
  2457. public void onComplete() {
  2458. }
  2459. });
  2460. }
  2461. /**
  2462. * Adds the local participant nick to offers and answers.
  2463. * <p>
  2464. * For legacy reasons the offers and answers sent when the internal signaling server is used are expected to
  2465. * provide the nick of the local participant.
  2466. *
  2467. * @param ncSignalingMessage the message to add the nick to
  2468. */
  2469. private void addLocalParticipantNickIfNeeded(NCSignalingMessage ncSignalingMessage) {
  2470. String type = ncSignalingMessage.getType();
  2471. if (!"offer".equals(type) && !"answer".equals(type)) {
  2472. return;
  2473. }
  2474. NCMessagePayload payload = ncSignalingMessage.getPayload();
  2475. if (payload == null) {
  2476. // Broken message, this should not happen
  2477. return;
  2478. }
  2479. payload.setNick(conversationUser.getDisplayName());
  2480. }
  2481. }
  2482. private class MicrophoneButtonTouchListener implements View.OnTouchListener {
  2483. @SuppressLint("ClickableViewAccessibility")
  2484. @Override
  2485. public boolean onTouch(View v, MotionEvent event) {
  2486. v.onTouchEvent(event);
  2487. if (event.getAction() == MotionEvent.ACTION_UP && isPushToTalkActive) {
  2488. isPushToTalkActive = false;
  2489. binding.microphoneButton.setImageResource(R.drawable.ic_mic_off_white_24px);
  2490. pulseAnimation.stop();
  2491. toggleMedia(false, false);
  2492. animateCallControls(false, 5000);
  2493. }
  2494. return true;
  2495. }
  2496. }
  2497. @Subscribe(threadMode = ThreadMode.BACKGROUND)
  2498. public void onMessageEvent(NetworkEvent networkEvent) {
  2499. if (networkEvent.getNetworkConnectionEvent() == NetworkEvent.NetworkConnectionEvent.NETWORK_CONNECTED) {
  2500. if (handler != null) {
  2501. handler.removeCallbacksAndMessages(null);
  2502. }
  2503. } else if (networkEvent.getNetworkConnectionEvent() ==
  2504. NetworkEvent.NetworkConnectionEvent.NETWORK_DISCONNECTED) {
  2505. if (handler != null) {
  2506. handler.removeCallbacksAndMessages(null);
  2507. }
  2508. }
  2509. }
  2510. @RequiresApi(api = Build.VERSION_CODES.O)
  2511. @Override
  2512. public void onPictureInPictureModeChanged(boolean isInPictureInPictureMode, Configuration newConfig) {
  2513. super.onPictureInPictureModeChanged(isInPictureInPictureMode, newConfig);
  2514. Log.d(TAG, "onPictureInPictureModeChanged");
  2515. Log.d(TAG, "isInPictureInPictureMode= " + isInPictureInPictureMode);
  2516. isInPipMode = isInPictureInPictureMode;
  2517. if (isInPictureInPictureMode) {
  2518. mReceiver =
  2519. new BroadcastReceiver() {
  2520. @Override
  2521. public void onReceive(Context context, Intent intent) {
  2522. if (intent == null || !MICROPHONE_PIP_INTENT_NAME.equals(intent.getAction())) {
  2523. return;
  2524. }
  2525. final int action = intent.getIntExtra(MICROPHONE_PIP_INTENT_EXTRA_ACTION, 0);
  2526. switch (action) {
  2527. case MICROPHONE_PIP_REQUEST_MUTE:
  2528. case MICROPHONE_PIP_REQUEST_UNMUTE:
  2529. onMicrophoneClick();
  2530. break;
  2531. }
  2532. }
  2533. };
  2534. registerReceiver(mReceiver,
  2535. new IntentFilter(MICROPHONE_PIP_INTENT_NAME),
  2536. permissionUtil.getPrivateBroadcastPermission(),
  2537. null);
  2538. updateUiForPipMode();
  2539. } else {
  2540. unregisterReceiver(mReceiver);
  2541. mReceiver = null;
  2542. updateUiForNormalMode();
  2543. }
  2544. }
  2545. void updatePictureInPictureActions(
  2546. @DrawableRes int iconId,
  2547. String title,
  2548. int requestCode) {
  2549. if (isGreaterEqualOreo() && isPipModePossible()) {
  2550. final ArrayList<RemoteAction> actions = new ArrayList<>();
  2551. final Icon icon = Icon.createWithResource(this, iconId);
  2552. int intentFlag;
  2553. if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
  2554. intentFlag = FLAG_IMMUTABLE;
  2555. } else {
  2556. intentFlag = 0;
  2557. }
  2558. final PendingIntent intent =
  2559. PendingIntent.getBroadcast(
  2560. this,
  2561. requestCode,
  2562. new Intent(MICROPHONE_PIP_INTENT_NAME).putExtra(MICROPHONE_PIP_INTENT_EXTRA_ACTION, requestCode),
  2563. intentFlag);
  2564. actions.add(new RemoteAction(icon, title, title, intent));
  2565. mPictureInPictureParamsBuilder.setActions(actions);
  2566. setPictureInPictureParams(mPictureInPictureParamsBuilder.build());
  2567. }
  2568. }
  2569. public void updateUiForPipMode() {
  2570. Log.d(TAG, "updateUiForPipMode");
  2571. RelativeLayout.LayoutParams params = new RelativeLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT,
  2572. ViewGroup.LayoutParams.WRAP_CONTENT);
  2573. params.setMargins(0, 0, 0, 0);
  2574. binding.gridview.setLayoutParams(params);
  2575. binding.callControls.setVisibility(View.GONE);
  2576. binding.callInfosLinearLayout.setVisibility(View.GONE);
  2577. binding.selfVideoViewWrapper.setVisibility(View.GONE);
  2578. binding.callStates.callStateRelativeLayout.setVisibility(View.GONE);
  2579. if (participantDisplayItems.size() > 1) {
  2580. binding.pipCallConversationNameTextView.setText(conversationName);
  2581. binding.pipGroupCallOverlay.setVisibility(View.VISIBLE);
  2582. } else {
  2583. binding.pipGroupCallOverlay.setVisibility(View.GONE);
  2584. }
  2585. binding.selfVideoRenderer.release();
  2586. }
  2587. public void updateUiForNormalMode() {
  2588. Log.d(TAG, "updateUiForNormalMode");
  2589. if (isVoiceOnlyCall) {
  2590. binding.callControls.setVisibility(View.VISIBLE);
  2591. } else {
  2592. // animateCallControls needs this to be invisible for a check.
  2593. binding.callControls.setVisibility(View.INVISIBLE);
  2594. }
  2595. initViews();
  2596. binding.callInfosLinearLayout.setVisibility(View.VISIBLE);
  2597. binding.selfVideoViewWrapper.setVisibility(View.VISIBLE);
  2598. binding.pipGroupCallOverlay.setVisibility(View.GONE);
  2599. }
  2600. @Override
  2601. void suppressFitsSystemWindows() {
  2602. binding.controllerCallLayout.setFitsSystemWindows(false);
  2603. }
  2604. public void onConfigurationChanged(Configuration newConfig) {
  2605. super.onConfigurationChanged(newConfig);
  2606. eventBus.post(new ConfigurationChangeEvent());
  2607. }
  2608. public boolean isAllowedToStartOrStopRecording() {
  2609. return CapabilitiesUtilNew.isCallRecordingAvailable(conversationUser)
  2610. && isModerator;
  2611. }
  2612. public boolean isAllowedToRaiseHand() {
  2613. return CapabilitiesUtilNew.hasSpreedFeatureCapability(conversationUser, "raise-hand") ||
  2614. isBreakoutRoom;
  2615. }
  2616. private class SelfVideoTouchListener implements View.OnTouchListener {
  2617. @SuppressLint("ClickableViewAccessibility")
  2618. @Override
  2619. public boolean onTouch(View view, MotionEvent event) {
  2620. long duration = event.getEventTime() - event.getDownTime();
  2621. if (event.getActionMasked() == MotionEvent.ACTION_MOVE) {
  2622. float newY = event.getRawY() - binding.selfVideoViewWrapper.getHeight() / (float) 2;
  2623. float newX = event.getRawX() - binding.selfVideoViewWrapper.getWidth() / (float) 2;
  2624. binding.selfVideoViewWrapper.setY(newY);
  2625. binding.selfVideoViewWrapper.setX(newX);
  2626. } else if (event.getActionMasked() == MotionEvent.ACTION_UP && duration < 100) {
  2627. switchCamera();
  2628. }
  2629. return true;
  2630. }
  2631. }
  2632. }