diff --git a/src/com/android/services/telephony/HoldTracker.java b/src/com/android/services/telephony/HoldTracker.java
index 5032b41..2c6e56c 100644
--- a/src/com/android/services/telephony/HoldTracker.java
+++ b/src/com/android/services/telephony/HoldTracker.java
@@ -16,71 +16,55 @@
 
 package com.android.services.telephony;
 
-import android.telecom.PhoneAccountHandle;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
+ * Tracks and updates the hold capability of every call or conference across PhoneAccountHandles.
+ *
  * @hide
  */
 public class HoldTracker {
-    private final Map<PhoneAccountHandle, List<Holdable>> mHoldables;
+    private final Set<Holdable> mHoldables;
 
     public HoldTracker() {
-        mHoldables = new HashMap<>();
+        mHoldables = new HashSet<>();
     }
 
     /**
-     * Adds the holdable associated with the {@code phoneAccountHandle}, this method may update
-     * the hold state for all holdable associated with the {@code phoneAccountHandle}.
+     * Adds the holdable, and updates the hold capability for all holdables.
      */
-    public void addHoldable(PhoneAccountHandle phoneAccountHandle, Holdable holdable) {
-        if (!mHoldables.containsKey(phoneAccountHandle)) {
-            mHoldables.put(phoneAccountHandle, new ArrayList<>(1));
-        }
-        List<Holdable> holdables = mHoldables.get(phoneAccountHandle);
-        if (!holdables.contains(holdable)) {
-            holdables.add(holdable);
-            updateHoldCapability(phoneAccountHandle);
+    public void addHoldable(Holdable holdable) {
+        if (!mHoldables.contains(holdable)) {
+            mHoldables.add(holdable);
+            updateHoldCapability();
         }
     }
 
     /**
-     * Removes the holdable associated with the {@code phoneAccountHandle}, this method may update
-     * the hold state for all holdable associated with the {@code phoneAccountHandle}.
+     * Removes the holdable, and updates the hold capability for all holdable.
      */
-    public void removeHoldable(PhoneAccountHandle phoneAccountHandle, Holdable holdable) {
-        if (!mHoldables.containsKey(phoneAccountHandle)) {
-            return;
-        }
-
-        if (mHoldables.get(phoneAccountHandle).remove(holdable)) {
-            updateHoldCapability(phoneAccountHandle);
+    public void removeHoldable(Holdable holdable) {
+        if (mHoldables.remove(holdable)) {
+            updateHoldCapability();
         }
     }
 
     /**
-     * Updates the hold capability for all holdables associated with the {@code phoneAccountHandle}.
+     * Updates the hold capability for all tracked holdables.
      */
-    public void updateHoldCapability(PhoneAccountHandle phoneAccountHandle) {
-        if (!mHoldables.containsKey(phoneAccountHandle)) {
-            return;
-        }
-
-        List<Holdable> holdables = mHoldables.get(phoneAccountHandle);
+    public void updateHoldCapability() {
         int topHoldableCount = 0;
-        for (Holdable holdable : holdables) {
+        for (Holdable holdable : mHoldables) {
             if (!holdable.isChildHoldable()) {
                 ++topHoldableCount;
             }
         }
 
-        Log.d(this, "topHoldableCount = " + topHoldableCount);
+        Log.d(this, "updateHoldCapability(): topHoldableCount = "
+                + topHoldableCount);
         boolean isHoldable = topHoldableCount < 2;
-        for (Holdable holdable : holdables) {
+        for (Holdable holdable : mHoldables) {
             holdable.setHoldable(holdable.isChildHoldable() ? false : isHoldable);
         }
     }
diff --git a/src/com/android/services/telephony/TelephonyConnectionService.java b/src/com/android/services/telephony/TelephonyConnectionService.java
index 02c413e..bf7ce00 100644
--- a/src/com/android/services/telephony/TelephonyConnectionService.java
+++ b/src/com/android/services/telephony/TelephonyConnectionService.java
@@ -780,7 +780,7 @@
             new TelephonyConferenceBase.TelephonyConferenceListener() {
         @Override
         public void onConferenceMembershipChanged(Connection connection) {
-            mHoldTracker.updateHoldCapability(connection.getPhoneAccountHandle());
+            mHoldTracker.updateHoldCapability();
         }
     };
 
@@ -1904,29 +1904,28 @@
     @Override
     public void onConnectionAdded(Connection connection) {
         if (connection instanceof Holdable && !isExternalConnection(connection)) {
-            mHoldTracker.addHoldable(
-                    connection.getPhoneAccountHandle(), (Holdable) connection);
+            mHoldTracker.addHoldable((Holdable) connection);
         }
     }
 
     @Override
     public void onConnectionRemoved(Connection connection) {
         if (connection instanceof Holdable && !isExternalConnection(connection)) {
-            mHoldTracker.removeHoldable(connection.getPhoneAccountHandle(), (Holdable) connection);
+            mHoldTracker.removeHoldable((Holdable) connection);
         }
     }
 
     @Override
     public void onConferenceAdded(Conference conference) {
         if (conference instanceof Holdable) {
-            mHoldTracker.addHoldable(conference.getPhoneAccountHandle(), (Holdable) conference);
+            mHoldTracker.addHoldable((Holdable) conference);
         }
     }
 
     @Override
     public void onConferenceRemoved(Conference conference) {
         if (conference instanceof Holdable) {
-            mHoldTracker.removeHoldable(conference.getPhoneAccountHandle(), (Holdable) conference);
+            mHoldTracker.removeHoldable((Holdable) conference);
         }
     }
 
diff --git a/tests/src/com/android/services/telephony/HoldTrackerTest.java b/tests/src/com/android/services/telephony/HoldTrackerTest.java
index 772a1a2..83195db 100644
--- a/tests/src/com/android/services/telephony/HoldTrackerTest.java
+++ b/tests/src/com/android/services/telephony/HoldTrackerTest.java
@@ -19,9 +19,6 @@
 import static junit.framework.Assert.assertFalse;
 import static junit.framework.Assert.assertTrue;
 
-import android.content.ComponentName;
-import android.telecom.PhoneAccountHandle;
-
 import androidx.test.runner.AndroidJUnit4;
 
 import org.junit.Before;
@@ -32,72 +29,53 @@
 public class HoldTrackerTest {
 
     private HoldTracker mHoldTrackerUT;
-    private PhoneAccountHandle mPhoneAccountHandle1;
-    private PhoneAccountHandle mPhoneAccountHandle2;
 
     @Before
     public void setUp() throws Exception {
         mHoldTrackerUT = new HoldTracker();
-        mPhoneAccountHandle1 =
-                new PhoneAccountHandle(new ComponentName("pkg1", "cls1"), "0");
-        mPhoneAccountHandle2 =
-                new PhoneAccountHandle(new ComponentName("pkg2", "cls2"), "1");
     }
 
     @Test
     public void oneTopHoldableCanBeHeld() {
         FakeHoldable topHoldable = createHoldable(false);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle1, topHoldable);
+        mHoldTrackerUT.addHoldable(topHoldable);
 
         assertTrue(topHoldable.canBeHeld());
     }
 
     @Test
-    public void childHoldableCanNotBeHeld() {
+    public void childHoldableCannotBeHeld() {
         FakeHoldable topHoldable = createHoldable(false);
         FakeHoldable childHoldable = createHoldable(true);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle1, topHoldable);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle1, childHoldable);
+        mHoldTrackerUT.addHoldable(topHoldable);
+        mHoldTrackerUT.addHoldable(childHoldable);
 
         assertTrue(topHoldable.canBeHeld());
         assertFalse(childHoldable.canBeHeld());
     }
 
     @Test
-    public void twoTopHoldableWithTheSamePhoneAccountCanNotBeHeld() {
+    public void twoTopHoldablesCannotBeHeld() {
         FakeHoldable topHoldable1 = createHoldable(false);
         FakeHoldable topHoldable2 = createHoldable(false);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle1, topHoldable1);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle1, topHoldable2);
+        mHoldTrackerUT.addHoldable(topHoldable1);
+        mHoldTrackerUT.addHoldable(topHoldable2);
 
-        mHoldTrackerUT.updateHoldCapability(mPhoneAccountHandle1);
+        mHoldTrackerUT.updateHoldCapability();
         assertFalse(topHoldable1.canBeHeld());
         assertFalse(topHoldable2.canBeHeld());
     }
 
     @Test
-    public void holdableWithDifferentPhoneAccountDoesNotAffectEachOther() {
-        FakeHoldable topHoldable1 = createHoldable(false);
-        FakeHoldable topHoldable2 = createHoldable(false);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle1, topHoldable1);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle2, topHoldable2);
-
-        // Both phones account have only one top holdable, so the holdable of each phone account can
-        // be held.
-        assertTrue(topHoldable1.canBeHeld());
-        assertTrue(topHoldable2.canBeHeld());
-    }
-
-    @Test
     public void removeOneTopHoldableAndUpdateHoldCapabilityCorrectly() {
         FakeHoldable topHoldable1 = createHoldable(false);
         FakeHoldable topHoldable2 = createHoldable(false);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle1, topHoldable1);
-        mHoldTrackerUT.addHoldable(mPhoneAccountHandle1, topHoldable2);
+        mHoldTrackerUT.addHoldable(topHoldable1);
+        mHoldTrackerUT.addHoldable(topHoldable2);
         assertFalse(topHoldable1.canBeHeld());
         assertFalse(topHoldable2.canBeHeld());
 
-        mHoldTrackerUT.removeHoldable(mPhoneAccountHandle1, topHoldable1);
+        mHoldTrackerUT.removeHoldable(topHoldable1);
         assertTrue(topHoldable2.canBeHeld());
     }