Always mark reference referents in transaction mode.

Fix a to-space invariant check failure in EnqueueFinalizerReferences.

Reference processing can be a problem and useless during transaction
because it's not easy to roll back what reference processing does and
there's no daemon threads running (in the unstarted runtime). To avoid
issues, always mark reference referents.

Add a do_atomic_update parameter to MarkHeapReference.

Bug: 35417063
Test: test-art-host with CC/CMS/SS.
Change-Id: If32eba8fca19ef86e5d13f7925d179c8aecb9e27
diff --git a/runtime/gc/reference_processor.cc b/runtime/gc/reference_processor.cc
index 86b1522..65a550e 100644
--- a/runtime/gc/reference_processor.cc
+++ b/runtime/gc/reference_processor.cc
@@ -139,6 +139,14 @@
       CHECK_EQ(!self->GetWeakRefAccessEnabled(), concurrent);
     }
   }
+  if (kIsDebugBuild && collector->IsTransactionActive()) {
+    // In transaction mode, we shouldn't enqueue any Reference to the queues.
+    // See DelayReferenceReferent().
+    DCHECK(soft_reference_queue_.IsEmpty());
+    DCHECK(weak_reference_queue_.IsEmpty());
+    DCHECK(finalizer_reference_queue_.IsEmpty());
+    DCHECK(phantom_reference_queue_.IsEmpty());
+  }
   // Unless required to clear soft references with white references, preserve some white referents.
   if (!clear_soft_references) {
     TimingLogger::ScopedTiming split(concurrent ? "ForwardSoftReferences" :
@@ -206,6 +214,15 @@
   // do_atomic_update needs to be true because this happens outside of the reference processing
   // phase.
   if (!collector->IsNullOrMarkedHeapReference(referent, /*do_atomic_update*/true)) {
+    if (UNLIKELY(collector->IsTransactionActive())) {
+      // In transaction mode, keep the referent alive and avoid any reference processing to avoid the
+      // issue of rolling back reference processing.  do_atomic_update needs to be true because this
+      // happens outside of the reference processing phase.
+      if (!referent->IsNull()) {
+        collector->MarkHeapReference(referent, /*do_atomic_update*/ true);
+      }
+      return;
+    }
     Thread* self = Thread::Current();
     // TODO: Remove these locks, and use atomic stacks for storing references?
     // We need to check that the references haven't already been enqueued since we can end up