diff --git a/libs/hwui/RenderNode.cpp b/libs/hwui/RenderNode.cpp
index 6ad52ad..3717aec 100644
--- a/libs/hwui/RenderNode.cpp
+++ b/libs/hwui/RenderNode.cpp
@@ -211,10 +211,10 @@
         info.renderer->pushLayerUpdate(mLayer);
     }
 
-    if (CC_UNLIKELY(info.canvasContext)) {
-        // If canvasContext is not null that means there are prefetched layers
-        // that need to be accounted for. That might be us, so tell CanvasContext
-        // that this layer is in the tree and should not be destroyed.
+    if (info.canvasContext) {
+        // There might be prefetched layers that need to be accounted for.
+        // That might be us, so tell CanvasContext that this layer is in the
+        // tree and should not be destroyed.
         info.canvasContext->markLayerInUse(this);
     }
 }
@@ -336,7 +336,8 @@
         TextureCache& cache = Caches::getInstance().textureCache;
         info.out.hasFunctors |= subtree->functors.size();
         for (size_t i = 0; info.prepareTextures && i < subtree->bitmapResources.size(); i++) {
-            info.prepareTextures = cache.prefetchAndMarkInUse(subtree->bitmapResources[i]);
+            info.prepareTextures = cache.prefetchAndMarkInUse(
+                    info.canvasContext, subtree->bitmapResources[i]);
         }
         for (size_t i = 0; i < subtree->children().size(); i++) {
             DrawRenderNodeOp* op = subtree->children()[i];
diff --git a/libs/hwui/Texture.h b/libs/hwui/Texture.h
index 7227ce0..4bcd96d 100644
--- a/libs/hwui/Texture.h
+++ b/libs/hwui/Texture.h
@@ -97,7 +97,7 @@
      * Whether or not the Texture is marked in use and thus not evictable for
      * the current frame. This is reset at the start of a new frame.
      */
-    bool isInUse = false;
+    void* isInUse = nullptr;
 
 private:
     /**
diff --git a/libs/hwui/TextureCache.cpp b/libs/hwui/TextureCache.cpp
index 386da72..c4467f1 100644
--- a/libs/hwui/TextureCache.cpp
+++ b/libs/hwui/TextureCache.cpp
@@ -119,10 +119,12 @@
     mAssetAtlas = assetAtlas;
 }
 
-void TextureCache::resetMarkInUse() {
+void TextureCache::resetMarkInUse(void* ownerToken) {
     LruCache<uint32_t, Texture*>::Iterator iter(mCache);
     while (iter.next()) {
-        iter.value()->isInUse = false;
+        if (iter.value()->isInUse == ownerToken) {
+            iter.value()->isInUse = nullptr;
+        }
     }
 }
 
@@ -186,10 +188,10 @@
     return texture;
 }
 
-bool TextureCache::prefetchAndMarkInUse(const SkBitmap* bitmap) {
+bool TextureCache::prefetchAndMarkInUse(void* ownerToken, const SkBitmap* bitmap) {
     Texture* texture = getCachedTexture(bitmap, AtlasUsageType::Use);
     if (texture) {
-        texture->isInUse = true;
+        texture->isInUse = ownerToken;
     }
     return texture;
 }
diff --git a/libs/hwui/TextureCache.h b/libs/hwui/TextureCache.h
index 83c6e91..7a7ee5a 100644
--- a/libs/hwui/TextureCache.h
+++ b/libs/hwui/TextureCache.h
@@ -66,14 +66,14 @@
     /**
      * Resets all Textures to not be marked as in use
      */
-    void resetMarkInUse();
+    void resetMarkInUse(void* ownerToken);
 
     /**
      * Attempts to precache the SkBitmap. Returns true if a Texture was successfully
      * acquired for the bitmap, false otherwise. If a Texture was acquired it is
      * marked as in use.
      */
-    bool prefetchAndMarkInUse(const SkBitmap* bitmap);
+    bool prefetchAndMarkInUse(void* ownerToken, const SkBitmap* bitmap);
 
     /**
      * Returns the texture associated with the specified bitmap from either within the cache, or
diff --git a/libs/hwui/TreeInfo.h b/libs/hwui/TreeInfo.h
index 0799c6c..ed853f7 100644
--- a/libs/hwui/TreeInfo.h
+++ b/libs/hwui/TreeInfo.h
@@ -95,7 +95,7 @@
     // layer updates or similar. May be NULL.
     OpenGLRenderer* renderer;
     ErrorHandler* errorHandler;
-    // TODO: Remove this? May be NULL
+    // May be NULL (TODO: can it really?)
     renderthread::CanvasContext* canvasContext;
 
     struct Out {
diff --git a/libs/hwui/renderthread/CanvasContext.cpp b/libs/hwui/renderthread/CanvasContext.cpp
index 09f93b8..e472e93 100644
--- a/libs/hwui/renderthread/CanvasContext.cpp
+++ b/libs/hwui/renderthread/CanvasContext.cpp
@@ -178,16 +178,13 @@
 
     info.damageAccumulator = &mDamageAccumulator;
     info.renderer = mCanvas;
-    if (mPrefetechedLayers.size() && info.mode == TreeInfo::MODE_FULL) {
-        info.canvasContext = this;
-    }
+    info.canvasContext = this;
+
     mAnimationContext->startFrame(info.mode);
     mRootRenderNode->prepareTree(info);
     mAnimationContext->runRemainingAnimations(info);
 
-    if (info.canvasContext) {
-        freePrefetechedLayers();
-    }
+    freePrefetechedLayers();
 
     if (CC_UNLIKELY(!mNativeWindow.get())) {
         mCurrentFrameInfo->addFlag(FrameInfoFlags::SkippedFrame);
@@ -369,7 +366,11 @@
     if (mEglManager.hasEglContext()) {
         freePrefetechedLayers();
         mRootRenderNode->destroyHardwareResources();
-        Caches::getInstance().flush(Caches::kFlushMode_Layers);
+        Caches& caches = Caches::getInstance();
+        // Make sure to release all the textures we were owning as there won't
+        // be another draw
+        caches.textureCache.resetMarkInUse(this);
+        caches.flush(Caches::kFlushMode_Layers);
     }
 }
 
diff --git a/libs/hwui/renderthread/DrawFrameTask.cpp b/libs/hwui/renderthread/DrawFrameTask.cpp
index 5cb868d..198906c 100644
--- a/libs/hwui/renderthread/DrawFrameTask.cpp
+++ b/libs/hwui/renderthread/DrawFrameTask.cpp
@@ -112,7 +112,7 @@
     int64_t vsync = mFrameInfo[static_cast<int>(FrameInfoIndex::Vsync)];
     mRenderThread->timeLord().vsyncReceived(vsync);
     mContext->makeCurrent();
-    Caches::getInstance().textureCache.resetMarkInUse();
+    Caches::getInstance().textureCache.resetMarkInUse(mContext);
 
     for (size_t i = 0; i < mLayers.size(); i++) {
         mContext->processLayerUpdate(mLayers[i].get());