diff --git a/include/libcamera/internal/v4l2_videodevice.h b/include/libcamera/internal/v4l2_videodevice.h
index d7a81e08..cfeae7bd 100644
--- a/include/libcamera/internal/v4l2_videodevice.h
+++ b/include/libcamera/internal/v4l2_videodevice.h
@@ -124,6 +124,7 @@ public:
 	V4L2BufferCache(const std::vector<std::unique_ptr<FrameBuffer>> &buffers);
 	~V4L2BufferCache();
 
+	bool isEmpty() const;
 	int get(const FrameBuffer &buffer);
 	void put(unsigned int index);
 
diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp
index 9b5807d3..009f6d55 100644
--- a/src/libcamera/v4l2_videodevice.cpp
+++ b/src/libcamera/v4l2_videodevice.cpp
@@ -201,6 +201,19 @@ V4L2BufferCache::~V4L2BufferCache()
 		LOG(V4L2, Debug) << "Cache misses: " << missCounter_;
 }
 
+/**
+ * \brief Check if all the entries in the cache are unused
+ */
+bool V4L2BufferCache::isEmpty() const
+{
+	for (auto const &entry : cache_) {
+		if (!entry.free_)
+			return false;
+	}
+
+	return true;
+}
+
 /**
  * \brief Find the best V4L2 buffer for a FrameBuffer
  * \param[in] buffer The FrameBuffer
@@ -1849,10 +1862,13 @@ int V4L2VideoDevice::streamOff()
 	for (auto it : queuedBuffers_) {
 		FrameBuffer *buffer = it.second;
 
+		cache_->put(it.first);
 		buffer->metadata_.status = FrameMetadata::FrameCancelled;
 		bufferReady.emit(buffer);
 	}
 
+	ASSERT(cache_->isEmpty());
+
 	queuedBuffers_.clear();
 	fdBufferNotifier_->setEnabled(false);
 	state_ = State::Stopped;