libcamera: buffer: Move captured metadata to FrameMetadata

Move the metadata retrieved when dequeuing a V4L2 buffer into a
FrameMetadata object. This is done as a step to migrate to the
FrameBuffer interface as the functions added to Buffer around
FrameMetadata match the ones in FrameBuffer.

Signed-off-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>
Reviewed-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>

include/libcamera/buffer.h

@@ -99,12 +99,6 @@ private:
 class Buffer final
 {
 public:
-	enum Status {
-		BufferSuccess,
-		BufferError,
-		BufferCancelled,
-	};
-
 	Buffer(unsigned int index = -1, const Buffer *metadata = nullptr);
 	Buffer(const Buffer &) = delete;
 	Buffer &operator=(const Buffer &) = delete;
@@ -113,11 +107,8 @@ public:
 	const std::array<int, 3> &dmabufs() const { return dmabuf_; }
 	BufferMemory *mem() { return mem_; }
 
-	unsigned int bytesused() const { return bytesused_; }
+	const FrameMetadata &metadata() const { return metadata_; };
-	uint64_t timestamp() const { return timestamp_; }
-	unsigned int sequence() const { return sequence_; }
 
-	Status status() const { return status_; }
 	Request *request() const { return request_; }
 	Stream *stream() const { return stream_; }
 
@@ -133,11 +124,8 @@ private:
 	std::array<int, 3> dmabuf_;
 	BufferMemory *mem_;
 
-	unsigned int bytesused_;
+	FrameMetadata metadata_;
-	uint64_t timestamp_;
-	unsigned int sequence_;
 
-	Status status_;
 	Request *request_;
 	Stream *stream_;
 };

src/android/camera_device.cpp

@@ -803,11 +803,11 @@ void CameraDevice::requestComplete(Request *request)
 
 	if (status == CAMERA3_BUFFER_STATUS_OK) {
 		notifyShutter(descriptor->frameNumber,
-			      libcameraBuffer->timestamp());
+			      libcameraBuffer->metadata().timestamp);
 
 		captureResult.partial_result = 1;
 		resultMetadata = getResultMetadata(descriptor->frameNumber,
-						   libcameraBuffer->timestamp());
+						   libcameraBuffer->metadata().timestamp);
 		captureResult.result = resultMetadata->get();
 	}
 

src/cam/buffer_writer.cpp

@@ -33,7 +33,7 @@ int BufferWriter::write(Buffer *buffer, const std::string &streamName)
 	if (pos != std::string::npos) {
 		std::stringstream ss;
 		ss << streamName << "-" << std::setw(6)
-		   << std::setfill('0') << buffer->sequence();
+		   << std::setfill('0') << buffer->metadata().sequence;
 		filename.replace(pos, 1, ss.str());
 	}
 

src/cam/capture.cpp

@@ -154,9 +154,18 @@ void Capture::requestComplete(Request *request)
 		Buffer *buffer = it->second;
 		const std::string &name = streamName_[stream];
 
+		const FrameMetadata &metadata = buffer->metadata();
+
 		info << " " << name
-		     << " seq: " << std::setw(6) << std::setfill('0') << buffer->sequence()
+		     << " seq: " << std::setw(6) << std::setfill('0') << metadata.sequence
-		     << " bytesused: " << buffer->bytesused();
+		     << " bytesused: ";
+
+		unsigned int nplane = 0;
+		for (const FrameMetadata::Plane &plane : metadata.planes) {
+			info << plane.bytesused;
+			if (++nplane < metadata.planes.size())
+				info << "/";
+		}
 
 		if (writer_)
 			writer_->write(buffer, name);

src/libcamera/buffer.cpp

@@ -177,20 +177,6 @@ void BufferPool::destroyBuffers()
  * deleted automatically after the request complete handler returns.
  */
 
-/**
- * \enum Buffer::Status
- * Buffer completion status
- * \var Buffer::BufferSuccess
- * The buffer has completed with success and contains valid data. All its other
- * metadata (such as bytesused(), timestamp() or sequence() number) are valid.
- * \var Buffer::BufferError
- * The buffer has completed with an error and doesn't contain valid data. Its
- * other metadata are valid.
- * \var Buffer::BufferCancelled
- * The buffer has been cancelled due to capture stop. Its other metadata are
- * invalid and shall not be used.
- */
-
 /**
  * \brief Construct a buffer not associated with any stream
  *
@@ -199,19 +185,15 @@ void BufferPool::destroyBuffers()
  * for a stream with Stream::createBuffer().
  */
 Buffer::Buffer(unsigned int index, const Buffer *metadata)
-	: index_(index), dmabuf_({ -1, -1, -1 }),
+	: index_(index), dmabuf_({ -1, -1, -1 }), request_(nullptr),
-	  status_(Buffer::BufferSuccess), request_(nullptr),
 	  stream_(nullptr)
 {
-	if (metadata) {
+	if (metadata)
-		bytesused_ = metadata->bytesused_;
+		metadata_ = metadata->metadata();
-		sequence_ = metadata->sequence_;
+	else
-		timestamp_ = metadata->timestamp_;
+		metadata_ = {};
-	} else {
+
-		bytesused_ = 0;
+	metadata_.status = FrameMetadata::FrameSuccess;
-		sequence_ = 0;
-		timestamp_ = 0;
-	}
 }
 
 /**
@@ -242,39 +224,13 @@ Buffer::Buffer(unsigned int index, const Buffer *metadata)
  */
 
 /**
- * \fn Buffer::bytesused()
+ * \fn Buffer::metadata()
- * \brief Retrieve the number of bytes occupied by the data in the buffer
+ * \brief Retrieve the buffer metadata
- * \return Number of bytes occupied in the buffer
- */
-
-/**
- * \fn Buffer::timestamp()
- * \brief Retrieve the time when the buffer was processed
  *
- * The timestamp is expressed as a number of nanoseconds since the epoch.
+ * The buffer metadata is updated when the buffer contents are modified, for
+ * example when a frame has been captured to the buffer by the hardware.
  *
- * \return Timestamp when the buffer was processed
+ * \return Metadata for the buffer
- */
-
-/**
- * \fn Buffer::sequence()
- * \brief Retrieve the buffer sequence number
- *
- * The sequence number is a monotonically increasing number assigned to the
- * buffer processed by the stream. Gaps in the sequence numbers indicate
- * dropped frames.
- *
- * \return Sequence number of the buffer
- */
-
-/**
- * \fn Buffer::status()
- * \brief Retrieve the buffer status
- *
- * The buffer status reports whether the buffer has completed successfully
- * (BufferSuccess) or if an error occurred (BufferError).
- *
- * \return The buffer status
  */
 
 /**
@@ -310,10 +266,10 @@ Buffer::Buffer(unsigned int index, const Buffer *metadata)
  */
 void Buffer::cancel()
 {
-	bytesused_ = 0;
+	metadata_.status = FrameMetadata::FrameCancelled;
-	timestamp_ = 0;
+	metadata_.sequence = 0;
-	sequence_ = 0;
+	metadata_.timestamp = 0;
-	status_ = BufferCancelled;
+	metadata_.planes = {};
 }
 
 /**

src/libcamera/pipeline/ipu3/ipu3.cpp

@@ -928,7 +928,7 @@ int PipelineHandlerIPU3::registerCameras()
 void IPU3CameraData::imguInputBufferReady(Buffer *buffer)
 {
 	/* \todo Handle buffer failures when state is set to BufferError. */
-	if (buffer->status() == Buffer::BufferCancelled)
+	if (buffer->metadata().status == FrameMetadata::FrameCancelled)
 		return;
 
 	cio2_.output_->queueBuffer(buffer);
@@ -962,7 +962,7 @@ void IPU3CameraData::imguOutputBufferReady(Buffer *buffer)
 void IPU3CameraData::cio2BufferReady(Buffer *buffer)
 {
 	/* \todo Handle buffer failures when state is set to BufferError. */
-	if (buffer->status() == Buffer::BufferCancelled)
+	if (buffer->metadata().status == FrameMetadata::FrameCancelled)
 		return;
 
 	imgu_->input_->queueBuffer(buffer);

src/libcamera/pipeline/rkisp1/rkisp1.cpp

@@ -100,10 +100,10 @@ public:
 		ASSERT(frameOffset(SOE) == 0);
 
 		utils::time_point soe = std::chrono::time_point<utils::clock>()
-			+ std::chrono::nanoseconds(buffer->timestamp())
+			+ std::chrono::nanoseconds(buffer->metadata().timestamp)
 			+ timeOffset(SOE);
 
-		notifyStartOfExposure(buffer->sequence(), soe);
+		notifyStartOfExposure(buffer->metadata().sequence, soe);
 	}
 
 	void setDelay(unsigned int type, int frame, int msdelay)
@@ -994,8 +994,8 @@ void PipelineHandlerRkISP1::bufferReady(Buffer *buffer)
 
 	data->timeline_.bufferReady(buffer);
 
-	if (data->frame_ <= buffer->sequence())
+	if (data->frame_ <= buffer->metadata().sequence)
-		data->frame_ = buffer->sequence() + 1;
+		data->frame_ = buffer->metadata().sequence + 1;
 
 	completeBuffer(activeCamera_, request, buffer);
 	tryCompleteRequest(request);

src/libcamera/request.cpp

@@ -238,7 +238,7 @@ bool Request::completeBuffer(Buffer *buffer)
 
 	buffer->request_ = nullptr;
 
-	if (buffer->status() == Buffer::BufferCancelled)
+	if (buffer->metadata().status == FrameMetadata::FrameCancelled)
 		cancelled_ = true;
 
 	return !hasPendingBuffers();

src/libcamera/v4l2_videodevice.cpp

@@ -1015,10 +1015,13 @@ int V4L2VideoDevice::queueBuffer(Buffer *buffer)
 	}
 
 	if (V4L2_TYPE_IS_OUTPUT(buf.type)) {
-		buf.bytesused = buffer->bytesused_;
+		const FrameMetadata &metadata = buffer->metadata();
-		buf.sequence = buffer->sequence_;
+
-		buf.timestamp.tv_sec = buffer->timestamp_ / 1000000000;
+		if (!metadata.planes.empty())
-		buf.timestamp.tv_usec = (buffer->timestamp_ / 1000) % 1000000;
+			buf.bytesused = metadata.planes[0].bytesused;
+		buf.sequence = metadata.sequence;
+		buf.timestamp.tv_sec = metadata.timestamp / 1000000000;
+		buf.timestamp.tv_usec = (metadata.timestamp / 1000) % 1000000;
 	}
 
 	LOG(V4L2, Debug) << "Queueing buffer " << buf.index;
@@ -1125,12 +1128,14 @@ Buffer *V4L2VideoDevice::dequeueBuffer()
 		fdEvent_->setEnabled(false);
 
 	buffer->index_ = buf.index;
-	buffer->bytesused_ = buf.bytesused;
+
-	buffer->timestamp_ = buf.timestamp.tv_sec * 1000000000ULL
+	buffer->metadata_.status = buf.flags & V4L2_BUF_FLAG_ERROR
-			   + buf.timestamp.tv_usec * 1000ULL;
+				 ? FrameMetadata::FrameError
-	buffer->sequence_ = buf.sequence;
+				 : FrameMetadata::FrameSuccess;
-	buffer->status_ = buf.flags & V4L2_BUF_FLAG_ERROR
+	buffer->metadata_.sequence = buf.sequence;
-			? Buffer::BufferError : Buffer::BufferSuccess;
+	buffer->metadata_.timestamp = buf.timestamp.tv_sec * 1000000000ULL
+				    + buf.timestamp.tv_usec * 1000ULL;
+	buffer->metadata_.planes = { { buf.bytesused } };
 
 	return buffer;
 }

src/qcam/main_window.cpp

@@ -259,14 +259,15 @@ void MainWindow::requestComplete(Request *request)
 	framesCaptured_++;
 
 	Buffer *buffer = buffers.begin()->second;
+	const FrameMetadata &metadata = buffer->metadata();
 
-	double fps = buffer->timestamp() - lastBufferTime_;
+	double fps = metadata.timestamp - lastBufferTime_;
 	fps = lastBufferTime_ && fps ? 1000000000.0 / fps : 0.0;
-	lastBufferTime_ = buffer->timestamp();
+	lastBufferTime_ = metadata.timestamp;
 
-	std::cout << "seq: " << std::setw(6) << std::setfill('0') << buffer->sequence()
+	std::cout << "seq: " << std::setw(6) << std::setfill('0') << metadata.sequence
-		  << " bytesused: " << buffer->bytesused()
+		  << " bytesused: " << metadata.planes[0].bytesused
-		  << " timestamp: " << buffer->timestamp()
+		  << " timestamp: " << metadata.timestamp
 		  << " fps: " << std::fixed << std::setprecision(2) << fps
 		  << std::endl;
 
@@ -306,7 +307,7 @@ int MainWindow::display(Buffer *buffer)
 			    plane.fd.fd(), 0);
 
 	unsigned char *raw = static_cast<unsigned char *>(memory);
-	viewfinder_->display(raw, buffer->bytesused());
+	viewfinder_->display(raw, buffer->metadata().planes[0].bytesused);
 
 	munmap(memory, plane.length);
 

src/v4l2/v4l2_camera.cpp

@@ -17,9 +17,11 @@ using namespace libcamera;
 LOG_DECLARE_CATEGORY(V4L2Compat);
 
 V4L2FrameMetadata::V4L2FrameMetadata(Buffer *buffer)
-	: index_(buffer->index()), bytesused_(buffer->bytesused()),
+	: index_(buffer->index()),
-	  timestamp_(buffer->timestamp()), sequence_(buffer->sequence()),
+	  bytesused_(buffer->metadata().planes[0].bytesused),
-	  status_(buffer->status())
+	  timestamp_(buffer->metadata().timestamp),
+	  sequence_(buffer->metadata().sequence),
+	  status_(buffer->metadata().status)
 {
 }
 

src/v4l2/v4l2_camera.h

@@ -31,7 +31,7 @@ public:
 	uint64_t timestamp() const { return timestamp_; }
 	unsigned int sequence() const { return sequence_; }
 
-	Buffer::Status status() const { return status_; }
+	FrameMetadata::Status status() const { return status_; }
 
 private:
 	int index_;
@@ -40,7 +40,7 @@ private:
 	uint64_t timestamp_;
 	unsigned int sequence_;
 
-	Buffer::Status status_;
+	FrameMetadata::Status status_;
 };
 
 class V4L2Camera : public Object

src/v4l2/v4l2_camera_proxy.cpp

@@ -192,7 +192,7 @@ void V4L2CameraProxy::updateBuffers()
 		struct v4l2_buffer &buf = buffers_[fmd.index()];
 
 		switch (fmd.status()) {
-		case Buffer::Status::BufferSuccess:
+		case FrameMetadata::FrameSuccess:
 			buf.bytesused = fmd.bytesused();
 			buf.field = V4L2_FIELD_NONE;
 			buf.timestamp.tv_sec = fmd.timestamp() / 1000000000;
@@ -201,7 +201,7 @@ void V4L2CameraProxy::updateBuffers()
 
 			buf.flags |= V4L2_BUF_FLAG_DONE;
 			break;
-		case Buffer::Status::BufferError:
+		case FrameMetadata::FrameError:
 			buf.flags |= V4L2_BUF_FLAG_ERROR;
 			break;
 		default:

test/camera/buffer_import.cpp

@@ -275,7 +275,7 @@ public:
 protected:
 	void bufferComplete(Request *request, Buffer *buffer)
 	{
-		if (buffer->status() != Buffer::BufferSuccess)
+		if (buffer->metadata().status != FrameMetadata::FrameSuccess)
 			return;
 
 		unsigned int index = buffer->index();

test/camera/capture.cpp

@@ -28,7 +28,7 @@ protected:
 
 	void bufferComplete(Request *request, Buffer *buffer)
 	{
-		if (buffer->status() != Buffer::BufferSuccess)
+		if (buffer->metadata().status != FrameMetadata::FrameSuccess)
 			return;
 
 		completeBuffersCount_++;

test/v4l2_videodevice/buffer_sharing.cpp

@@ -92,9 +92,11 @@ protected:
 
 	void captureBufferReady(Buffer *buffer)
 	{
+		const FrameMetadata &metadata = buffer->metadata();
+
 		std::cout << "Received capture buffer" << std::endl;
 
-		if (buffer->status() != Buffer::BufferSuccess)
+		if (metadata.status != FrameMetadata::FrameSuccess)
 			return;
 
 		output_->queueBuffer(buffer);
@@ -103,9 +105,11 @@ protected:
 
 	void outputBufferReady(Buffer *buffer)
 	{
+		const FrameMetadata &metadata = buffer->metadata();
+
 		std::cout << "Received output buffer" << std::endl;
 
-		if (buffer->status() != Buffer::BufferSuccess)
+		if (metadata.status != FrameMetadata::FrameSuccess)
 			return;
 
 		capture_->queueBuffer(buffer);