pipeline: ipa: raspberrypi: Handle any externally allocated FrameBuffer

Handle the case where a FrameBuffer that has been externally allocated
(i.e. not through the v4l2 video device) is passed into a Request.

We must store the buffer pointer in the stream internal buffer list to
identify when used.

Signed-off-by: Naushir Patuck <naush@raspberrypi.com>
Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
Reviewed-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>
Signed-off-by: Niklas Söderlund <niklas.soderlund@ragnatech.se>

include/libcamera/ipa/raspberrypi.h

@@ -28,11 +28,12 @@ enum RPiOperations {
 	RPI_IPA_EVENT_QUEUE_REQUEST,
 };
 
-enum RPiIpaMask {
+enum RPiBufferMask {
-	ID		= 0x0ffff,
+	ID		= 0x00ffff,
-	STATS		= 0x10000,
+	STATS		= 0x010000,
-	EMBEDDED_DATA	= 0x20000,
+	EMBEDDED_DATA	= 0x020000,
-	BAYER_DATA	= 0x40000
+	BAYER_DATA	= 0x040000,
+	EXTERNAL_BUFFER	= 0x100000,
 };
 
 /* Size of the LS grid allocation. */

src/ipa/raspberrypi/raspberrypi.cpp

@@ -352,7 +352,7 @@ void IPARPi::processEvent(const IPAOperationData &event)
 
 		IPAOperationData op;
 		op.operation = RPI_IPA_ACTION_STATS_METADATA_COMPLETE;
-		op.data = { bufferId & RPiIpaMask::ID };
+		op.data = { bufferId & RPiBufferMask::ID };
 		op.controls = { libcameraMetadata_ };
 		queueFrameAction.emit(0, op);
 		break;
@@ -373,7 +373,7 @@ void IPARPi::processEvent(const IPAOperationData &event)
 		/* Ready to push the input buffer into the ISP. */
 		IPAOperationData op;
 		op.operation = RPI_IPA_ACTION_RUN_ISP;
-		op.data = { bayerbufferId & RPiIpaMask::ID };
+		op.data = { bayerbufferId & RPiBufferMask::ID };
 		queueFrameAction.emit(0, op);
 		break;
 	}
@@ -700,7 +700,7 @@ void IPARPi::returnEmbeddedBuffer(unsigned int bufferId)
 {
 	IPAOperationData op;
 	op.operation = RPI_IPA_ACTION_EMBEDDED_COMPLETE;
-	op.data = { bufferId & RPiIpaMask::ID };
+	op.data = { bufferId & RPiBufferMask::ID };
 	queueFrameAction.emit(0, op);
 }
 

src/libcamera/pipeline/raspberrypi/raspberrypi.cpp

@@ -151,6 +151,7 @@ public:
 
 	void clearIncompleteRequests();
 	void handleStreamBuffer(FrameBuffer *buffer, RPi::RPiStream *stream);
+	void handleExternalBuffer(FrameBuffer *buffer, RPi::RPiStream *stream);
 	void handleState();
 
 	CameraSensor *sensor_;
@@ -725,23 +726,32 @@ int PipelineHandlerRPi::queueRequestDevice(Camera *camera, Request *request)
 
 	/* Push all buffers supplied in the Request to the respective streams. */
 	for (auto stream : data->streams_) {
-		if (stream->isExternal()) {
+		if (!stream->isExternal())
-			FrameBuffer *buffer = request->findBuffer(stream);
+			continue;
+
+		FrameBuffer *buffer = request->findBuffer(stream);
+		if (buffer && stream->getBufferId(buffer) == -1) {
 			/*
-			 * If no buffer is provided by the request for this stream, we
+			 * This buffer is not recognised, so it must have been allocated
-			 * queue a nullptr to the stream to signify that it must use an
+			 * outside the v4l2 device. Store it in the stream buffer list
-			 * internally allocated buffer for this capture request. This
+			 * so we can track it.
-			 * buffer will not be given back to the application, but is used
-			 * to support the internal pipeline flow.
-			 *
-			 * The below queueBuffer() call will do nothing if there are not
-			 * enough internal buffers allocated, but this will be handled by
-			 * queuing the request for buffers in the RPiStream object.
 			 */
-			int ret = stream->queueBuffer(buffer);
+			stream->setExternalBuffer(buffer);
-			if (ret)
-				return ret;
 		}
+		/*
+		 * If no buffer is provided by the request for this stream, we
+		 * queue a nullptr to the stream to signify that it must use an
+		 * internally allocated buffer for this capture request. This
+		 * buffer will not be given back to the application, but is used
+		 * to support the internal pipeline flow.
+		 *
+		 * The below queueBuffer() call will do nothing if there are not
+		 * enough internal buffers allocated, but this will be handled by
+		 * queuing the request for buffers in the RPiStream object.
+		 */
+		int ret = stream->queueBuffer(buffer);
+		if (ret)
+			return ret;
 	}
 
 	/* Push the request to the back of the queue. */
@@ -915,8 +925,8 @@ int PipelineHandlerRPi::prepareBuffers(Camera *camera)
 	 * Pass the stats and embedded data buffers to the IPA. No other
 	 * buffers need to be passed.
 	 */
-	mapBuffers(camera, data->isp_[Isp::Stats].getBuffers(), RPiIpaMask::STATS);
+	mapBuffers(camera, data->isp_[Isp::Stats].getBuffers(), RPiBufferMask::STATS);
-	mapBuffers(camera, data->unicam_[Unicam::Embedded].getBuffers(), RPiIpaMask::EMBEDDED_DATA);
+	mapBuffers(camera, data->unicam_[Unicam::Embedded].getBuffers(), RPiBufferMask::EMBEDDED_DATA);
 
 	return 0;
 }
@@ -1219,7 +1229,7 @@ void RPiCameraData::ispOutputDequeue(FrameBuffer *buffer)
 	if (stream == &isp_[Isp::Stats]) {
 		IPAOperationData op;
 		op.operation = RPI_IPA_EVENT_SIGNAL_STAT_READY;
-		op.data = { RPiIpaMask::STATS | static_cast<unsigned int>(index) };
+		op.data = { RPiBufferMask::STATS | static_cast<unsigned int>(index) };
 		ipa_->processEvent(op);
 	} else {
 		/* Any other ISP output can be handed back to the application now. */
@@ -1297,6 +1307,11 @@ void RPiCameraData::handleStreamBuffer(FrameBuffer *buffer, RPi::RPiStream *stre
 		 */
 		Request *request = requestQueue_.empty() ? nullptr : requestQueue_.front();
 		if (!dropFrameCount_ && request && request->findBuffer(stream) == buffer) {
+			/*
+			 * Check if this is an externally provided buffer, and if
+			 * so, we must stop tracking it in the pipeline handler.
+			 */
+			handleExternalBuffer(buffer, stream);
 			/*
 			 * Tag the buffer as completed, returning it to the
 			 * application.
@@ -1315,6 +1330,17 @@ void RPiCameraData::handleStreamBuffer(FrameBuffer *buffer, RPi::RPiStream *stre
 	}
 }
 
+void RPiCameraData::handleExternalBuffer(FrameBuffer *buffer, RPi::RPiStream *stream)
+{
+	unsigned int id = stream->getBufferId(buffer);
+
+	if (!(id & RPiBufferMask::EXTERNAL_BUFFER))
+		return;
+
+	/* Stop the Stream object from tracking the buffer. */
+	stream->removeExternalBuffer(buffer);
+}
+
 void RPiCameraData::handleState()
 {
 	switch (state_) {
@@ -1447,8 +1473,8 @@ void RPiCameraData::tryRunPipeline()
 			<< " Embedded buffer id: " << embeddedId;
 
 	op.operation = RPI_IPA_EVENT_SIGNAL_ISP_PREPARE;
-	op.data = { RPiIpaMask::EMBEDDED_DATA | embeddedId,
+	op.data = { RPiBufferMask::EMBEDDED_DATA | embeddedId,
-		    RPiIpaMask::BAYER_DATA | bayerId };
+		    RPiBufferMask::BAYER_DATA | bayerId };
 	ipa_->processEvent(op);
 }
 

src/libcamera/pipeline/raspberrypi/rpi_stream.cpp

@@ -68,6 +68,20 @@ int RPiStream::getBufferId(FrameBuffer *buffer) const
 	return it->first;
 }
 
+void RPiStream::setExternalBuffer(FrameBuffer *buffer)
+{
+	bufferMap_.emplace(RPiBufferMask::EXTERNAL_BUFFER | id_.get(), buffer);
+}
+
+void RPiStream::removeExternalBuffer(FrameBuffer *buffer)
+{
+	int id = getBufferId(buffer);
+
+	/* Ensure we have this buffer in the stream, and it is marked external. */
+	ASSERT(id != -1 && (id & RPiBufferMask::EXTERNAL_BUFFER));
+	bufferMap_.erase(id);
+}
+
 int RPiStream::prepareBuffers(unsigned int count)
 {
 	int ret;

src/libcamera/pipeline/raspberrypi/rpi_stream.h

@@ -31,13 +31,13 @@ class RPiStream : public Stream
 {
 public:
 	RPiStream()
-		: id_(RPiIpaMask::ID)
+		: id_(RPiBufferMask::ID)
 	{
 	}
 
 	RPiStream(const char *name, MediaEntity *dev, bool importOnly = false)
 		: external_(false), importOnly_(importOnly), name_(name),
-		  dev_(std::make_unique<V4L2VideoDevice>(dev)), id_(RPiIpaMask::ID)
+		  dev_(std::make_unique<V4L2VideoDevice>(dev)), id_(RPiBufferMask::ID)
 	{
 	}
 
@@ -53,6 +53,9 @@ public:
 	const BufferMap &getBuffers() const;
 	int getBufferId(FrameBuffer *buffer) const;
 
+	void setExternalBuffer(FrameBuffer *buffer);
+	void removeExternalBuffer(FrameBuffer *buffer);
+
 	int prepareBuffers(unsigned int count);
 	int queueBuffer(FrameBuffer *buffer);
 	void returnBuffer(FrameBuffer *buffer);