ipa: libipa: Introduce FrameContextQueue

Introduce a common implementation in libipa to represent the queue of
frame contexts.

Signed-off-by: Umang Jain <umang.jain@ideasonboard.com>
Signed-off-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
Signed-off-by: Jacopo Mondi <jacopo@jmondi.org>
Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>

src/ipa/libipa/fc_queue.cpp

@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2022, Google Inc.
+ *
+ * fc_queue.cpp - IPA Frame context queue
+ */
+
+#include "fc_queue.h"
+
+#include <libcamera/base/log.h>
+
+namespace libcamera {
+
+LOG_DEFINE_CATEGORY(FCQueue)
+
+namespace ipa {
+
+/**
+ * \file fc_queue.h
+ * \brief Queue of per-frame contexts
+ */
+
+/**
+ * \class FCQueue
+ * \brief A support class for managing FrameContext instances in IPA modules
+ * \tparam FrameContext The IPA module-specific FrameContext derived class type
+ *
+ * Along with the Module and Algorithm classes, the frame context queue is a
+ * core component of the libipa infrastructure. It stores per-frame contexts
+ * used by the Algorithm operations. By centralizing the lifetime management of
+ * the contexts and implementing safeguards against underflows and overflows, it
+ * simplifies IPA modules and improves their reliability.
+ *
+ * The queue references frame contexts by a monotonically increasing sequence
+ * number. The FCQueue design assumes that this number matches both the sequence
+ * number of the corresponding frame, as generated by the camera sensor, and the
+ * sequence number of the request. This allows IPA modules to obtain the frame
+ * context from any location where a request or a frame is available.
+ *
+ * A frame context normally begins its lifetime when the corresponding request
+ * is queued, way before the frame is captured by the camera sensor. IPA modules
+ * allocate the context from the queue at that point, calling alloc() using the
+ * request number. The queue initializes the context, and the IPA module then
+ * populates it with data from the request. The context can be later retrieved
+ * with a call to get(), typically when the IPA module is requested to provide
+ * sensor or ISP parameters or receives statistics for a frame. The frame number
+ * is used at that point to identify the context.
+ *
+ * If an application fails to queue requests to the camera fast enough, frames
+ * may be produced by the camera sensor and processed by the IPA module without
+ * a corresponding request having been queued to the IPA module. This creates an
+ * underrun condition, where the IPA module will try to get a frame context that
+ * hasn't been allocated. In this case, the get() function will allocate and
+ * initialize a context for the frame, and log a message. Algorithms will not
+ * apply the controls associated with the late request, but should otherwise
+ * behave correctly.
+ *
+ * \todo Mark the frame context with a per-frame control error flag in case of
+ * underrun, and research how algorithms should handle this.
+ *
+ * At its core, the queue uses a circular buffer to avoid dynamic memory
+ * allocation at runtime. The buffer is pre-allocated with a maximum number of
+ * entries when the FCQueue instance is constructed. Entries are initialized on
+ * first use by alloc() or, in underrun conditions, get(). The queue is not
+ * allowed to overflow, which must be ensured by pipeline handlers never
+ * queuing more in-flight requests to the IPA module than the queue size. If an
+ * overflow condition is detected, the queue will log a fatal error.
+ */
+
+/**
+ * \fn FCQueue::FCQueue(unsigned int size)
+ * \brief Construct a frame contexts queue of a specified size
+ * \param[in] size The number of contexts in the queue
+ */
+
+/**
+ * \fn FCQueue::clear()
+ * \brief Clear the contexts queue
+ *
+ * IPA modules must clear the frame context queue at the beginning of a new
+ * streaming session, in IPAModule::start().
+ *
+ * \todo Fix any issue this may cause with requests queued before the camera is
+ * started.
+ */
+
+/**
+ * \fn FCQueue::alloc(uint32_t frame)
+ * \brief Allocate and return a FrameContext for the \a frame
+ * \param[in] frame The frame context sequence number
+ *
+ * The first call to obtain a FrameContext from the FCQueue should be handled
+ * through this function. The FrameContext will be initialised, if not
+ * initialised already, and returned to the caller.
+ *
+ * If the FrameContext was already initialized for this \a frame, a warning will
+ * be reported and the previously initialized FrameContext is returned.
+ *
+ * Frame contexts are expected to be initialised when a Request is first passed
+ * to the IPA module in IPAModule::queueRequest().
+ *
+ * \return A reference to the FrameContext for sequence \a frame
+ */
+
+/**
+ * \fn FCQueue::get(uint32_t frame)
+ * \brief Obtain the FrameContext for the \a frame
+ * \param[in] frame The frame context sequence number
+ *
+ * If the FrameContext is not correctly initialised for the \a frame, it will be
+ * initialised.
+ *
+ * \return A reference to the FrameContext for sequence \a frame
+ */
+
+} /* namespace ipa */
+
+} /* namespace libcamera */

src/ipa/libipa/fc_queue.h

@@ -0,0 +1,108 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+/*
+ * Copyright (C) 2022, Google Inc.
+ *
+ * fc_queue.h - IPA Frame context queue
+ */
+
+#pragma once
+
+#include <vector>
+
+#include <libcamera/base/log.h>
+
+namespace libcamera {
+
+LOG_DECLARE_CATEGORY(FCQueue)
+
+namespace ipa {
+
+template<typename FrameContext>
+class FCQueue
+{
+public:
+	FCQueue(unsigned int size)
+		: contexts_(size)
+	{
+	}
+
+	void clear()
+	{
+		for (FrameContext &ctx : contexts_)
+			ctx.frame = 0;
+	}
+
+	FrameContext &alloc(const uint32_t frame)
+	{
+		FrameContext &frameContext = contexts_[frame % contexts_.size()];
+
+		/*
+		 * Do not re-initialise if a get() call has already fetched this
+		 * frame context to preseve the context.
+		 *
+		 * \todo If the the sequence number of the context to initialise
+		 * is smaller than the sequence number of the queue slot to use,
+		 * it means that we had a serious request underrun and more
+		 * frames than the queue size has been produced since the last
+		 * time the application has queued a request. Does this deserve
+		 * an error condition ?
+		 */
+		if (frame != 0 && frame <= frameContext.frame)
+			LOG(FCQueue, Warning)
+				<< "Frame " << frame << " already initialised";
+		else
+			init(frameContext, frame);
+
+		return frameContext;
+	}
+
+	FrameContext &get(uint32_t frame)
+	{
+		FrameContext &frameContext = contexts_[frame % contexts_.size()];
+
+		/*
+		 * If the IPA algorithms try to access a frame context slot which
+		 * has been already overwritten by a newer context, it means the
+		 * frame context queue has overflowed and the desired context
+		 * has been forever lost. The pipeline handler shall avoid
+		 * queueing more requests to the IPA than the frame context
+		 * queue size.
+		 */
+		if (frame < frameContext.frame)
+			LOG(FCQueue, Fatal) << "Frame context for " << frame
+					    << " has been overwritten by "
+					    << frameContext.frame;
+
+		if (frame == frameContext.frame)
+			return frameContext;
+
+		/*
+		 * The frame context has been retrieved before it was
+		 * initialised through the initialise() call. This indicates an
+		 * algorithm attempted to access a Frame context before it was
+		 * queued to the IPA. Controls applied for this request may be
+		 * left unhandled.
+		 *
+		 * \todo Set an error flag for per-frame control errors.
+		 */
+		LOG(FCQueue, Warning)
+			<< "Obtained an uninitialised FrameContext for " << frame;
+
+		init(frameContext, frame);
+
+		return frameContext;
+	}
+
+private:
+	void init(FrameContext &frameContext, const uint32_t frame)
+	{
+		frameContext = {};
+		frameContext.frame = frame;
+	}
+
+	std::vector<FrameContext> contexts_;
+};
+
+} /* namespace ipa */
+
+} /* namespace libcamera */

src/ipa/libipa/meson.build

@@ -3,6 +3,7 @@
 libipa_headers = files([
     'algorithm.h',
     'camera_sensor_helper.h',
+    'fc_queue.h',
     'histogram.h',
     'module.h',
 ])
@@ -10,6 +11,7 @@ libipa_headers = files([
 libipa_sources = files([
     'algorithm.cpp',
     'camera_sensor_helper.cpp',
+    'fc_queue.cpp',
     'histogram.cpp',
     'module.cpp',
 ])