From 633063e099ccb0618359f9ecde9b05852c34259d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Tue, 4 Mar 2025 13:42:19 +0100 Subject: [PATCH 01/64] android: camera_device: Do not pass `nullptr` to `Request::addBuffer()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default argument already takes care of passing no fence to `addBuffer()`, so there is no reason to specify `nullptr` explicitly. Signed-off-by: Barnabás Pőcze Reviewed-by: Laurent Pinchart Reviewed-by: Kieran Bingham --- src/android/camera_device.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index a038131ae..80ff248c2 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -1079,7 +1079,7 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques buffer.internalBuffer = frameBuffer; descriptor->request_->addBuffer(sourceStream->stream(), - frameBuffer, nullptr); + frameBuffer); requestedStreams.insert(sourceStream); } From 081554db346a9ff2e610be1cff8d463ff3bc3b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Wed, 22 Jan 2025 12:27:12 +0100 Subject: [PATCH 02/64] libcamera: process: Disable copy/move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `Process` object has address identity because a pointer to it is stored inside the `ProcessManager`. However, copy/move special methods are still generated by the compiler. So disable them to avoid potential issues and confusion. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- include/libcamera/internal/process.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/libcamera/internal/process.h b/include/libcamera/internal/process.h index b1d07a5a5..6c34aef2f 100644 --- a/include/libcamera/internal/process.h +++ b/include/libcamera/internal/process.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -42,6 +43,8 @@ public: Signal finished; private: + LIBCAMERA_DISABLE_COPY_AND_MOVE(Process) + void closeAllFdsExcept(const std::vector &fds); int isolate(); void died(int wstatus); From 0a591eaf8c36ffd34dc0d69e6221c4fe88fd4a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Mon, 24 Mar 2025 12:39:11 +0100 Subject: [PATCH 03/64] libcamera: process: Misc. cleanup around `execv()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Firstly, get the number of arguments first, and use that to determine the size of the allocation instead of retrieving it twice. Secondly, use `const_cast` instead of a C-style cast when calling `execv()`. Third, use `size_t` to match the type of `args.size()`. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/libcamera/process.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libcamera/process.cpp b/src/libcamera/process.cpp index d836fb07a..f4e4b004c 100644 --- a/src/libcamera/process.cpp +++ b/src/libcamera/process.cpp @@ -279,14 +279,15 @@ int Process::start(const std::string &path, if (file && strcmp(file, "syslog")) unsetenv("LIBCAMERA_LOG_FILE"); - const char **argv = new const char *[args.size() + 2]; - unsigned int len = args.size(); + const size_t len = args.size(); + auto argv = std::make_unique(len + 2); + argv[0] = path.c_str(); - for (unsigned int i = 0; i < len; i++) + for (size_t i = 0; i < len; i++) argv[i + 1] = args[i].c_str(); argv[len + 1] = nullptr; - execv(path.c_str(), (char **)argv); + execv(path.c_str(), const_cast(argv.get())); _exit(EXIT_FAILURE); } From fae2b506d7ce8df6149e1eedf147cb35838fbb16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Tue, 25 Mar 2025 18:02:59 +0100 Subject: [PATCH 04/64] libcamera: process: Return error if already running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Returning 0 when a running process is already managed can be confusing since the parameters might be completely different, causing the caller to mistakenly assume that the program it specified has been started. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/libcamera/process.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/process.cpp b/src/libcamera/process.cpp index f4e4b004c..edbe8664a 100644 --- a/src/libcamera/process.cpp +++ b/src/libcamera/process.cpp @@ -241,7 +241,7 @@ int Process::start(const std::string &path, int ret; if (running_) - return 0; + return -EBUSY; int childPid = fork(); if (childPid == -1) { From 8d168f33486951c7121b923bd28ce5d4a9494099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Mon, 24 Mar 2025 16:05:44 +0100 Subject: [PATCH 05/64] libcamera: process: Ensure that file descriptors are nonnegative MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return `-EINVAL` from `Process::start()` if any of the file descriptors are negative as those most likely signal some kind of issue such as missed error checking. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/libcamera/process.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libcamera/process.cpp b/src/libcamera/process.cpp index edbe8664a..0eae68072 100644 --- a/src/libcamera/process.cpp +++ b/src/libcamera/process.cpp @@ -243,6 +243,11 @@ int Process::start(const std::string &path, if (running_) return -EBUSY; + for (int fd : fds) { + if (fd < 0) + return -EINVAL; + } + int childPid = fork(); if (childPid == -1) { ret = -errno; @@ -298,6 +303,8 @@ void Process::closeAllFdsExcept(const std::vector &fds) std::vector v(fds); sort(v.begin(), v.end()); + ASSERT(v.empty() || v.front() >= 0); + DIR *dir = opendir("/proc/self/fd"); if (!dir) return; From c50eb1f04a644cf82cbbc83b1446436b2220efac Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:19 +0100 Subject: [PATCH 06/64] libcamera: framebuffer: Add FrameMetadata::Status::FrameStartup Add a new status enum, FrameStartup, used to denote that even though the frame has been successfully captured, the IQ parameters set by the IPA will cause the frame to be unusable and applications are advised to not consume this frame. An example of this would be on a cold-start of the 3A algorithms, and there will be large oscillations to converge to a stable state quickly. Additional, update the definition of the FrameError state to cover the usage when the sensor is known to produce a number of invalid/error frames after stream-on. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- include/libcamera/framebuffer.h | 1 + src/libcamera/framebuffer.cpp | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/libcamera/framebuffer.h b/include/libcamera/framebuffer.h index ff8392430..e83825b46 100644 --- a/include/libcamera/framebuffer.h +++ b/include/libcamera/framebuffer.h @@ -26,6 +26,7 @@ struct FrameMetadata { FrameSuccess, FrameError, FrameCancelled, + FrameStartup, }; struct Plane { diff --git a/src/libcamera/framebuffer.cpp b/src/libcamera/framebuffer.cpp index 826848f75..219db50d6 100644 --- a/src/libcamera/framebuffer.cpp +++ b/src/libcamera/framebuffer.cpp @@ -43,12 +43,19 @@ LOG_DEFINE_CATEGORY(Buffer) * The frame has been captured with success and contains valid data. All fields * of the FrameMetadata structure are valid. * \var FrameMetadata::FrameError - * An error occurred during capture of the frame. The frame data may be partly - * or fully invalid. The sequence and timestamp fields of the FrameMetadata - * structure is valid, the other fields may be invalid. + * The frame data is partly or fully corrupted, missing or otherwise invalid. + * This can for instance indicate a hardware transmission error, or invalid data + * produced by the sensor during its startup phase. The sequence and timestamp + * fields of the FrameMetadata structure is valid, all the other fields may be + * invalid. * \var FrameMetadata::FrameCancelled * Capture stopped before the frame completed. The frame data is not valid. All * fields of the FrameMetadata structure but the status field are invalid. + * \var FrameMetadata::FrameStartup + * The frame has been successfully captured. However, the IPA is in a + * cold-start or reset phase and will result in image quality parameters + * producing unusable images. Applications are recommended to not consume these + * frames. All other fields of the FrameMetadata structure are valid. */ /** From b114c155a71660964c1947daacc5100fa047c9f3 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:20 +0100 Subject: [PATCH 07/64] ipa: rpi: Replace dropFrameCount in the IPA -> PH interface Replace the dropFrameCount parameter returned from ipa::start() to the pipeline handler by startupFrameCount and invalidFrameCount. The former counts the number of frames required for AWB/AGC to converge, and the latter counts the number of invalid frames produced by the sensor when starting up. In the pipeline handler, use the sum of these 2 values to replicate the existing dropFrameCount behaviour. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- include/libcamera/ipa/raspberrypi.mojom | 3 ++- src/ipa/rpi/common/ipa_base.cpp | 14 ++++++++------ .../pipeline/rpi/common/pipeline_base.cpp | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/include/libcamera/ipa/raspberrypi.mojom b/include/libcamera/ipa/raspberrypi.mojom index e30c70bde..12b083e9d 100644 --- a/include/libcamera/ipa/raspberrypi.mojom +++ b/include/libcamera/ipa/raspberrypi.mojom @@ -52,7 +52,8 @@ struct ConfigResult { struct StartResult { libcamera.ControlList controls; - int32 dropFrameCount; + int32 startupFrameCount; + int32 invalidFrameCount; }; struct PrepareParams { diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index e0f8b7e78..eee81e064 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -324,6 +324,7 @@ void IpaBase::start(const ControlList &controls, StartResult *result) * "mistrusted", which depends on whether this is a startup from cold, * or merely a mode switch in a running system. */ + unsigned int agcConvergenceFrames = 0, awbConvergenceFrames = 0; frameCount_ = 0; if (firstStart_) { dropFrameCount_ = helper_->hideFramesStartup(); @@ -336,7 +337,6 @@ void IpaBase::start(const ControlList &controls, StartResult *result) * (mistrustCount_) that they won't see. But if zero (i.e. * no convergence necessary), no frames need to be dropped. */ - unsigned int agcConvergenceFrames = 0; RPiController::AgcAlgorithm *agc = dynamic_cast( controller_.getAlgorithm("agc")); if (agc) { @@ -345,7 +345,6 @@ void IpaBase::start(const ControlList &controls, StartResult *result) agcConvergenceFrames += mistrustCount_; } - unsigned int awbConvergenceFrames = 0; RPiController::AwbAlgorithm *awb = dynamic_cast( controller_.getAlgorithm("awb")); if (awb) { @@ -353,15 +352,18 @@ void IpaBase::start(const ControlList &controls, StartResult *result) if (awbConvergenceFrames) awbConvergenceFrames += mistrustCount_; } - - dropFrameCount_ = std::max({ dropFrameCount_, agcConvergenceFrames, awbConvergenceFrames }); - LOG(IPARPI, Debug) << "Drop " << dropFrameCount_ << " frames on startup"; } else { dropFrameCount_ = helper_->hideFramesModeSwitch(); mistrustCount_ = helper_->mistrustFramesModeSwitch(); } - result->dropFrameCount = dropFrameCount_; + result->startupFrameCount = std::max({ agcConvergenceFrames, awbConvergenceFrames }); + result->invalidFrameCount = dropFrameCount_; + + dropFrameCount_ = std::max({ dropFrameCount_, agcConvergenceFrames, awbConvergenceFrames }); + + LOG(IPARPI, Debug) << "Startup frames: " << result->startupFrameCount + << " Invalid frames: " << result->invalidFrameCount; firstStart_ = false; lastRunTimestamp_ = 0; diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 1f13e5230..595648595 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -660,8 +660,8 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) data->setSensorControls(result.controls); /* Configure the number of dropped frames required on startup. */ - data->dropFrameCount_ = data->config_.disableStartupFrameDrops - ? 0 : result.dropFrameCount; + data->dropFrameCount_ = data->config_.disableStartupFrameDrops ? + 0 : result.startupFrameCount + result.invalidFrameCount; for (auto const stream : data->streams_) stream->resetBuffers(); From 6cf9c4d34fbda570f61c09fa44e6bc430600b84c Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:21 +0100 Subject: [PATCH 08/64] pipeline: ipa: rpi: Split RPiCameraData::dropFrameCount_ Split the pipeline handler drop frame tracking into startup frames and invalid frames, as reported by the IPA. Remove the drop buffer handling logic in the pipeline handler. Now all image buffers are returned out with the appropriate FrameStatus set for startup or invalid frames. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- .../pipeline/rpi/common/pipeline_base.cpp | 98 ++++++++----------- .../pipeline/rpi/common/pipeline_base.h | 5 +- 2 files changed, 42 insertions(+), 61 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 595648595..3f0b7abdc 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -659,9 +659,9 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) if (!result.controls.empty()) data->setSensorControls(result.controls); - /* Configure the number of dropped frames required on startup. */ - data->dropFrameCount_ = data->config_.disableStartupFrameDrops ? - 0 : result.startupFrameCount + result.invalidFrameCount; + /* Configure the number of startup and invalid frames reported by the IPA. */ + data->startupFrameCount_ = result.startupFrameCount; + data->invalidFrameCount_ = result.invalidFrameCount; for (auto const stream : data->streams_) stream->resetBuffers(); @@ -678,7 +678,6 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) data->buffersAllocated_ = true; } - /* We need to set the dropFrameCount_ before queueing buffers. */ ret = queueAllBuffers(camera); if (ret) { LOG(RPI, Error) << "Failed to queue buffers"; @@ -894,28 +893,12 @@ int PipelineHandlerBase::queueAllBuffers(Camera *camera) int ret; for (auto const stream : data->streams_) { - if (!(stream->getFlags() & StreamFlag::External)) { - ret = stream->queueAllBuffers(); - if (ret < 0) - return ret; - } else { - /* - * For external streams, we must queue up a set of internal - * buffers to handle the number of drop frames requested by - * the IPA. This is done by passing nullptr in queueBuffer(). - * - * The below queueBuffer() call will do nothing if there - * are not enough internal buffers allocated, but this will - * be handled by queuing the request for buffers in the - * RPiStream object. - */ - unsigned int i; - for (i = 0; i < data->dropFrameCount_; i++) { - ret = stream->queueBuffer(nullptr); - if (ret) - return ret; - } - } + if (stream->getFlags() & StreamFlag::External) + continue; + + ret = stream->queueAllBuffers(); + if (ret < 0) + return ret; } return 0; @@ -1412,7 +1395,15 @@ void CameraData::handleStreamBuffer(FrameBuffer *buffer, RPi::Stream *stream) * buffer back to the stream. */ Request *request = requestQueue_.empty() ? nullptr : requestQueue_.front(); - if (!dropFrameCount_ && request && request->findBuffer(stream) == buffer) { + if (request && request->findBuffer(stream) == buffer) { + FrameMetadata &md = buffer->_d()->metadata(); + + /* Mark the non-converged and invalid frames in the metadata. */ + if (invalidFrameCount_) + md.status = FrameMetadata::Status::FrameError; + else if (startupFrameCount_) + md.status = FrameMetadata::Status::FrameStartup; + /* * Tag the buffer as completed, returning it to the * application. @@ -1458,42 +1449,31 @@ void CameraData::handleState() void CameraData::checkRequestCompleted() { - bool requestCompleted = false; - /* - * If we are dropping this frame, do not touch the request, simply - * change the state to IDLE when ready. - */ - if (!dropFrameCount_) { - Request *request = requestQueue_.front(); - if (request->hasPendingBuffers()) - return; + Request *request = requestQueue_.front(); + if (request->hasPendingBuffers()) + return; - /* Must wait for metadata to be filled in before completing. */ - if (state_ != State::IpaComplete) - return; + /* Must wait for metadata to be filled in before completing. */ + if (state_ != State::IpaComplete) + return; - LOG(RPI, Debug) << "Completing request sequence: " - << request->sequence(); + LOG(RPI, Debug) << "Completing request sequence: " + << request->sequence(); - pipe()->completeRequest(request); - requestQueue_.pop(); - requestCompleted = true; - } + pipe()->completeRequest(request); + requestQueue_.pop(); - /* - * Make sure we have three outputs completed in the case of a dropped - * frame. - */ - if (state_ == State::IpaComplete && - ((ispOutputCount_ == ispOutputTotal_ && dropFrameCount_) || - requestCompleted)) { - LOG(RPI, Debug) << "Going into Idle state"; - state_ = State::Idle; - if (dropFrameCount_) { - dropFrameCount_--; - LOG(RPI, Debug) << "Dropping frame at the request of the IPA (" - << dropFrameCount_ << " left)"; - } + LOG(RPI, Debug) << "Going into Idle state"; + state_ = State::Idle; + + if (invalidFrameCount_) { + invalidFrameCount_--; + LOG(RPI, Debug) << "Decrementing invalid frames to " + << invalidFrameCount_; + } else if (startupFrameCount_) { + startupFrameCount_--; + LOG(RPI, Debug) << "Decrementing startup frames to " + << startupFrameCount_; } } diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index aae0c2f35..6023f9f9d 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -48,7 +48,7 @@ class CameraData : public Camera::Private public: CameraData(PipelineHandler *pipe) : Camera::Private(pipe), state_(State::Stopped), - dropFrameCount_(0), buffersAllocated_(false), + startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false), ispOutputCount_(0), ispOutputTotal_(0) { } @@ -151,7 +151,8 @@ public: /* Mapping of CropParams keyed by the output stream order in CameraConfiguration */ std::map cropParams_; - unsigned int dropFrameCount_; + unsigned int startupFrameCount_; + unsigned int invalidFrameCount_; /* * If set, this stores the value that represets a gain of one for From 98d144fef39877d79df8a948ced0b2ce4377a6be Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:22 +0100 Subject: [PATCH 09/64] pipeline: rpi: Remove disable_startup_frame_drops config option With the previous change to not drop frames in the pipeline handler, the "disable_startup_frame_drops" pipeline config option is not used. Remove it, and throw a warning if the option is present in the YAML config file. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/libcamera/pipeline/rpi/common/pipeline_base.cpp | 7 ++++--- src/libcamera/pipeline/rpi/common/pipeline_base.h | 5 ----- src/libcamera/pipeline/rpi/pisp/data/example.yaml | 5 ----- src/libcamera/pipeline/rpi/vc4/data/example.yaml | 5 ----- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 3f0b7abdc..d8c7ca935 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -1078,7 +1078,6 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front int CameraData::loadPipelineConfiguration() { config_ = { - .disableStartupFrameDrops = false, .cameraTimeoutValue = 0, }; @@ -1115,8 +1114,10 @@ int CameraData::loadPipelineConfiguration() const YamlObject &phConfig = (*root)["pipeline_handler"]; - config_.disableStartupFrameDrops = - phConfig["disable_startup_frame_drops"].get(config_.disableStartupFrameDrops); + if (phConfig.contains("disable_startup_frame_drops")) + LOG(RPI, Warning) + << "The disable_startup_frame_drops key is now deprecated, " + << "startup frames are now identified by the FrameMetadata::Status::FrameStartup flag"; config_.cameraTimeoutValue = phConfig["camera_timeout_value_ms"].get(config_.cameraTimeoutValue); diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 6023f9f9d..e27c4f860 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -164,11 +164,6 @@ public: bool buffersAllocated_; struct Config { - /* - * Override any request from the IPA to drop a number of startup - * frames. - */ - bool disableStartupFrameDrops; /* * Override the camera timeout value calculated by the IPA based * on frame durations. diff --git a/src/libcamera/pipeline/rpi/pisp/data/example.yaml b/src/libcamera/pipeline/rpi/pisp/data/example.yaml index d67e654a8..baf03be79 100644 --- a/src/libcamera/pipeline/rpi/pisp/data/example.yaml +++ b/src/libcamera/pipeline/rpi/pisp/data/example.yaml @@ -16,11 +16,6 @@ # # "num_cfe_config_queue": 2, - # Override any request from the IPA to drop a number of startup - # frames. - # - # "disable_startup_frame_drops": false, - # Custom timeout value (in ms) for camera to use. This overrides # the value computed by the pipeline handler based on frame # durations. diff --git a/src/libcamera/pipeline/rpi/vc4/data/example.yaml b/src/libcamera/pipeline/rpi/vc4/data/example.yaml index b8e01adea..27e543488 100644 --- a/src/libcamera/pipeline/rpi/vc4/data/example.yaml +++ b/src/libcamera/pipeline/rpi/vc4/data/example.yaml @@ -29,11 +29,6 @@ # # "min_total_unicam_buffers": 4, - # Override any request from the IPA to drop a number of startup - # frames. - # - # "disable_startup_frame_drops": false, - # Custom timeout value (in ms) for camera to use. This overrides # the value computed by the pipeline handler based on frame # durations. From a402f9ebc1225b684300fcc664fd37e0ddf95f30 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:23 +0100 Subject: [PATCH 10/64] pipeline: rpi: Remove ispOutputCount_ and ispOutputTotal_ With the drop frame logic removed from the pipeline handler, these member variables and not used, so remove them. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/libcamera/pipeline/rpi/common/pipeline_base.h | 6 +----- src/libcamera/pipeline/rpi/pisp/pisp.cpp | 12 ------------ src/libcamera/pipeline/rpi/vc4/vc4.cpp | 15 --------------- 3 files changed, 1 insertion(+), 32 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index e27c4f860..898f31577 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -48,8 +48,7 @@ class CameraData : public Camera::Private public: CameraData(PipelineHandler *pipe) : Camera::Private(pipe), state_(State::Stopped), - startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false), - ispOutputCount_(0), ispOutputTotal_(0) + startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false) { } @@ -179,9 +178,6 @@ protected: virtual void tryRunPipeline() = 0; - unsigned int ispOutputCount_; - unsigned int ispOutputTotal_; - private: void checkRequestCompleted(); }; diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index 91e7f4c94..ccf135c3d 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -1834,12 +1834,6 @@ void PiSPCameraData::beOutputDequeue(FrameBuffer *buffer) dmabufSyncEnd(buffer->planes()[0].fd); handleStreamBuffer(buffer, stream); - - /* - * Increment the number of ISP outputs generated. - * This is needed to track dropped frames. - */ - ispOutputCount_++; handleState(); } @@ -1885,7 +1879,6 @@ void PiSPCameraData::prepareIspComplete(const ipa::RPi::BufferIds &buffers, bool * If there is no need to run the Backend, just signal that the * input buffer is completed and all Backend outputs are ready. */ - ispOutputCount_ = ispOutputTotal_; buffer = cfe_[Cfe::Output0].getBuffers().at(bayerId).buffer; handleStreamBuffer(buffer, &cfe_[Cfe::Output0]); } else @@ -1994,7 +1987,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) global.bayer_enables |= PISP_BE_BAYER_ENABLE_INPUT; global.bayer_order = toPiSPBayerOrder(cfeFormat.fourcc); - ispOutputTotal_ = 1; /* Config buffer */ if (PISP_IMAGE_FORMAT_COMPRESSED(inputFormat.format)) { pisp_decompress_config decompress; decompress.offset = DefaultCompressionOffset; @@ -2025,7 +2017,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) setupOutputClipping(ispFormat0, outputFormat0); be_->SetOutputFormat(0, outputFormat0); - ispOutputTotal_++; } if (global.rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT1) { @@ -2049,7 +2040,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) setupOutputClipping(ispFormat1, outputFormat1); be_->SetOutputFormat(1, outputFormat1); - ispOutputTotal_++; } /* Setup the TDN I/O blocks in case TDN gets turned on later. */ @@ -2256,8 +2246,6 @@ void PiSPCameraData::prepareCfe() void PiSPCameraData::prepareBe(uint32_t bufferId, bool stitchSwapBuffers) { - ispOutputCount_ = 0; - FrameBuffer *buffer = cfe_[Cfe::Output0].getBuffers().at(bufferId).buffer; LOG(RPI, Debug) << "Input re-queue to ISP, buffer id " << bufferId diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index fe910bdf2..ac6dab814 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -597,8 +597,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi stream->setFlags(StreamFlag::External); } - ispOutputTotal_ = outStreams.size(); - /* * If ISP::Output0 stream has not been configured by the application, * we must allow the hardware to generate an output so that the data @@ -625,8 +623,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi return -EINVAL; } - ispOutputTotal_++; - LOG(RPI, Debug) << "Defaulting ISP Output0 format to " << format; } @@ -662,8 +658,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi << ret; return -EINVAL; } - - ispOutputTotal_++; } /* ISP statistics output format. */ @@ -676,8 +670,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi return ret; } - ispOutputTotal_++; - /* * Configure the Unicam embedded data output format only if the sensor * supports it. @@ -843,12 +835,6 @@ void Vc4CameraData::ispOutputDequeue(FrameBuffer *buffer) handleStreamBuffer(buffer, stream); } - /* - * Increment the number of ISP outputs generated. - * This is needed to track dropped frames. - */ - ispOutputCount_++; - handleState(); } @@ -880,7 +866,6 @@ void Vc4CameraData::prepareIspComplete(const ipa::RPi::BufferIds &buffers, << ", timestamp: " << buffer->metadata().timestamp; isp_[Isp::Input].queueBuffer(buffer); - ispOutputCount_ = 0; if (sensorMetadata_ && embeddedId) { buffer = unicam_[Unicam::Embedded].getBuffers().at(embeddedId & RPi::MaskID).buffer; From 8d2cd0b5b813bf5d26918df2218f2c0493992de6 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:24 +0100 Subject: [PATCH 11/64] ipa: rpi: Rename dropFrameCount_ to invalidCount_ Rename dropFrameCount_ to invalidCount_ to better reflect its use as frames are no longer dropped by the pipeline handler. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/ipa/rpi/common/ipa_base.cpp | 10 +++++----- src/ipa/rpi/common/ipa_base.h | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index eee81e064..50e25f057 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -327,7 +327,7 @@ void IpaBase::start(const ControlList &controls, StartResult *result) unsigned int agcConvergenceFrames = 0, awbConvergenceFrames = 0; frameCount_ = 0; if (firstStart_) { - dropFrameCount_ = helper_->hideFramesStartup(); + invalidCount_ = helper_->hideFramesStartup(); mistrustCount_ = helper_->mistrustFramesStartup(); /* @@ -353,14 +353,14 @@ void IpaBase::start(const ControlList &controls, StartResult *result) awbConvergenceFrames += mistrustCount_; } } else { - dropFrameCount_ = helper_->hideFramesModeSwitch(); + invalidCount_ = helper_->hideFramesModeSwitch(); mistrustCount_ = helper_->mistrustFramesModeSwitch(); } result->startupFrameCount = std::max({ agcConvergenceFrames, awbConvergenceFrames }); - result->invalidFrameCount = dropFrameCount_; + result->invalidFrameCount = invalidCount_; - dropFrameCount_ = std::max({ dropFrameCount_, agcConvergenceFrames, awbConvergenceFrames }); + invalidCount_ = std::max({ invalidCount_, agcConvergenceFrames, awbConvergenceFrames }); LOG(IPARPI, Debug) << "Startup frames: " << result->startupFrameCount << " Invalid frames: " << result->invalidFrameCount; @@ -443,7 +443,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) /* Allow a 10% margin on the comparison below. */ Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns; - if (lastRunTimestamp_ && frameCount_ > dropFrameCount_ && + if (lastRunTimestamp_ && frameCount_ > invalidCount_ && delta < controllerMinFrameDuration * 0.9 && !hdrChange) { /* * Ensure we merge the previous frame's metadata with the current diff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h index 1a811beb3..e818104ba 100644 --- a/src/ipa/rpi/common/ipa_base.h +++ b/src/ipa/rpi/common/ipa_base.h @@ -115,8 +115,8 @@ private: /* How many frames we should avoid running control algos on. */ unsigned int mistrustCount_; - /* Number of frames that need to be dropped on startup. */ - unsigned int dropFrameCount_; + /* Number of frames that need to be marked as dropped on startup. */ + unsigned int invalidCount_; /* Frame timestamp for the last run of the controller. */ uint64_t lastRunTimestamp_; From b544ce1c19c96917b207c0ba2efbe5f186b48212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Fri, 13 Jun 2025 16:31:53 +0200 Subject: [PATCH 12/64] apps: common: image: Fix assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `plane` must be strictly less than the vector's size, it cannot be equal to it. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/apps/common/image.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/common/image.cpp b/src/apps/common/image.cpp index a2a0f58f3..9a67238aa 100644 --- a/src/apps/common/image.cpp +++ b/src/apps/common/image.cpp @@ -98,12 +98,12 @@ unsigned int Image::numPlanes() const Span Image::data(unsigned int plane) { - assert(plane <= planes_.size()); + assert(plane < planes_.size()); return planes_[plane]; } Span Image::data(unsigned int plane) const { - assert(plane <= planes_.size()); + assert(plane < planes_.size()); return planes_[plane]; } From 5c8de8a08e0af64c7666b00d022e9f1fe352bee5 Mon Sep 17 00:00:00 2001 From: Antoine Bouyer Date: Wed, 4 Jun 2025 15:14:02 +0200 Subject: [PATCH 13/64] pipeline: imx8-isi: Cosmetic changes Change indentation to pass checkstyle script. Fixes: 680cde600509 ("libcamera: imx8-isi: Split Bayer/YUV config generation") Signed-off-by: Antoine Bouyer Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/libcamera/pipeline/imx8-isi/imx8-isi.cpp | 42 ++++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp index ecda426a6..c393319de 100644 --- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp +++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp @@ -761,30 +761,28 @@ PipelineHandlerISI::generateConfiguration(Camera *camera, */ StreamConfiguration cfg; - switch (role) { - case StreamRole::StillCapture: - case StreamRole::Viewfinder: - case StreamRole::VideoRecording: { - Size size = role == StreamRole::StillCapture - ? data->sensor_->resolution() - : PipelineHandlerISI::kPreviewSize; - cfg = generateYUVConfiguration(camera, size); - if (cfg.pixelFormat.isValid()) - break; + switch (role) { + case StreamRole::StillCapture: + case StreamRole::Viewfinder: + case StreamRole::VideoRecording: { + Size size = role == StreamRole::StillCapture + ? data->sensor_->resolution() + : PipelineHandlerISI::kPreviewSize; + cfg = generateYUVConfiguration(camera, size); + if (cfg.pixelFormat.isValid()) + break; + /* + * Fallback to use a Bayer format if that's what the + * sensor supports. + */ + [[fallthrough]]; + } - /* - * Fallback to use a Bayer format if that's what the - * sensor supports. - */ - [[fallthrough]]; - - } - - case StreamRole::Raw: { - cfg = generateRawConfiguration(camera); - break; - } + case StreamRole::Raw: { + cfg = generateRawConfiguration(camera); + break; + } default: LOG(ISI, Error) << "Requested stream role not supported: " << role; From 5621ac27a2c639c8e84f8c838486c8eaf72f3d74 Mon Sep 17 00:00:00 2001 From: Antoine Bouyer Date: Wed, 4 Jun 2025 15:14:03 +0200 Subject: [PATCH 14/64] pipeline: imx8-isi: Fix match returned value in error case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The match() function returns a boolean type, while it could return int in case of error when opening the capture file. Fixes: 0ec982d21086 ("libcamera: pipeline: Add IMX8 ISI pipeline") Signed-off-by: Antoine Bouyer Reviewed-by: Barnabás Pőcze Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/libcamera/pipeline/imx8-isi/imx8-isi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp index c393319de..b81e436b2 100644 --- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp +++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp @@ -1003,7 +1003,7 @@ bool PipelineHandlerISI::match(DeviceEnumerator *enumerator) ret = capture->open(); if (ret) - return ret; + return false; pipes_.push_back({ std::move(isi), std::move(capture) }); } From d3f3b95b64d2239adca7e0c6e9eb35db690ca98d Mon Sep 17 00:00:00 2001 From: Antoine Bouyer Date: Fri, 6 Jun 2025 09:34:59 +0200 Subject: [PATCH 15/64] pipeline: imx8-isi: Dynamically compute crossbar subdevice's first source. So far, imx8-isi pipeline supports _symetrical_ crossbar, with same amount of sink and source pads. But for some other imx SoCs, such as i.MX8QM or i.MX95, crossbar is not symetric anymore. Since each crossbar source is already captured as a pipes_ vector entry, we use pipes_ vector's size to compute 1st source index. "1st source index" = "total number of crossbar pads" - pipes_.count() Signed-off-by: Antoine Bouyer Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/libcamera/pipeline/imx8-isi/imx8-isi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp index b81e436b2..f4014b95d 100644 --- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp +++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp @@ -820,7 +820,7 @@ int PipelineHandlerISI::configure(Camera *camera, CameraConfiguration *c) * routing table instead of resetting it. */ V4L2Subdevice::Routing routing = {}; - unsigned int xbarFirstSource = crossbar_->entity()->pads().size() / 2 + 1; + unsigned int xbarFirstSource = crossbar_->entity()->pads().size() - pipes_.size(); for (const auto &[idx, config] : utils::enumerate(*c)) { uint32_t sourcePad = xbarFirstSource + idx; From 04e7823eb24bbc665f98fcb4dc5c3d494b39a356 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 22 May 2025 11:47:29 +0200 Subject: [PATCH 16/64] gstreamer: Document improvements when updating minimum GStreamer version A const_cast<> was recently added to fix a compilation issue with older GStreamer versions. Add a comment to indicate it can be removed when bumping the minimum GStreamer version requirement. While at it, also document a possible future improvement in the same function, and wrap long lines. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index b34f08977..6ede43b06 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -285,10 +285,19 @@ gst_libcamera_extrapolate_info(GstVideoInfo *info, guint32 stride) } static GstFlowReturn -gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, const GstVideoInfo *dest_info, guint32 stride) +gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, + const GstVideoInfo *dest_info, guint32 stride) { - GstVideoInfo src_info = *dest_info; + /* + * When dropping support for versions earlier than v1.22.0, use + * + * g_auto (GstVideoFrame) src_frame = GST_VIDEO_FRAME_INIT; + * g_auto (GstVideoFrame) dest_frame = GST_VIDEO_FRAME_INIT; + * + * and drop the gst_video_frame_unmap() calls. + */ GstVideoFrame src_frame, dest_frame; + GstVideoInfo src_info = *dest_info; gst_libcamera_extrapolate_info(&src_info, stride); src_info.size = gst_buffer_get_size(src); @@ -298,7 +307,12 @@ gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, const GstVideoIn return GST_FLOW_ERROR; } - if (!gst_video_frame_map(&dest_frame, const_cast(dest_info), dest, GST_MAP_WRITE)) { + /* + * When dropping support for versions earlier than 1.20.0, drop the + * const_cast<>(). + */ + if (!gst_video_frame_map(&dest_frame, const_cast(dest_info), + dest, GST_MAP_WRITE)) { GST_ERROR("Could not map dest buffer"); gst_video_frame_unmap(&src_frame); return GST_FLOW_ERROR; From 3b6820778918a9971c6b9b39385d408682d6193e Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 22 May 2025 11:47:29 +0200 Subject: [PATCH 17/64] gstreamer: Factor out video pool creation The gst_libcamera_src_negotiate() function uses 5 indentation levels, causing long lines. Move video pool creation to a separate function to increase readability. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 113 +++++++++++++++++++----------- 1 file changed, 71 insertions(+), 42 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 6ede43b06..34a08f8ea 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -521,6 +522,71 @@ gst_libcamera_src_open(GstLibcameraSrc *self) return true; } +/** + * \brief Create a video pool for a pad + * \param[in] self The libcamerasrc instance + * \param[in] srcpad The pad + * \param[in] caps The pad caps + * \param[in] info The video info for the pad + * + * This function creates and returns a video buffer pool for the given pad if + * needed to accommodate stride mismatch. If the peer element supports stride + * negotiation through the meta API, no pool is needed and the function will + * return a null pool. + * + * \return A tuple containing the video buffers pool pointer and an error code + */ +static std::tuple +gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, + GstCaps *caps, const GstVideoInfo *info) +{ + GstQuery *query = NULL; + const gboolean need_pool = true; + gboolean has_video_meta = false; + GstBufferPool *video_pool = NULL; + + query = gst_query_new_allocation(caps, need_pool); + if (!gst_pad_peer_query(srcpad, query)) + GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); + else + has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + + if (!has_video_meta) { + GstBufferPool *pool = NULL; + + if (gst_query_get_n_allocation_pools(query) > 0) + gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); + + if (pool) + video_pool = pool; + else { + GstStructure *config; + guint min_buffers = 3; + video_pool = gst_video_buffer_pool_new(); + + config = gst_buffer_pool_get_config(video_pool); + gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); + + GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); + + gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); + } + + GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); + + if (!gst_buffer_pool_set_active(video_pool, true)) { + gst_caps_unref(caps); + GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, + ("Failed to active buffer pool"), + ("gst_libcamera_src_negotiate() failed.")); + return { NULL, -EINVAL }; + } + } + + gst_query_unref(query); + return { video_pool, 0 }; +} + /* Must be called with stream_lock held. */ static bool gst_libcamera_src_negotiate(GstLibcameraSrc *self) @@ -603,50 +669,13 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self) /* Stride mismatch between camera stride and that calculated by video-info. */ if (static_cast(info.stride[0]) != stream_cfg.stride && GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) { - GstQuery *query = NULL; - const gboolean need_pool = true; - gboolean has_video_meta = false; - gst_libcamera_extrapolate_info(&info, stream_cfg.stride); - query = gst_query_new_allocation(caps, need_pool); - if (!gst_pad_peer_query(srcpad, query)) - GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); - else - has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); - - if (!has_video_meta) { - GstBufferPool *pool = NULL; - - if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); - - if (pool) - video_pool = pool; - else { - GstStructure *config; - guint min_buffers = 3; - video_pool = gst_video_buffer_pool_new(); - - config = gst_buffer_pool_get_config(video_pool); - gst_buffer_pool_config_set_params(config, caps, info.size, min_buffers, 0); - - GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); - - gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); - } - - GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); - - if (!gst_buffer_pool_set_active(video_pool, true)) { - gst_caps_unref(caps); - GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, - ("Failed to active buffer pool"), - ("gst_libcamera_src_negotiate() failed.")); - return false; - } - } - gst_query_unref(query); + std::tie(video_pool, ret) = + gst_libcamera_create_video_pool(self, srcpad, + caps, &info); + if (ret) + return false; } GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator, From 613202b80963c8290964f93c1e5090af8c2957f2 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 22 May 2025 11:47:29 +0200 Subject: [PATCH 18/64] gstreamer: Reduce indentation in gst_libcamera_create_video_pool() Now that video pool creation is handled by a dedicated function, the logic can be simplified by returning early instead of nesting scopes. Do so to decrease indentation and improve readability, and document the implementation of the function with comments. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 57 ++++++++++++++++--------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 34a08f8ea..1294c9a09 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -542,45 +542,48 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, { GstQuery *query = NULL; const gboolean need_pool = true; - gboolean has_video_meta = false; GstBufferPool *video_pool = NULL; + /* + * Get the peer allocation hints to check if it supports the meta API. + * If so, the stride will be negotiated, and there's no need to create a + * video pool. + */ query = gst_query_new_allocation(caps, need_pool); + if (!gst_pad_peer_query(srcpad, query)) GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); - else - has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + else if (gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL)) + return { NULL, 0 }; - if (!has_video_meta) { - GstBufferPool *pool = NULL; + GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); - if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); + /* + * If the allocation query has pools, use the first one. Otherwise, + * create a new pool. + */ + if (gst_query_get_n_allocation_pools(query) > 0) + gst_query_parse_nth_allocation_pool(query, 0, &video_pool, NULL, NULL, NULL); - if (pool) - video_pool = pool; - else { - GstStructure *config; - guint min_buffers = 3; - video_pool = gst_video_buffer_pool_new(); + if (!video_pool) { + GstStructure *config; + guint min_buffers = 3; - config = gst_buffer_pool_get_config(video_pool); - gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); + video_pool = gst_video_buffer_pool_new(); + config = gst_buffer_pool_get_config(video_pool); + gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); - GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); + GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); - gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); - } + gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); + } - GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); - - if (!gst_buffer_pool_set_active(video_pool, true)) { - gst_caps_unref(caps); - GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, - ("Failed to active buffer pool"), - ("gst_libcamera_src_negotiate() failed.")); - return { NULL, -EINVAL }; - } + if (!gst_buffer_pool_set_active(video_pool, true)) { + gst_caps_unref(caps); + GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, + ("Failed to active buffer pool"), + ("gst_libcamera_src_negotiate() failed.")); + return { NULL, -EINVAL }; } gst_query_unref(query); From f7c4fcd30191bc87fa00f7886df32bf285898195 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 4 Jun 2025 04:08:20 +0300 Subject: [PATCH 19/64] gstreamer: Rename variable in gst_libcamera_create_video_pool() Now that the code is isolated in a function, the video_pool variable in gst_libcamera_create_video_pool() can be renamed to pool without clashing with another local variable. Do so to reduce line length. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 1294c9a09..9cc8d9e3b 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -542,7 +542,7 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, { GstQuery *query = NULL; const gboolean need_pool = true; - GstBufferPool *video_pool = NULL; + GstBufferPool *pool = NULL; /* * Get the peer allocation hints to check if it supports the meta API. @@ -563,22 +563,22 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, * create a new pool. */ if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &video_pool, NULL, NULL, NULL); + gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); - if (!video_pool) { + if (!pool) { GstStructure *config; guint min_buffers = 3; - video_pool = gst_video_buffer_pool_new(); - config = gst_buffer_pool_get_config(video_pool); + pool = gst_video_buffer_pool_new(); + config = gst_buffer_pool_get_config(pool); gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); - gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); + gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(pool), config); } - if (!gst_buffer_pool_set_active(video_pool, true)) { + if (!gst_buffer_pool_set_active(pool, true)) { gst_caps_unref(caps); GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, ("Failed to active buffer pool"), @@ -587,7 +587,7 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, } gst_query_unref(query); - return { video_pool, 0 }; + return { pool, 0 }; } /* Must be called with stream_lock held. */ From 772b06bd8c7e989c7fc041e7ad982cbec87e0635 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 22 May 2025 11:47:29 +0200 Subject: [PATCH 20/64] gstreamer: Fix leak of GstQuery and GstBufferPool in error path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gst_libcamera_create_video_pool() function leaks a GstQuery instance and a GstBufferPool instance in an error path. Fix the leaks with g_autoptr(). Signed-off-by: Laurent Pinchart Reviewed-by: Barnabás Pőcze --- src/gstreamer/gstlibcamerasrc.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 9cc8d9e3b..05b47ece5 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -540,9 +541,9 @@ static std::tuple gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, GstCaps *caps, const GstVideoInfo *info) { - GstQuery *query = NULL; + g_autoptr(GstQuery) query = NULL; + g_autoptr(GstBufferPool) pool = NULL; const gboolean need_pool = true; - GstBufferPool *pool = NULL; /* * Get the peer allocation hints to check if it supports the meta API. @@ -586,8 +587,7 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, return { NULL, -EINVAL }; } - gst_query_unref(query); - return { pool, 0 }; + return { std::exchange(pool, nullptr), 0 }; } /* Must be called with stream_lock held. */ From a8f90517e0f752148b2b26dfc2771df32962ec45 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 4 Jun 2025 02:36:01 +0300 Subject: [PATCH 21/64] gstreamer: Drop incorrect unref on caps The caps object passeed to the gst_libcamera_create_video_pool() function is managed as a g_autoptr() in the caller. The function doesn't acquire any new reference, so it shouldn't call gst_caps_unref(). Fix it. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 05b47ece5..d86b423a9 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -580,7 +580,6 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, } if (!gst_buffer_pool_set_active(pool, true)) { - gst_caps_unref(caps); GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, ("Failed to active buffer pool"), ("gst_libcamera_src_negotiate() failed.")); From b3ff75d7589a263412ad63008b3c8518d40e6316 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 4 Jun 2025 04:17:10 +0300 Subject: [PATCH 22/64] gstreamer: Replace NULL with nullptr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Usage of NULL has slowly crept in the libcamerasrc sources. Replace it with nullptr. Reported-by: Nicolas Dufresne Signed-off-by: Laurent Pinchart Reviewed-by: Barnabás Pőcze Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamera-controls.cpp.in | 2 +- src/gstreamer/gstlibcamerapad.cpp | 2 +- src/gstreamer/gstlibcamerasrc.cpp | 23 +++++++++++----------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/gstreamer/gstlibcamera-controls.cpp.in b/src/gstreamer/gstlibcamera-controls.cpp.in index 89c530da0..2a16b39a9 100644 --- a/src/gstreamer/gstlibcamera-controls.cpp.in +++ b/src/gstreamer/gstlibcamera-controls.cpp.in @@ -68,7 +68,7 @@ static const GEnumValue {{ ctrl.name|snake_case }}_types[] = { "{{ enum.gst_name }}" }, {%- endfor %} - {0, NULL, NULL} + {0, nullptr, nullptr} }; #define TYPE_{{ ctrl.name|snake_case|upper }} \ diff --git a/src/gstreamer/gstlibcamerapad.cpp b/src/gstreamer/gstlibcamerapad.cpp index 3bc2bc87e..81a0ef44c 100644 --- a/src/gstreamer/gstlibcamerapad.cpp +++ b/src/gstreamer/gstlibcamerapad.cpp @@ -102,7 +102,7 @@ gst_libcamera_stream_role_get_type() "libcamera::Viewfinder", "view-finder", }, - { 0, NULL, NULL } + { 0, nullptr, nullptr } }; if (!type) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index d86b423a9..ebea34e47 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -368,10 +368,10 @@ int GstLibcameraSrcState::processRequest() if (video_pool) { /* Only set video pool when a copy is needed. */ - GstBuffer *copy = NULL; + GstBuffer *copy = nullptr; const GstVideoInfo info = gst_libcamera_pad_get_video_info(srcpad); - ret = gst_buffer_pool_acquire_buffer(video_pool, ©, NULL); + ret = gst_buffer_pool_acquire_buffer(video_pool, ©, nullptr); if (ret != GST_FLOW_OK) { gst_buffer_unref(buffer); GST_ELEMENT_ERROR(src_, RESOURCE, SETTINGS, @@ -541,8 +541,8 @@ static std::tuple gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, GstCaps *caps, const GstVideoInfo *info) { - g_autoptr(GstQuery) query = NULL; - g_autoptr(GstBufferPool) pool = NULL; + g_autoptr(GstQuery) query = nullptr; + g_autoptr(GstBufferPool) pool = nullptr; const gboolean need_pool = true; /* @@ -554,8 +554,8 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, if (!gst_pad_peer_query(srcpad, query)) GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); - else if (gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL)) - return { NULL, 0 }; + else if (gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, nullptr)) + return { nullptr, 0 }; GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); @@ -564,7 +564,8 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, * create a new pool. */ if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); + gst_query_parse_nth_allocation_pool(query, 0, &pool, nullptr, + nullptr, nullptr); if (!pool) { GstStructure *config; @@ -583,7 +584,7 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, ("Failed to active buffer pool"), ("gst_libcamera_src_negotiate() failed.")); - return { NULL, -EINVAL }; + return { nullptr, -EINVAL }; } return { std::exchange(pool, nullptr), 0 }; @@ -660,7 +661,7 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self) for (gsize i = 0; i < state->srcpads_.size(); i++) { GstPad *srcpad = state->srcpads_[i]; const StreamConfiguration &stream_cfg = state->config_->at(i); - GstBufferPool *video_pool = NULL; + GstBufferPool *video_pool = nullptr; GstVideoInfo info; g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg, transfer[i]); @@ -1065,7 +1066,7 @@ gst_libcamera_src_request_new_pad(GstElement *element, GstPadTemplate *templ, const gchar *name, [[maybe_unused]] const GstCaps *caps) { GstLibcameraSrc *self = GST_LIBCAMERA_SRC(element); - g_autoptr(GstPad) pad = NULL; + g_autoptr(GstPad) pad = nullptr; GST_DEBUG_OBJECT(self, "new request pad created"); @@ -1079,7 +1080,7 @@ gst_libcamera_src_request_new_pad(GstElement *element, GstPadTemplate *templ, GST_ELEMENT_ERROR(element, STREAM, FAILED, ("Internal data stream error."), ("Could not add pad to element")); - return NULL; + return nullptr; } gst_child_proxy_child_added(GST_CHILD_PROXY(self), G_OBJECT(pad), GST_OBJECT_NAME(pad)); From b4c92a61bf6bafd3f9fd55f377c274a257814b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Wed, 4 Jun 2025 17:00:46 +0200 Subject: [PATCH 23/64] ipa: rpi: Initialize enum controls with a list of values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is how uvcvideo and rkisp1 do it. See ee918b370a08b ("ipa: rkisp1: agc: Initialize enum controls with a list of values") for the motivation. In summary, having a list of values is used as a sign that the control is an enum in multiple places (e.g. `cam`, `camshark`). Signed-off-by: Barnabás Pőcze Reviewed-by: Naushir Patuck Reviewed-by: Laurent Pinchart --- src/ipa/rpi/common/ipa_base.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index 50e25f057..6565f5366 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -58,23 +58,24 @@ const ControlInfoMap::Map ipaControls{ /* \todo Move this to the Camera class */ { &controls::AeEnable, ControlInfo(false, true, true) }, { &controls::ExposureTimeMode, - ControlInfo(static_cast(controls::ExposureTimeModeAuto), - static_cast(controls::ExposureTimeModeManual), - static_cast(controls::ExposureTimeModeAuto)) }, + ControlInfo({ { ControlValue(controls::ExposureTimeModeAuto), + ControlValue(controls::ExposureTimeModeManual) } }, + ControlValue(controls::ExposureTimeModeAuto)) }, { &controls::ExposureTime, ControlInfo(1, 66666, static_cast(defaultExposureTime.get())) }, { &controls::AnalogueGainMode, - ControlInfo(static_cast(controls::AnalogueGainModeAuto), - static_cast(controls::AnalogueGainModeManual), - static_cast(controls::AnalogueGainModeAuto)) }, + ControlInfo({ { ControlValue(controls::AnalogueGainModeAuto), + ControlValue(controls::AnalogueGainModeManual) } }, + ControlValue(controls::AnalogueGainModeAuto)) }, { &controls::AnalogueGain, ControlInfo(1.0f, 16.0f, 1.0f) }, { &controls::AeMeteringMode, ControlInfo(controls::AeMeteringModeValues) }, { &controls::AeConstraintMode, ControlInfo(controls::AeConstraintModeValues) }, { &controls::AeExposureMode, ControlInfo(controls::AeExposureModeValues) }, { &controls::ExposureValue, ControlInfo(-8.0f, 8.0f, 0.0f) }, - { &controls::AeFlickerMode, ControlInfo(static_cast(controls::FlickerOff), - static_cast(controls::FlickerManual), - static_cast(controls::FlickerOff)) }, + { &controls::AeFlickerMode, + ControlInfo({ { ControlValue(controls::FlickerOff), + ControlValue(controls::FlickerManual) } }, + ControlValue(controls::FlickerOff)) }, { &controls::AeFlickerPeriod, ControlInfo(100, 1000000) }, { &controls::Brightness, ControlInfo(-1.0f, 1.0f, 0.0f) }, { &controls::Contrast, ControlInfo(0.0f, 32.0f, 1.0f) }, From 4a277906a41e73b4e35e1c528f906e3c70938b16 Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Thu, 5 Jun 2025 15:37:41 +0900 Subject: [PATCH 24/64] gstreamer: Fix libcamerasrc responding latency before setting caps Whenever a downstream element queries latency, libcamerasrc will always reply, even though it has not yet determined the latency. However some downstream elements (e.g. glvideomixer/aggregator) will query the latency before libcamerasrc sets the caps. When these elements get the latency, they will start the caps negotiation. Since libcamerasrc has not yet determined caps, invalid negotiation is performed and workflow is disrupted. So, set latency to 'GST_CLOCK_TIME_NONE' during initialization, and reply to the query after libcamerasrc confirms the latency. At this time, libcamerasrc has also completed caps negotiation and downstream elements work fine. In addition, every time the src pad task stops, we reset the latency to GST_CLOCK_TIME_NONE to ensure that when next time task starts, the downstream elements can generate out buffers after receiving the effective latency. Signed-off-by: Hou Qi Reviewed-by: Nicolas Dufresne Signed-off-by: Kieran Bingham --- src/gstreamer/gstlibcamerapad.cpp | 5 +++++ src/gstreamer/gstlibcamerasrc.cpp | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/gstreamer/gstlibcamerapad.cpp b/src/gstreamer/gstlibcamerapad.cpp index 81a0ef44c..22b967198 100644 --- a/src/gstreamer/gstlibcamerapad.cpp +++ b/src/gstreamer/gstlibcamerapad.cpp @@ -72,6 +72,10 @@ gst_libcamera_pad_query(GstPad *pad, GstObject *parent, GstQuery *query) if (query->type != GST_QUERY_LATENCY) return gst_pad_query_default(pad, parent, query); + GLibLocker lock(GST_OBJECT(self)); + if (self->latency == GST_CLOCK_TIME_NONE) + return FALSE; + /* TRUE here means live, we assumes that max latency is the same as min * as we have no idea that duration of frames. */ gst_query_set_latency(query, TRUE, self->latency, self->latency); @@ -81,6 +85,7 @@ gst_libcamera_pad_query(GstPad *pad, GstObject *parent, GstQuery *query) static void gst_libcamera_pad_init(GstLibcameraPad *self) { + self->latency = GST_CLOCK_TIME_NONE; GST_PAD_QUERYFUNC(self) = gst_libcamera_pad_query; } diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index ebea34e47..51ba8b67a 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -881,8 +881,10 @@ gst_libcamera_src_task_leave([[maybe_unused]] GstTask *task, { GLibRecLocker locker(&self->stream_lock); - for (GstPad *srcpad : state->srcpads_) + for (GstPad *srcpad : state->srcpads_) { + gst_libcamera_pad_set_latency(srcpad, GST_CLOCK_TIME_NONE); gst_libcamera_pad_set_pool(srcpad, nullptr); + } } g_clear_object(&self->allocator); From 6a09deaf7d4a402051ee86ddc83d5af18e92eec9 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 19 Jun 2025 11:05:53 +0100 Subject: [PATCH 25/64] controls: Add FrameWallClock control Add a FrameWallClock control that reports the same moment as the frame's SensorTimestamp, but in wallclock units. Signed-off-by: David Plowman Reviewed-by: Kieran Bingham Reviewed-by: Naushir Patuck Reviewed-by: Laurent Pinchart Signed-off-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_core.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index aa7448645..028919ef3 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -1,6 +1,4 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2019, Google Inc. + # %YAML 1.1 --- @@ -1268,4 +1266,19 @@ controls: description: | Enable or disable the debug metadata. + - FrameWallClock: + type: int64_t + direction: out + description: | + This timestamp corresponds to the same moment in time as the + SensorTimestamp, but is represented as a wall clock time as measured by + the CLOCK_REALTIME clock. + + Being a wall clock measurement, it can be used to synchronise timing + across different devices. + + \sa SensorTimestamp + + The FrameWallClock control can only be returned in metadata. + ... From 2a4e347dfea352901cb2d609489f47157e4c2c2c Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 19 Jun 2025 11:05:54 +0100 Subject: [PATCH 26/64] libcamera: Add ClockRecovery class to generate wallclock timestamps The ClockRecovery class takes pairs of timestamps from two different clocks, and models the second ("output") clock from the first ("input") clock. We can use it, in particular, to get a good wallclock estimate for a frame's SensorTimestamp. Signed-off-by: David Plowman Reviewed-by: Kieran Bingham Reviewed-by: Naushir Patuck Signed-off-by: Naushir Patuck Signed-off-by: Kieran Bingham --- include/libcamera/internal/clock_recovery.h | 68 ++++++ include/libcamera/internal/meson.build | 1 + src/libcamera/clock_recovery.cpp | 230 ++++++++++++++++++++ src/libcamera/meson.build | 1 + 4 files changed, 300 insertions(+) create mode 100644 include/libcamera/internal/clock_recovery.h create mode 100644 src/libcamera/clock_recovery.cpp diff --git a/include/libcamera/internal/clock_recovery.h b/include/libcamera/internal/clock_recovery.h new file mode 100644 index 000000000..43e46b7dc --- /dev/null +++ b/include/libcamera/internal/clock_recovery.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Raspberry Pi Ltd + * + * Camera recovery algorithm + */ +#pragma once + +#include + +namespace libcamera { + +class ClockRecovery +{ +public: + ClockRecovery(); + + void configure(unsigned int numSamples = 100, unsigned int maxJitter = 2000, + unsigned int minSamples = 10, unsigned int errorThreshold = 50000); + void reset(); + + void addSample(); + void addSample(uint64_t input, uint64_t output); + + uint64_t getOutput(uint64_t input); + +private: + /* Approximate number of samples over which the model state persists. */ + unsigned int numSamples_; + /* Remove any output jitter larger than this immediately. */ + unsigned int maxJitter_; + /* Number of samples required before we start to use model estimates. */ + unsigned int minSamples_; + /* Threshold above which we assume the wallclock has been reset. */ + unsigned int errorThreshold_; + + /* How many samples seen (up to numSamples_). */ + unsigned int count_; + /* This gets subtracted from all input values, just to make the numbers easier. */ + uint64_t inputBase_; + /* As above, for the output. */ + uint64_t outputBase_; + /* The previous input sample. */ + uint64_t lastInput_; + /* The previous output sample. */ + uint64_t lastOutput_; + + /* Average x value seen so far. */ + double xAve_; + /* Average y value seen so far */ + double yAve_; + /* Average x^2 value seen so far. */ + double x2Ave_; + /* Average x*y value seen so far. */ + double xyAve_; + + /* + * The latest estimate of linear parameters to derive the output clock + * from the input. + */ + double slope_; + double offset_; + + /* Use this cumulative error to monitor for spontaneous clock updates. */ + double error_; +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index 33f318b2b..5c80a28c4 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -11,6 +11,7 @@ libcamera_internal_headers = files([ 'camera_manager.h', 'camera_sensor.h', 'camera_sensor_properties.h', + 'clock_recovery.h', 'control_serializer.h', 'control_validator.h', 'converter.h', diff --git a/src/libcamera/clock_recovery.cpp b/src/libcamera/clock_recovery.cpp new file mode 100644 index 000000000..abacf444f --- /dev/null +++ b/src/libcamera/clock_recovery.cpp @@ -0,0 +1,230 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Raspberry Pi Ltd + * + * Clock recovery algorithm + */ + +#include "libcamera/internal/clock_recovery.h" + +#include + +#include + +/** + * \file clock_recovery.h + * \brief Clock recovery - deriving one clock from another independent clock + */ + +namespace libcamera { + +LOG_DEFINE_CATEGORY(ClockRec) + +/** + * \class ClockRecovery + * \brief Recover an output clock from an input clock + * + * The ClockRecovery class derives an output clock from an input clock, + * modelling the output clock as being linearly related to the input clock. + * For example, we may use it to derive wall clock timestamps from timestamps + * measured by the internal system clock which counts local time since boot. + * + * When pairs of corresponding input and output timestamps are available, + * they should be submitted to the model with addSample(). The model will + * update, and output clock values for known input clock values can be + * obtained using getOutput(). + * + * As a convenience, if the input clock is indeed the time since boot, and the + * output clock represents a real wallclock time, then addSample() can be + * called with no arguments, and a pair of timestamps will be captured at + * that moment. + * + * The configure() function accepts some configuration parameters to control + * the linear fitting process. + */ + +/** + * \brief Construct a ClockRecovery + */ +ClockRecovery::ClockRecovery() +{ + configure(); + reset(); +} + +/** + * \brief Set configuration parameters + * \param[in] numSamples The approximate duration for which the state of the model + * is persistent + * \param[in] maxJitter New output samples are clamped to no more than this + * amount of jitter, to prevent sudden swings from having a large effect + * \param[in] minSamples The fitted clock model is not used to generate outputs + * until this many samples have been received + * \param[in] errorThreshold If the accumulated differences between input and + * output clocks reaches this amount over a few frames, the model is reset + */ +void ClockRecovery::configure(unsigned int numSamples, unsigned int maxJitter, + unsigned int minSamples, unsigned int errorThreshold) +{ + LOG(ClockRec, Debug) + << "configure " << numSamples << " " << maxJitter << " " << minSamples << " " << errorThreshold; + + numSamples_ = numSamples; + maxJitter_ = maxJitter; + minSamples_ = minSamples; + errorThreshold_ = errorThreshold; +} + +/** + * \brief Reset the clock recovery model and start again from scratch + */ +void ClockRecovery::reset() +{ + LOG(ClockRec, Debug) << "reset"; + + lastInput_ = 0; + lastOutput_ = 0; + xAve_ = 0; + yAve_ = 0; + x2Ave_ = 0; + xyAve_ = 0; + count_ = 0; + error_ = 0.0; + /* + * Setting slope_ and offset_ to zero initially means that the clocks + * advance at exactly the same rate. + */ + slope_ = 0.0; + offset_ = 0.0; +} + +/** + * \brief Add a sample point to the clock recovery model, for recovering a wall + * clock value from the internal system time since boot + * + * This is a convenience function to make it easy to derive a wall clock value + * (using the Linux CLOCK_REALTIME) from the time since the system started + * (measured by CLOCK_BOOTTIME). + */ +void ClockRecovery::addSample() +{ + LOG(ClockRec, Debug) << "addSample"; + + struct timespec bootTime1; + struct timespec bootTime2; + struct timespec wallTime; + + /* Get boot and wall clocks in microseconds. */ + clock_gettime(CLOCK_BOOTTIME, &bootTime1); + clock_gettime(CLOCK_REALTIME, &wallTime); + clock_gettime(CLOCK_BOOTTIME, &bootTime2); + uint64_t boot1 = bootTime1.tv_sec * 1000000ULL + bootTime1.tv_nsec / 1000; + uint64_t boot2 = bootTime2.tv_sec * 1000000ULL + bootTime2.tv_nsec / 1000; + uint64_t boot = (boot1 + boot2) / 2; + uint64_t wall = wallTime.tv_sec * 1000000ULL + wallTime.tv_nsec / 1000; + + addSample(boot, wall); +} + +/** + * \brief Add a sample point to the clock recovery model, specifying the exact + * input and output clock values + * \param[in] input The input clock value + * \param[in] output The value of the output clock at the same moment, as far + * as possible, that the input clock was sampled + * + * This function should be used for corresponding clocks other than the Linux + * BOOTTIME and REALTIME clocks. + */ +void ClockRecovery::addSample(uint64_t input, uint64_t output) +{ + LOG(ClockRec, Debug) << "addSample " << input << " " << output; + + if (count_ == 0) { + inputBase_ = input; + outputBase_ = output; + } + + /* + * We keep an eye on cumulative drift over the last several frames. If this exceeds a + * threshold, then probably the system clock has been updated and we're going to have to + * reset everything and start over. + */ + if (lastOutput_) { + int64_t inputDiff = getOutput(input) - getOutput(lastInput_); + int64_t outputDiff = output - lastOutput_; + error_ = error_ * 0.95 + (outputDiff - inputDiff); + if (std::abs(error_) > errorThreshold_) { + reset(); + inputBase_ = input; + outputBase_ = output; + } + } + lastInput_ = input; + lastOutput_ = output; + + /* + * Never let the new output value be more than maxJitter_ away from what + * we would have expected. This is just to reduce the effect of sudden + * large delays in the measured output. + */ + uint64_t expectedOutput = getOutput(input); + output = std::clamp(output, expectedOutput - maxJitter_, expectedOutput + maxJitter_); + + /* + * We use x, y, x^2 and x*y sums to calculate the best fit line. Here we + * update them by pretending we have count_ samples at the previous fit, + * and now one new one. Gradually the effect of the older values gets + * lost. This is a very simple way of updating the fit (there are much + * more complicated ones!), but it works well enough. Using averages + * instead of sums makes the relative effect of old values and the new + * sample clearer. + */ + double x = static_cast(input - inputBase_); + double y = static_cast(output - outputBase_) - x; + unsigned int count1 = count_ + 1; + xAve_ = (count_ * xAve_ + x) / count1; + yAve_ = (count_ * yAve_ + y) / count1; + x2Ave_ = (count_ * x2Ave_ + x * x) / count1; + xyAve_ = (count_ * xyAve_ + x * y) / count1; + + /* + * Don't update slope and offset until we've seen "enough" sample + * points. Note that the initial settings for slope_ and offset_ + * ensures that the wallclock advances at the same rate as the realtime + * clock (but with their respective initial offsets). + */ + if (count_ > minSamples_) { + /* These are the standard equations for least squares linear regression. */ + slope_ = (count1 * count1 * xyAve_ - count1 * xAve_ * count1 * yAve_) / + (count1 * count1 * x2Ave_ - count1 * xAve_ * count1 * xAve_); + offset_ = yAve_ - slope_ * xAve_; + } + + /* + * Don't increase count_ above numSamples_, as this controls the long-term + * amount of the residual fit. + */ + if (count1 < numSamples_) + count_++; +} + +/** + * \brief Calculate the output clock value according to the model from an input + * clock value + * \param[in] input The input clock value + * + * \return Output clock value + */ +uint64_t ClockRecovery::getOutput(uint64_t input) +{ + double x = static_cast(input - inputBase_); + double y = slope_ * x + offset_; + uint64_t output = y + x + outputBase_; + + LOG(ClockRec, Debug) << "getOutput " << input << " " << output; + + return output; +} + +} /* namespace libcamera */ diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 202db1efe..28a3b0f36 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -21,6 +21,7 @@ libcamera_internal_sources = files([ 'byte_stream_buffer.cpp', 'camera_controls.cpp', 'camera_lens.cpp', + 'clock_recovery.cpp', 'control_serializer.cpp', 'control_validator.cpp', 'converter.cpp', From 1d1ba78b457d636a19c784784cea876f7776f7b4 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 19 Jun 2025 11:05:55 +0100 Subject: [PATCH 27/64] controls: Add camera synchronisation controls for Raspberry Pi New controls are added to control the camera "sync" algorithm, which allows different cameras to synchronise their frames. For the time being, the controls are Raspberry Pi specific, though this is expected to change in future. Signed-off-by: David Plowman Reviewed-by: Naushir Patuck Signed-off-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_rpi.yaml | 112 +++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/src/libcamera/control_ids_rpi.yaml b/src/libcamera/control_ids_rpi.yaml index 8d1e8b47c..a86151123 100644 --- a/src/libcamera/control_ids_rpi.yaml +++ b/src/libcamera/control_ids_rpi.yaml @@ -71,4 +71,116 @@ controls: \sa StatsOutputEnable + - SyncMode: + type: int32_t + direction: in + description: | + Enable or disable camera synchronisation ("sync") mode. + + When sync mode is enabled, a camera will synchronise frames temporally + with other cameras, either attached to the same device or a different + one. There should be one "server" device, which broadcasts timing + information to one or more "clients". Communication is one-way, from + server to clients only, and it is only clients that adjust their frame + timings to match the server. + + Sync mode requires all cameras to be running at (as far as possible) the + same fixed framerate. Clients may continue to make adjustments to keep + their cameras synchronised with the server for the duration of the + session, though any updates after the initial ones should remain small. + + \sa SyncReady + \sa SyncTimer + \sa SyncFrames + + enum: + - name: SyncModeOff + value: 0 + description: Disable sync mode. + - name: SyncModeServer + value: 1 + description: | + Enable sync mode, act as server. The server broadcasts timing + messages to any clients that are listening, so that the clients can + synchronise their camera frames with the server's. + - name: SyncModeClient + value: 2 + description: | + Enable sync mode, act as client. A client listens for any server + messages, and arranges for its camera frames to synchronise as + closely as possible with the server's. Many clients can listen out + for the same server. Clients can also be started ahead of any + servers, causing them merely to wait for the server to start. + + - SyncReady: + type: bool + direction: out + description: | + When using the camera synchronisation algorithm, the server broadcasts + timing information to the clients. This also includes the time (some + number of frames in the future, called the "ready time") at which the + server will signal its controlling application, using this control, to + start using the image frames. + + The client receives the "ready time" from the server, and will signal + its application to start using the frames at this same moment. + + While this control value is false, applications (on both client and + server) should continue to wait, and not use the frames. + + Once this value becomes true, it means that this is the first frame + where the server and its clients have agreed that they will both be + synchronised and that applications should begin consuming frames. + Thereafter, this control will continue to signal the value true for + the rest of the session. + + \sa SyncMode + \sa SyncTimer + \sa SyncFrames + + - SyncTimer: + type: int64_t + direction: out + description: | + This reports the amount of time, in microseconds, until the "ready + time", at which the server and client will signal their controlling + applications that the frames are now synchronised and should be + used. The value may be refined slightly over time, becoming more precise + as the "ready time" approaches. + + Servers always report this value, whereas clients will omit this control + until they have received a message from the server that enables them to + calculate it. + + Normally the value will start positive (the "ready time" is in the + future), and decrease towards zero, before becoming negative (the "ready + time" has elapsed). So there should be just one frame where the timer + value is, or is very close to, zero - the one for which the SyncReady + control becomes true. At this moment, the value indicates how closely + synchronised the client believes it is with the server. + + But note that if frames are being dropped, then the "near zero" valued + frame, or indeed any other, could be skipped. In these cases the timer + value allows an application to deduce that this has happened. + + \sa SyncMode + \sa SyncReady + \sa SyncFrames + + - SyncFrames: + type: int32_t + direction: in + description: | + The number of frames the server should wait, after enabling + SyncModeServer, before signalling (via the SyncReady control) that + frames should be used. This therefore determines the "ready time" for + all synchronised cameras. + + This control value should be set only for the device that is to act as + the server, before or at the same moment at which SyncModeServer is + enabled. + + \sa SyncMode + \sa SyncReady + \sa SyncTimer ... From 1537da74427791bb3b5880e7d002daf8ea42db31 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 19 Jun 2025 11:05:56 +0100 Subject: [PATCH 28/64] pipeline: rpi: Add wallclock timestamp support A ClockRecovery object is added for derived classes to use, and wallclock timestamps are copied into the request metadata for applications. Wallclock timestamps are derived corresponding to the sensor timestamp, and made available to the base pipeline handler class and to IPAs, for both vc4 and pisp platforms. Signed-off-by: David Plowman Reviewed-by: Kieran Bingham Reviewed-by: Naushir Patuck Signed-off-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/libcamera/pipeline/rpi/common/pipeline_base.cpp | 5 +++++ src/libcamera/pipeline/rpi/common/pipeline_base.h | 3 +++ src/libcamera/pipeline/rpi/pisp/pisp.cpp | 10 ++++++++-- src/libcamera/pipeline/rpi/vc4/vc4.cpp | 10 ++++++++-- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index d8c7ca935..e14d3b913 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -685,6 +685,9 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) return ret; } + /* A good moment to add an initial clock sample. */ + data->wallClockRecovery_.addSample(); + /* * Reset the delayed controls with the gain and exposure values set by * the IPA. @@ -1482,6 +1485,8 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request { request->metadata().set(controls::SensorTimestamp, bufferControls.get(controls::SensorTimestamp).value_or(0)); + request->metadata().set(controls::FrameWallClock, + bufferControls.get(controls::FrameWallClock).value_or(0)); if (cropParams_.size()) { std::vector crops; diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 898f31577..4c5743e04 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -20,6 +20,7 @@ #include "libcamera/internal/bayer_format.h" #include "libcamera/internal/camera.h" #include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/clock_recovery.h" #include "libcamera/internal/framebuffer.h" #include "libcamera/internal/media_device.h" #include "libcamera/internal/media_object.h" @@ -172,6 +173,8 @@ public: Config config_; + ClockRecovery wallClockRecovery_; + protected: void fillRequestMetadata(const ControlList &bufferControls, Request *request); diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index ccf135c3d..2df91bacf 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -1755,9 +1755,15 @@ void PiSPCameraData::cfeBufferDequeue(FrameBuffer *buffer) auto [ctrl, delayContext] = delayedCtrls_->get(buffer->metadata().sequence); /* * Add the frame timestamp to the ControlList for the IPA to use - * as it does not receive the FrameBuffer object. + * as it does not receive the FrameBuffer object. Also derive a + * corresponding wallclock value. */ - ctrl.set(controls::SensorTimestamp, buffer->metadata().timestamp); + wallClockRecovery_.addSample(); + uint64_t sensorTimestamp = buffer->metadata().timestamp; + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp / 1000); + + ctrl.set(controls::SensorTimestamp, sensorTimestamp); + ctrl.set(controls::FrameWallClock, wallClockTimestamp); job.sensorControls = std::move(ctrl); job.delayContext = delayContext; } else if (stream == &cfe_[Cfe::Config]) { diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index ac6dab814..e99a7edf8 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -773,9 +773,15 @@ void Vc4CameraData::unicamBufferDequeue(FrameBuffer *buffer) auto [ctrl, delayContext] = delayedCtrls_->get(buffer->metadata().sequence); /* * Add the frame timestamp to the ControlList for the IPA to use - * as it does not receive the FrameBuffer object. + * as it does not receive the FrameBuffer object. Also derive a + * corresponding wallclock value. */ - ctrl.set(controls::SensorTimestamp, buffer->metadata().timestamp); + wallClockRecovery_.addSample(); + uint64_t sensorTimestamp = buffer->metadata().timestamp; + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp / 1000); + + ctrl.set(controls::SensorTimestamp, sensorTimestamp); + ctrl.set(controls::FrameWallClock, wallClockTimestamp); bayerQueue_.push({ buffer, std::move(ctrl), delayContext }); } else { embeddedQueue_.push(buffer); From 02a3b436c4d16498e05c79cd84daacef81906ab4 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 9 Jun 2025 18:01:51 +0300 Subject: [PATCH 29/64] ipa: rkisp1: Move Sharpness control creation to Filter algorithm The Sharpness control is used solely by the Filter algorithm. Create it there, to avoid exposing it to applications when the algorithm is disabled. Signed-off-by: Laurent Pinchart Reviewed-by: Kieran Bingham Reviewed-by: Paul Elder --- src/ipa/rkisp1/algorithms/filter.cpp | 11 +++++++++++ src/ipa/rkisp1/algorithms/filter.h | 1 + src/ipa/rkisp1/rkisp1.cpp | 1 - 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ipa/rkisp1/algorithms/filter.cpp b/src/ipa/rkisp1/algorithms/filter.cpp index 7598ef8a9..8ad798017 100644 --- a/src/ipa/rkisp1/algorithms/filter.cpp +++ b/src/ipa/rkisp1/algorithms/filter.cpp @@ -39,6 +39,17 @@ LOG_DEFINE_CATEGORY(RkISP1Filter) static constexpr uint32_t kFiltLumWeightDefault = 0x00022040; static constexpr uint32_t kFiltModeDefault = 0x000004f2; +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Filter::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + auto &cmap = context.ctrlMap; + cmap[&controls::Sharpness] = ControlInfo(0.0f, 10.0f, 1.0f); + + return 0; +} /** * \copydoc libcamera::ipa::Algorithm::queueRequest */ diff --git a/src/ipa/rkisp1/algorithms/filter.h b/src/ipa/rkisp1/algorithms/filter.h index 8f858e574..37d8938d3 100644 --- a/src/ipa/rkisp1/algorithms/filter.h +++ b/src/ipa/rkisp1/algorithms/filter.h @@ -21,6 +21,7 @@ public: Filter() = default; ~Filter() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; void queueRequest(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const ControlList &controls) override; diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index 1ed7d7d92..cf66d5553 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -116,7 +116,6 @@ const IPAHwSettings ipaHwSettingsV12{ /* List of controls handled by the RkISP1 IPA */ const ControlInfoMap::Map rkisp1Controls{ { &controls::DebugMetadataEnable, ControlInfo(false, true, false) }, - { &controls::Sharpness, ControlInfo(0.0f, 10.0f, 1.0f) }, { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) }, }; From c19047dfdfb451deac404841b79e1b5a6b89ff5f Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 4 Jun 2025 16:51:46 +0300 Subject: [PATCH 30/64] gstreamer: Use std::exchange() instead of g_steal_pointer() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit g_steal_pointer) only preserves the type since glib 2.68, requiring casts. Use std::exchange() instead. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne Reviewed-by: Barnabás Pőcze --- src/gstreamer/gstlibcamerasrc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 51ba8b67a..3aca4eeda 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -1087,7 +1087,7 @@ gst_libcamera_src_request_new_pad(GstElement *element, GstPadTemplate *templ, gst_child_proxy_child_added(GST_CHILD_PROXY(self), G_OBJECT(pad), GST_OBJECT_NAME(pad)); - return reinterpret_cast(g_steal_pointer(&pad)); + return std::exchange(pad, nullptr); } static void From 8ea3ef083fd009c1be38a0ff4001ba24da5f5cc1 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Thu, 12 Jun 2025 15:22:36 +0200 Subject: [PATCH 31/64] libcamera: test: Add a failing test for the log level parser Log level parsing doesn't always work as expected. Add a failing test for that. Signed-off-by: Stefan Klug Reviewed-by: Laurent Pinchart Reviewed-by: Kieran Bingham --- test/log/log_api.cpp | 39 ++++++++++++++++++++++++++++++++++++++- test/log/meson.build | 5 +++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/test/log/log_api.cpp b/test/log/log_api.cpp index 0b999738d..8d19cf0ce 100644 --- a/test/log/log_api.cpp +++ b/test/log/log_api.cpp @@ -26,6 +26,11 @@ using namespace std; using namespace libcamera; LOG_DEFINE_CATEGORY(LogAPITest) +LOG_DEFINE_CATEGORY(Cat0) +LOG_DEFINE_CATEGORY(Cat1) +LOG_DEFINE_CATEGORY(Cat2) +LOG_DEFINE_CATEGORY(Cat3) +LOG_DEFINE_CATEGORY(Cat4) class LogAPITest : public Test { @@ -74,6 +79,34 @@ protected: return TestPass; } + int testEnvLevels() + { + setenv("LIBCAMERA_LOG_LEVELS", + "Cat0:0,Cat0:9999,Cat1:INFO,Cat1:INVALID,Cat2:2,Cat2:-1," + "Cat3:ERROR,Cat3:{[]},Cat4:4,Cat4:rubbish", + true); + logSetTarget(libcamera::LoggingTargetNone); + + const std::pair expected[] = { + { _LOG_CATEGORY(Cat0)(), libcamera::LogDebug }, + { _LOG_CATEGORY(Cat1)(), libcamera::LogInfo }, + { _LOG_CATEGORY(Cat2)(), libcamera::LogWarning }, + { _LOG_CATEGORY(Cat3)(), libcamera::LogError }, + { _LOG_CATEGORY(Cat4)(), libcamera::LogFatal }, + }; + bool ok = true; + + for (const auto &[c, s] : expected) { + if (c.severity() != s) { + ok = false; + cerr << "Severity of " << c.name() << " (" << c.severity() << ") " + << "does not equal " << s << endl; + } + } + + return ok ? TestPass : TestFail; + } + int testFile() { int fd = open("/tmp", O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR); @@ -135,7 +168,11 @@ protected: int run() override { - int ret = testFile(); + int ret = testEnvLevels(); + if (ret != TestPass) + return TestFail; + + ret = testFile(); if (ret != TestPass) return TestFail; diff --git a/test/log/meson.build b/test/log/meson.build index 2298ff84e..80b0697a0 100644 --- a/test/log/meson.build +++ b/test/log/meson.build @@ -1,7 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 log_test = [ - {'name': 'log_api', 'sources': ['log_api.cpp']}, + {'name': 'log_api', 'sources': ['log_api.cpp'], 'should_fail': true}, {'name': 'log_process', 'sources': ['log_process.cpp']}, ] @@ -11,5 +11,6 @@ foreach test : log_test link_with : test_libraries, include_directories : test_includes_internal) - test(test['name'], exe, suite : 'log') + test(test['name'], exe, suite : 'log', + should_fail : test.get('should_fail', false)) endforeach From 0dfb052fbdd641f1127d8db00b8656f5584d68d9 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Fri, 6 Jun 2025 10:21:35 +0200 Subject: [PATCH 32/64] libcamera: base: Fix log level parsing when multiple categories are listed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For a list of log levels like LIBCAMERA_LOG_LEVELS="CatA:0,CatB:1" only the severity of the last entry is correctly parsed. Due to the change of level to a string_view in 24c2caa1c1b3 ("libcamera: base: log: Use `std::string_view` to avoid some copies") the level is no longer necessarily null terminated as it is a view on the original data. Replace the check for a terminating null by a check for the end position to fix the issue. Fixes: 24c2caa1c1b3 ("libcamera: base: log: Use `std::string_view` to avoid some copies") Signed-off-by: Stefan Klug Reviewed-by: Barnabás Pőcze Reviewed-by: Laurent Pinchart --- src/libcamera/base/log.cpp | 5 +++-- test/log/meson.build | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcamera/base/log.cpp b/src/libcamera/base/log.cpp index 8bf3e1daa..6a8e2a3eb 100644 --- a/src/libcamera/base/log.cpp +++ b/src/libcamera/base/log.cpp @@ -690,8 +690,9 @@ LogSeverity Logger::parseLogLevel(std::string_view level) unsigned int severity = LogInvalid; if (std::isdigit(level[0])) { - auto [end, ec] = std::from_chars(level.data(), level.data() + level.size(), severity); - if (ec != std::errc() || *end != '\0' || severity > LogFatal) + const char *levelEnd = level.data() + level.size(); + auto [end, ec] = std::from_chars(level.data(), levelEnd, severity); + if (ec != std::errc() || end != levelEnd || severity > LogFatal) severity = LogInvalid; } else { for (unsigned int i = 0; i < std::size(names); ++i) { diff --git a/test/log/meson.build b/test/log/meson.build index 80b0697a0..f413c3898 100644 --- a/test/log/meson.build +++ b/test/log/meson.build @@ -1,7 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 log_test = [ - {'name': 'log_api', 'sources': ['log_api.cpp'], 'should_fail': true}, + {'name': 'log_api', 'sources': ['log_api.cpp']}, {'name': 'log_process', 'sources': ['log_process.cpp']}, ] From 5f4d2ac935ee40e901bacc844869c96a20c296c7 Mon Sep 17 00:00:00 2001 From: Kieran Bingham Date: Thu, 19 Jun 2025 12:16:14 +0100 Subject: [PATCH 33/64] libcamera: controls: Revert incorrect SPDX removal In commit 6a09deaf7d4a ("controls: Add FrameWallClock control") the existing SPDX was accidentally removed, likely from a rebase operation at some point. Unfortunately as this patch had already collected Reviewed-by tags, the surruptious removal wasn't noticed until after it was merged. Re-insert the existing SPDX and copyright banner as the header to the control definitions file. Signed-off-by: Kieran Bingham Reviewed-by: Naushir Patuck Reviewed-by: Laurent Pinchart Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_core.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index 028919ef3..46f51436c 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -1,4 +1,6 @@ - +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# Copyright (C) 2019, Google Inc. # %YAML 1.1 --- From a29c53f6a646ed9644a94b2face62d078d8290be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Wed, 22 Jan 2025 13:09:11 +0100 Subject: [PATCH 34/64] meson: Use libyaml wrap file from wrapdb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the libyaml wrap file from the meson wrapdb instead of creating the wrap file manually and using the cmake module. This provides better integration with meson, such as the `force_fallback_for` built-in option. This is also needed because the upstream CMakeLists.txt is out of date, failing with a sufficiently new cmake version: CMake Error at CMakeLists.txt:2 (cmake_minimum_required): Compatibility with CMake < 3.5 has been removed from CMake. The above is nonetheless addressed by https://github.com/yaml/libyaml/pull/314, but the project seems a bit inactive at the moment. The wrap file was added using `meson wrap install libyaml`, and it can be updated using `meson wrap update libyaml`. `default_library=static` is used to match the behaviour of the previously used cmake build. `werror=false` needs to be set because libyaml does not compile without warnings, and that would abort the build process otherwise. Signed-off-by: Barnabás Pőcze Reviewed-by: Laurent Pinchart Reviewed-by: Kieran Bingham --- src/libcamera/meson.build | 16 ++++------------ subprojects/libyaml.wrap | 16 +++++++++++----- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 28a3b0f36..de1eb99b2 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -84,7 +84,10 @@ if not cc.has_function('dlopen') libdl = cc.find_library('dl') endif libudev = dependency('libudev', required : get_option('udev')) -libyaml = dependency('yaml-0.1', required : false) +libyaml = dependency('yaml-0.1', default_options : [ + 'default_library=static', + 'werror=false', +]) # Use one of gnutls or libcrypto (provided by OpenSSL), trying gnutls first. libcrypto = dependency('gnutls', required : false) @@ -120,17 +123,6 @@ if libudev.found() ]) endif -# Fallback to a subproject if libyaml isn't found, as it's not packaged in AOSP. -if not libyaml.found() - cmake = import('cmake') - - libyaml_vars = cmake.subproject_options() - libyaml_vars.add_cmake_defines({'CMAKE_POSITION_INDEPENDENT_CODE': 'ON'}) - libyaml_vars.append_compile_args('c', '-Wno-unused-value') - libyaml_wrap = cmake.subproject('libyaml', options : libyaml_vars) - libyaml = libyaml_wrap.dependency('yaml') -endif - control_sources = [] controls_mode_files = { diff --git a/subprojects/libyaml.wrap b/subprojects/libyaml.wrap index 392416c61..44ac0ff8e 100644 --- a/subprojects/libyaml.wrap +++ b/subprojects/libyaml.wrap @@ -1,7 +1,13 @@ # SPDX-License-Identifier: CC0-1.0 -[wrap-git] -directory = libyaml -url = https://github.com/yaml/libyaml -# tags/0.2.5 -revision = 2c891fc7a770e8ba2fec34fc6b545c672beb37e6 +[wrap-file] +directory = yaml-0.2.5 +source_url = https://pyyaml.org/download/libyaml/yaml-0.2.5.tar.gz +source_filename = yaml-0.2.5.tar.gz +source_hash = c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4 +patch_filename = libyaml_0.2.5-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/libyaml_0.2.5-1/get_patch +patch_hash = bf2e9b922be00b6b00c5fce29d9fb8dc83f0431c77239f3b73e8b254d3f3f5b5 + +[provide] +yaml-0.1 = yaml_dep From e9528306f257416c94bc882b89548e20d0aeebc3 Mon Sep 17 00:00:00 2001 From: Umang Jain Date: Thu, 26 Jun 2025 10:54:41 +0530 Subject: [PATCH 35/64] camera_sensor: Expand on computeTransform() documentation The description for computeTransform() when the desired orientation cannot be achieved, can be expanded a further bit, to clearly report that orientation will be adjusted to native camera sensor mounting rotation. Signed-off-by: Umang Jain Reviewed-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/libcamera/sensor/camera_sensor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libcamera/sensor/camera_sensor.cpp b/src/libcamera/sensor/camera_sensor.cpp index d19b5e2e7..4f2fd2690 100644 --- a/src/libcamera/sensor/camera_sensor.cpp +++ b/src/libcamera/sensor/camera_sensor.cpp @@ -302,8 +302,9 @@ int CameraSensor::setEmbeddedDataEnabled(bool enable) * camera sensor, likely at configure() time. * * If the requested \a orientation cannot be obtained, the \a orientation - * parameter is adjusted to report the current image orientation and - * Transform::Identity is returned. + * parameter is adjusted to report the native image orientation (i.e. resulting + * from the physical mounting rotation of the camera sensor, without any + * transformation) and Transform::Identity is returned. * * If the requested \a orientation can be obtained, the function computes a * Transform and does not adjust \a orientation. From 35ee8752b745024d5f00c34c3c17072e57b93d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Thu, 5 Jun 2025 14:18:38 +0200 Subject: [PATCH 36/64] libcamera: pipeline: uvcvideo: Silently ignore `AeEnable` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `AeEnable` control is handled in `Camera::queueRequest()` but it still reaches the pipeline handler because a single element cannot be removed from a `ControlList`. So ignore it silently. Fixes: ffcecda4d5b9b5 ("libcamera: pipeline: uvcvideo: Report new AeEnable control as available") Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart Reviewed-by: Paul Elder --- src/libcamera/pipeline/uvcvideo/uvcvideo.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp index 58aa0eb4c..d52b88042 100644 --- a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp +++ b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp @@ -331,6 +331,8 @@ int PipelineHandlerUVC::processControl(const UVCCameraData *data, ControlList *c cid = V4L2_CID_GAIN; else if (id == controls::Gamma) cid = V4L2_CID_GAMMA; + else if (id == controls::AeEnable) + return 0; /* Handled in `Camera::queueRequest()`. */ else return -EINVAL; From 5f94209b1d2149a3ceea3692bae49256038c2bb3 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 27 Jun 2025 10:46:26 +0100 Subject: [PATCH 37/64] pipeline: rpi: Fix for enumerating the media graphs When there are multiple entities between the sensor and CFE device (e.g. a serialiser and deserialiser or multiple mux devices), the media graph enumeration would work incorrectly and report that the frontend entity was not found. This is because the found flag was stored locally in a boolean and got lost in the recursion. Fix this by explicitly tracking and returning the frontend found flag through the return value of enumerateVideoDevices(). This ensures the flag does not get lost through nested recursion. This flag can also be used to fail a camera registration if the frontend is not found. Signed-off-by: Naushir Patuck Reviewed-by: Umang Jain Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- .../pipeline/rpi/common/pipeline_base.cpp | 22 +++++++++++-------- .../pipeline/rpi/common/pipeline_base.h | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index e14d3b913..eafe94427 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -806,7 +806,8 @@ int PipelineHandlerBase::registerCamera(std::unique_ptr &camera * chain. There may be a cascade of devices in this chain! */ MediaLink *link = sensorEntity->getPadByIndex(0)->links()[0]; - data->enumerateVideoDevices(link, frontendName); + if (!data->enumerateVideoDevices(link, frontendName)) + return -EINVAL; ipa::RPi::InitResult result; if (data->loadIPA(&result)) { @@ -1018,16 +1019,20 @@ void CameraData::freeBuffers() * | Sensor2 | | Sensor3 | * +---------+ +---------+ */ -void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &frontend) +bool CameraData::enumerateVideoDevices(MediaLink *link, const std::string &frontend) { const MediaPad *sinkPad = link->sink(); const MediaEntity *entity = sinkPad->entity(); bool frontendFound = false; + /* Once we reach the Frontend entity, we are done. */ + if (link->sink()->entity()->name() == frontend) + return true; + /* We only deal with Video Mux and Bridge devices in cascade. */ if (entity->function() != MEDIA_ENT_F_VID_MUX && entity->function() != MEDIA_ENT_F_VID_IF_BRIDGE) - return; + return false; /* Find the source pad for this Video Mux or Bridge device. */ const MediaPad *sourcePad = nullptr; @@ -1039,7 +1044,7 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front * and this branch in the cascade. */ if (sourcePad) - return; + return false; sourcePad = pad; } @@ -1056,12 +1061,9 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front * other Video Mux and Bridge devices. */ for (MediaLink *l : sourcePad->links()) { - enumerateVideoDevices(l, frontend); - /* Once we reach the Frontend entity, we are done. */ - if (l->sink()->entity()->name() == frontend) { - frontendFound = true; + frontendFound = enumerateVideoDevices(l, frontend); + if (frontendFound) break; - } } /* This identifies the end of our entity enumeration recursion. */ @@ -1076,6 +1078,8 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front bridgeDevices_.clear(); } } + + return frontendFound; } int CameraData::loadPipelineConfiguration() diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 4c5743e04..4bce4ec4f 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -68,7 +68,7 @@ public: void freeBuffers(); virtual void platformFreeBuffers() = 0; - void enumerateVideoDevices(MediaLink *link, const std::string &frontend); + bool enumerateVideoDevices(MediaLink *link, const std::string &frontend); int loadPipelineConfiguration(); int loadIPA(ipa::RPi::InitResult *result); From 6b5cc1c92a33704e23bab2e5da699fa1d0678803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Thu, 22 May 2025 13:06:02 +0200 Subject: [PATCH 38/64] libcamera: pipeline: uvcvideo: Handle controls during startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Process the control list passed to `Camera::start()`, and set the V4L2 controls accordingly. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Umang Jain Reviewed-by: Paul Elder --- src/libcamera/pipeline/uvcvideo/uvcvideo.cpp | 27 +++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp index d52b88042..4b5816dfd 100644 --- a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp +++ b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp @@ -100,7 +100,7 @@ public: private: int processControl(const UVCCameraData *data, ControlList *controls, unsigned int id, const ControlValue &value); - int processControls(UVCCameraData *data, Request *request); + int processControls(UVCCameraData *data, const ControlList &reqControls); bool acquireDevice(Camera *camera) override; void releaseDevice(Camera *camera) override; @@ -287,7 +287,7 @@ int PipelineHandlerUVC::exportFrameBuffers(Camera *camera, Stream *stream, return data->video_->exportBuffers(count, buffers); } -int PipelineHandlerUVC::start(Camera *camera, [[maybe_unused]] const ControlList *controls) +int PipelineHandlerUVC::start(Camera *camera, const ControlList *controls) { UVCCameraData *data = cameraData(camera); unsigned int count = data->stream_.configuration().bufferCount; @@ -296,13 +296,22 @@ int PipelineHandlerUVC::start(Camera *camera, [[maybe_unused]] const ControlList if (ret < 0) return ret; - ret = data->video_->streamOn(); - if (ret < 0) { - data->video_->releaseBuffers(); - return ret; + if (controls) { + ret = processControls(data, *controls); + if (ret < 0) + goto err_release_buffers; } + ret = data->video_->streamOn(); + if (ret < 0) + goto err_release_buffers; + return 0; + +err_release_buffers: + data->video_->releaseBuffers(); + + return ret; } void PipelineHandlerUVC::stopDevice(Camera *camera) @@ -412,11 +421,11 @@ int PipelineHandlerUVC::processControl(const UVCCameraData *data, ControlList *c return 0; } -int PipelineHandlerUVC::processControls(UVCCameraData *data, Request *request) +int PipelineHandlerUVC::processControls(UVCCameraData *data, const ControlList &reqControls) { ControlList controls(data->video_->controls()); - for (const auto &[id, value] : request->controls()) + for (const auto &[id, value] : reqControls) processControl(data, &controls, id, value); for (const auto &ctrl : controls) @@ -444,7 +453,7 @@ int PipelineHandlerUVC::queueRequestDevice(Camera *camera, Request *request) return -ENOENT; } - int ret = processControls(data, request); + int ret = processControls(data, request->controls()); if (ret < 0) return ret; From 30114cadd8cb651231efcd5fe32f2c8a0a72b7b4 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:22 +0100 Subject: [PATCH 39/64] ipa: rpi: Defer initialising AF LensPosition ControlInfo and value This fixes two small bugs: We previously populated LensPosition's ControlInfo with hard-coded values, ignoring the tuning file. Now we query the AfAlgorithm to get limits (over all AF ranges) and default (for AfRangeNormal). We previously sent a default position to the lens driver, even when a user-specified starting position would follow. Defer doing this, to reduce unnecessary lens movement at startup (for some drivers). Bug: https://bugs.libcamera.org/show_bug.cgi?id=258 Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/common/ipa_base.cpp | 53 +++++++++++++++++---------- src/ipa/rpi/controller/af_algorithm.h | 8 +++- src/ipa/rpi/controller/rpi/af.cpp | 16 +++++++- src/ipa/rpi/controller/rpi/af.h | 4 +- 4 files changed, 57 insertions(+), 24 deletions(-) diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index 6565f5366..a5bdcbb58 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -233,25 +233,6 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa agcStatus.analogueGain = defaultAnalogueGain; applyAGC(&agcStatus, ctrls); - /* - * Set the lens to the default (typically hyperfocal) position - * on first start. - */ - if (lensPresent_) { - RPiController::AfAlgorithm *af = - dynamic_cast(controller_.getAlgorithm("af")); - - if (af) { - float defaultPos = - ipaAfControls.at(&controls::LensPosition).def().get(); - ControlList lensCtrl(lensCtrls_); - int32_t hwpos; - - af->setLensPosition(defaultPos, &hwpos); - lensCtrl.set(V4L2_CID_FOCUS_ABSOLUTE, hwpos); - result->lensControls = std::move(lensCtrl); - } - } } result->sensorControls = std::move(ctrls); @@ -281,8 +262,20 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa ctrlMap.merge(ControlInfoMap::Map(ipaColourControls)); /* Declare Autofocus controls, only if we have a controllable lens */ - if (lensPresent_) + if (lensPresent_) { ctrlMap.merge(ControlInfoMap::Map(ipaAfControls)); + RPiController::AfAlgorithm *af = + dynamic_cast(controller_.getAlgorithm("af")); + if (af) { + double min, max, dflt; + af->getLensLimits(min, max); + dflt = af->getDefaultLensPosition(); + ctrlMap[&controls::LensPosition] = + ControlInfo(static_cast(min), + static_cast(max), + static_cast(dflt)); + } + } result->controlInfo = ControlInfoMap(std::move(ctrlMap), controls::controls); @@ -320,6 +313,26 @@ void IpaBase::start(const ControlList &controls, StartResult *result) /* Make a note of this as it tells us the HDR status of the first few frames. */ hdrStatus_ = agcStatus.hdr; + /* + * AF: If no lens position was specified, drive lens to a default position. + * This had to be deferred (not initialised by a constructor) until here + * to ensure that exactly ONE starting position is sent to the lens driver. + * It should be the static API default, not dependent on AF range or mode. + */ + if (firstStart_ && lensPresent_) { + RPiController::AfAlgorithm *af = dynamic_cast( + controller_.getAlgorithm("af")); + if (af && !af->getLensPosition()) { + int32_t hwpos; + double pos = af->getDefaultLensPosition(); + if (af->setLensPosition(pos, &hwpos, true)) { + ControlList lensCtrls(lensCtrls_); + lensCtrls.set(V4L2_CID_FOCUS_ABSOLUTE, hwpos); + setLensControls.emit(lensCtrls); + } + } + } + /* * Initialise frame counts, and decide how many frames must be hidden or * "mistrusted", which depends on whether this is a startup from cold, diff --git a/src/ipa/rpi/controller/af_algorithm.h b/src/ipa/rpi/controller/af_algorithm.h index ad9b57545..382609f9b 100644 --- a/src/ipa/rpi/controller/af_algorithm.h +++ b/src/ipa/rpi/controller/af_algorithm.h @@ -33,6 +33,10 @@ public: * * getMode() is provided mainly for validating controls. * getLensPosition() is provided for populating DeviceStatus. + * + * getDefaultlensPosition() and getLensLimits() were added for + * populating ControlInfoMap. They return the static API limits + * which should be independent of the current range or mode. */ enum AfRange { AfRangeNormal = 0, @@ -66,7 +70,9 @@ public: } virtual void setMode(AfMode mode) = 0; virtual AfMode getMode() const = 0; - virtual bool setLensPosition(double dioptres, int32_t *hwpos) = 0; + virtual double getDefaultLensPosition() const = 0; + virtual void getLensLimits(double &min, double &max) const = 0; + virtual bool setLensPosition(double dioptres, int32_t *hwpos, bool force = false) = 0; virtual std::optional getLensPosition() const = 0; virtual void triggerScan() = 0; virtual void cancelScan() = 0; diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 2157eb94f..041cb51db 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -715,11 +715,23 @@ void Af::setWindows(libcamera::Span const &wins) invalidateWeights(); } -bool Af::setLensPosition(double dioptres, int *hwpos) +double Af::getDefaultLensPosition() const +{ + return cfg_.ranges[AfRangeNormal].focusDefault; +} + +void Af::getLensLimits(double &min, double &max) const +{ + /* Limits for manual focus are set by map, not by ranges */ + min = cfg_.map.domain().start; + max = cfg_.map.domain().end; +} + +bool Af::setLensPosition(double dioptres, int *hwpos, bool force) { bool changed = false; - if (mode_ == AfModeManual) { + if (mode_ == AfModeManual || force) { LOG(RPiAf, Debug) << "setLensPosition: " << dioptres; ftarget_ = cfg_.map.domain().clamp(dioptres); changed = !(initted_ && fsmooth_ == ftarget_); diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index 317a51f3e..d8ece1ab1 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -54,7 +54,9 @@ public: void setWindows(libcamera::Span const &wins) override; void setMode(AfMode mode) override; AfMode getMode() const override; - bool setLensPosition(double dioptres, int32_t *hwpos) override; + double getDefaultLensPosition() const override; + void getLensLimits(double &min, double &max) const override; + bool setLensPosition(double dioptres, int32_t *hwpos, bool force) override; std::optional getLensPosition() const override; void triggerScan() override; void cancelScan() override; From a283287fbf0c633591698bef6670729748296f02 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:23 +0100 Subject: [PATCH 40/64] ipa: rpi: controller: Improve findPeak() function in AF algorithm Improve quadratic peak fitting in findPeak(). The old approximation was good but only valid when points were equally spaced and the MAX was not at one end of the series. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 041cb51db..8df614ed7 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -436,15 +436,28 @@ double Af::findPeak(unsigned i) const { double f = scanData_[i].focus; - if (i > 0 && i + 1 < scanData_.size()) { - double dropLo = scanData_[i].contrast - scanData_[i - 1].contrast; - double dropHi = scanData_[i].contrast - scanData_[i + 1].contrast; - if (0.0 <= dropLo && dropLo < dropHi) { - double param = 0.3125 * (1.0 - dropLo / dropHi) * (1.6 - dropLo / dropHi); - f += param * (scanData_[i - 1].focus - f); - } else if (0.0 <= dropHi && dropHi < dropLo) { - double param = 0.3125 * (1.0 - dropHi / dropLo) * (1.6 - dropHi / dropLo); - f += param * (scanData_[i + 1].focus - f); + if (scanData_.size() >= 3) { + /* + * Given the sample with the highest contrast score and its two + * neighbours either side (or same side if at the end of a scan), + * solve for the best lens position by fitting a parabola. + * Adapted from awb.cpp: interpolateQaudaratic() + */ + + if (i == 0) + i++; + else if (i + 1 >= scanData_.size()) + i--; + + double abx = scanData_[i - 1].focus - scanData_[i].focus; + double aby = scanData_[i - 1].contrast - scanData_[i].contrast; + double cbx = scanData_[i + 1].focus - scanData_[i].focus; + double cby = scanData_[i + 1].contrast - scanData_[i].contrast; + double denom = 2.0 * (aby * cbx - cby * abx); + if (std::abs(denom) >= (1.0 / 64.0) && denom * abx > 0.0) { + f = (aby * cbx * cbx - cby * abx * abx) / denom; + f = std::clamp(f, std::min(abx, cbx), std::max(abx, cbx)); + f += scanData_[i].focus; } } From 0fa2b05a868665c27d74433b227d54d8f3af6c89 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:24 +0100 Subject: [PATCH 41/64] ipa: rpi: controller: AutoFocus weighting tweak In getPhase(), stop using different weights for sumWc and sumWcp. This should improve linearity e.g. in earlyTerminationByPhase(). Phases are slightly larger but confidence values slightly reduced. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 8df614ed7..5304f54ca 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -328,9 +328,8 @@ bool Af::getPhase(PdafRegions const ®ions, double &phase, double &conf) if (c >= cfg_.confThresh) { if (c > cfg_.confClip) c = cfg_.confClip; - c -= (cfg_.confThresh >> 2); + c -= (cfg_.confThresh >> 1); sumWc += w * c; - c -= (cfg_.confThresh >> 2); sumWcp += (int64_t)(w * c) * (int64_t)data.phase; } } From 429a5ab48fe8e5ba1179f179a02a9b57cf037a6c Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:25 +0100 Subject: [PATCH 42/64] ipa: rpi: controller: Autofocus CAF/PDAF stability tweak When in Continuous AF mode using PDAF, only move the lens when phase has had the same sign for at least 4 frames. This reduces lens wobble in e.g. noisy conditions. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 12 +++++++++++- src/ipa/rpi/controller/rpi/af.h | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 5304f54ca..3b2d6167d 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -181,9 +181,11 @@ Af::Af(Controller *controller) ftarget_(-1.0), fsmooth_(-1.0), prevContrast_(0.0), + prevPhase_(0.0), skipCount_(0), stepCount_(0), dropCount_(0), + sameSignCount_(0), scanMaxContrast_(0.0), scanMinContrast_(1.0e9), scanData_(), @@ -513,6 +515,13 @@ void Af::doAF(double contrast, double phase, double conf) return; } + /* Count frames for which PDAF phase has had same sign */ + if (phase * prevPhase_ <= 0.0) + sameSignCount_ = 0; + else + sameSignCount_++; + prevPhase_ = phase; + if (scanState_ == ScanState::Pdaf) { /* * Use PDAF closed-loop control whenever available, in both CAF @@ -522,7 +531,8 @@ void Af::doAF(double contrast, double phase, double conf) * scan only after a number of frames with low PDAF confidence. */ if (conf > (dropCount_ ? 1.0 : 0.25) * cfg_.confEpsilon) { - doPDAF(phase, conf); + if (mode_ == AfModeAuto || sameSignCount_ >= 3) + doPDAF(phase, conf); if (stepCount_ > 0) stepCount_--; else if (mode_ != AfModeContinuous) diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index d8ece1ab1..b06a3a16f 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -158,7 +158,9 @@ private: bool initted_; double ftarget_, fsmooth_; double prevContrast_; + double prevPhase_; unsigned skipCount_, stepCount_, dropCount_; + unsigned sameSignCount_; unsigned scanMaxIndex_; double scanMaxContrast_, scanMinContrast_; std::vector scanData_; From 3d44987bc606a84a9191b39900b0fd17444b2a6c Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:26 +0100 Subject: [PATCH 43/64] ipa: rpi: controller: AutoFocus tweak earlyTerminationByPhase() Increase threshold for ETBP, from "confEpsilon" to "confThresh". Correct sign test to take account of pdafGain sign (typically -ve). Reduce allowed extrapolation range, but relax the check in the case of Continuous AF, when we go back into the PDAF closed loop. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 3b2d6167d..ecc0fc417 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -411,7 +411,7 @@ void Af::doPDAF(double phase, double conf) bool Af::earlyTerminationByPhase(double phase) { if (scanData_.size() > 0 && - scanData_[scanData_.size() - 1].conf >= cfg_.confEpsilon) { + scanData_[scanData_.size() - 1].conf >= cfg_.confThresh) { double oldFocus = scanData_[scanData_.size() - 1].focus; double oldPhase = scanData_[scanData_.size() - 1].phase; @@ -420,11 +420,12 @@ bool Af::earlyTerminationByPhase(double phase) * Interpolate/extrapolate the lens position for zero phase. * Check that the extrapolation is well-conditioned. */ - if ((ftarget_ - oldFocus) * (phase - oldPhase) > 0.0) { + if ((ftarget_ - oldFocus) * (phase - oldPhase) * cfg_.speeds[speed_].pdafGain < 0.0) { double param = phase / (phase - oldPhase); - if (-3.0 <= param && param <= 3.5) { - ftarget_ += param * (oldFocus - ftarget_); + if ((-2.5 <= param || mode_ == AfModeContinuous) && param <= 3.0) { LOG(RPiAf, Debug) << "ETBP: param=" << param; + param = std::max(param, -2.5); + ftarget_ += param * (oldFocus - ftarget_); return true; } } @@ -562,7 +563,7 @@ void Af::doAF(double contrast, double phase, double conf) else scanState_ = ScanState::Idle; scanData_.clear(); - } else if (conf >= cfg_.confEpsilon && earlyTerminationByPhase(phase)) { + } else if (conf >= cfg_.confThresh && earlyTerminationByPhase(phase)) { scanState_ = ScanState::Settle; stepCount_ = (mode_ == AfModeContinuous) ? 0 : cfg_.speeds[speed_].stepFrames; From 686f88707caa51895057f8f22a611af9f8af2c16 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:27 +0100 Subject: [PATCH 44/64] ipa: rpi: controller: Autofocus to use AWB statistics; re-trigger Analyse AWB statistics: used both for scene change detection and to detect IR lighting (when a flag is set in the tuning file). Option to suppress PDAF altogether when IR lighting is detected. Rather than being based solely on PDAF "dropout", allow a scan to be (re-)triggered whenever the scene changes and then stabilizes, based on contrast and average RGB statistics within the AF window. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 137 +++++++++++++++++++++++++++--- src/ipa/rpi/controller/rpi/af.h | 37 +++++--- 2 files changed, 149 insertions(+), 25 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index ecc0fc417..4396420a0 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -46,6 +46,8 @@ Af::SpeedDependentParams::SpeedDependentParams() : stepCoarse(1.0), stepFine(0.25), contrastRatio(0.75), + retriggerRatio(0.75), + retriggerDelay(10), pdafGain(-0.02), pdafSquelch(0.125), maxSlew(2.0), @@ -60,6 +62,7 @@ Af::CfgParams::CfgParams() confThresh(16), confClip(512), skipFrames(5), + checkForIR(false), map() { } @@ -87,6 +90,8 @@ void Af::SpeedDependentParams::read(const libcamera::YamlObject ¶ms) readNumber(stepCoarse, params, "step_coarse"); readNumber(stepFine, params, "step_fine"); readNumber(contrastRatio, params, "contrast_ratio"); + readNumber(retriggerRatio, params, "retrigger_ratio"); + readNumber(retriggerDelay, params, "retrigger_delay"); readNumber(pdafGain, params, "pdaf_gain"); readNumber(pdafSquelch, params, "pdaf_squelch"); readNumber(maxSlew, params, "max_slew"); @@ -137,6 +142,7 @@ int Af::CfgParams::read(const libcamera::YamlObject ¶ms) readNumber(confThresh, params, "conf_thresh"); readNumber(confClip, params, "conf_clip"); readNumber(skipFrames, params, "skip_frames"); + readNumber(checkForIR, params, "check_for_ir"); if (params.contains("map")) map = params["map"].get(ipa::Pwl{}); @@ -176,29 +182,37 @@ Af::Af(Controller *controller) useWindows_(false), phaseWeights_(), contrastWeights_(), + awbWeights_(), scanState_(ScanState::Idle), initted_(false), + irFlag_(false), ftarget_(-1.0), fsmooth_(-1.0), prevContrast_(0.0), + oldSceneContrast_(0.0), + prevAverage_{ 0.0, 0.0, 0.0 }, + oldSceneAverage_{ 0.0, 0.0, 0.0 }, prevPhase_(0.0), skipCount_(0), stepCount_(0), dropCount_(0), sameSignCount_(0), + sceneChangeCount_(0), scanMaxContrast_(0.0), scanMinContrast_(1.0e9), scanData_(), reportState_(AfState::Idle) { /* - * Reserve space for data, to reduce memory fragmentation. It's too early - * to query the size of the PDAF (from camera) and Contrast (from ISP) - * statistics, but these are plausible upper bounds. + * Reserve space for data structures, to reduce memory fragmentation. + * It's too early to query the size of the PDAF sensor data, so guess. */ + windows_.reserve(1); phaseWeights_.w.reserve(16 * 12); contrastWeights_.w.reserve(getHardwareConfig().focusRegions.width * getHardwareConfig().focusRegions.height); + contrastWeights_.w.reserve(getHardwareConfig().awbRegions.width * + getHardwareConfig().awbRegions.height); scanData_.reserve(32); } @@ -309,6 +323,7 @@ void Af::invalidateWeights() { phaseWeights_.sum = 0; contrastWeights_.sum = 0; + awbWeights_.sum = 0; } bool Af::getPhase(PdafRegions const ®ions, double &phase, double &conf) @@ -365,6 +380,54 @@ double Af::getContrast(const FocusRegions &focusStats) return (contrastWeights_.sum > 0) ? ((double)sumWc / (double)contrastWeights_.sum) : 0.0; } +/* + * Get the average R, G, B values in AF window[s] (from AWB statistics). + * Optionally, check if all of {R,G,B} are within 4:5 of each other + * across more than 50% of the counted area and within the AF window: + * for an RGB sensor this strongly suggests that IR lighting is in use. + */ + +bool Af::getAverageAndTestIr(const RgbyRegions &awbStats, double rgb[3]) +{ + libcamera::Size size = awbStats.size(); + if (size.height != awbWeights_.rows || + size.width != awbWeights_.cols || awbWeights_.sum == 0) { + LOG(RPiAf, Debug) << "Recompute RGB weights " << size.width << 'x' << size.height; + computeWeights(&awbWeights_, size.height, size.width); + } + + uint64_t sr = 0, sg = 0, sb = 0, sw = 1; + uint64_t greyCount = 0, allCount = 0; + for (unsigned i = 0; i < awbStats.numRegions(); ++i) { + uint64_t r = awbStats.get(i).val.rSum; + uint64_t g = awbStats.get(i).val.gSum; + uint64_t b = awbStats.get(i).val.bSum; + uint64_t w = awbWeights_.w[i]; + if (w) { + sw += w; + sr += w * r; + sg += w * g; + sb += w * b; + } + if (cfg_.checkForIR) { + if (4 * r < 5 * b && 4 * b < 5 * r && + 4 * r < 5 * g && 4 * g < 5 * r && + 4 * b < 5 * g && 4 * g < 5 * b) + greyCount += awbStats.get(i).counted; + allCount += awbStats.get(i).counted; + } + } + + rgb[0] = sr / (double)sw; + rgb[1] = sg / (double)sw; + rgb[2] = sb / (double)sw; + + return (cfg_.checkForIR && 2 * greyCount > allCount && + 4 * sr < 5 * sb && 4 * sb < 5 * sr && + 4 * sr < 5 * sg && 4 * sg < 5 * sr && + 4 * sb < 5 * sg && 4 * sg < 5 * sb); +} + void Af::doPDAF(double phase, double conf) { /* Apply loop gain */ @@ -473,6 +536,8 @@ void Af::doScan(double contrast, double phase, double conf) if (scanData_.empty() || contrast > scanMaxContrast_) { scanMaxContrast_ = contrast; scanMaxIndex_ = scanData_.size(); + if (scanState_ != ScanState::Fine) + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); } if (contrast < scanMinContrast_) scanMinContrast_ = contrast; @@ -523,27 +588,63 @@ void Af::doAF(double contrast, double phase, double conf) sameSignCount_++; prevPhase_ = phase; + if (mode_ == AfModeManual) + return; /* nothing to do */ + if (scanState_ == ScanState::Pdaf) { /* * Use PDAF closed-loop control whenever available, in both CAF * mode and (for a limited number of iterations) when triggered. - * If PDAF fails (due to poor contrast, noise or large defocus), - * fall back to a CDAF-based scan. To avoid "nuisance" scans, - * scan only after a number of frames with low PDAF confidence. + * If PDAF fails (due to poor contrast, noise or large defocus) + * for at least dropoutFrames, fall back to a CDAF-based scan + * immediately (in triggered-auto) or on scene change (in CAF). */ - if (conf > (dropCount_ ? 1.0 : 0.25) * cfg_.confEpsilon) { + if (conf >= cfg_.confEpsilon) { if (mode_ == AfModeAuto || sameSignCount_ >= 3) doPDAF(phase, conf); if (stepCount_ > 0) stepCount_--; else if (mode_ != AfModeContinuous) scanState_ = ScanState::Idle; + oldSceneContrast_ = contrast; + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); + sceneChangeCount_ = 0; dropCount_ = 0; - } else if (++dropCount_ == cfg_.speeds[speed_].dropoutFrames) + return; + } else { + dropCount_++; + if (dropCount_ < cfg_.speeds[speed_].dropoutFrames) + return; + if (mode_ != AfModeContinuous) { + startProgrammedScan(); + return; + } + /* else fall through to waiting for a scene change */ + } + } + if (scanState_ < ScanState::Coarse && mode_ == AfModeContinuous) { + /* + * In CAF mode, not in a scan, and PDAF is unavailable. + * Wait for a scene change, followed by stability. + */ + if (contrast + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneContrast_ || + oldSceneContrast_ + 1.0 < cfg_.speeds[speed_].retriggerRatio * contrast || + prevAverage_[0] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[0] || + oldSceneAverage_[0] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[0] || + prevAverage_[1] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[1] || + oldSceneAverage_[1] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[1] || + prevAverage_[2] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[2] || + oldSceneAverage_[2] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[2]) { + oldSceneContrast_ = contrast; + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); + sceneChangeCount_ = 1; + } else if (sceneChangeCount_) + sceneChangeCount_++; + if (sceneChangeCount_ >= cfg_.speeds[speed_].retriggerDelay) startProgrammedScan(); } else if (scanState_ >= ScanState::Coarse && fsmooth_ == ftarget_) { /* - * Scanning sequence. This means PDAF has become unavailable. + * CDAF-based scanning sequence. * Allow a delay between steps for CDAF FoM statistics to be * updated, and a "settling time" at the end of the sequence. * [A coarse or fine scan can be abandoned if two PDAF samples @@ -562,11 +663,14 @@ void Af::doAF(double contrast, double phase, double conf) scanState_ = ScanState::Pdaf; else scanState_ = ScanState::Idle; + dropCount_ = 0; + sceneChangeCount_ = 0; + oldSceneContrast_ = std::max(scanMaxContrast_, prevContrast_); scanData_.clear(); } else if (conf >= cfg_.confThresh && earlyTerminationByPhase(phase)) { + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); scanState_ = ScanState::Settle; - stepCount_ = (mode_ == AfModeContinuous) ? 0 - : cfg_.speeds[speed_].stepFrames; + stepCount_ = (mode_ == AfModeContinuous) ? 0 : cfg_.speeds[speed_].stepFrames; } else doScan(contrast, phase, conf); } @@ -596,7 +700,8 @@ void Af::updateLensPosition() void Af::startAF() { /* Use PDAF if the tuning file allows it; else CDAF. */ - if (cfg_.speeds[speed_].dropoutFrames > 0 && + if (cfg_.speeds[speed_].pdafGain != 0.0 && + cfg_.speeds[speed_].dropoutFrames > 0 && (mode_ == AfModeContinuous || cfg_.speeds[speed_].pdafFrames > 0)) { if (!initted_) { ftarget_ = cfg_.ranges[range_].focusDefault; @@ -606,6 +711,8 @@ void Af::startAF() scanState_ = ScanState::Pdaf; scanData_.clear(); dropCount_ = 0; + oldSceneContrast_ = 0.0; + sceneChangeCount_ = 0; reportState_ = AfState::Scanning; } else startProgrammedScan(); @@ -656,7 +763,7 @@ void Af::prepare(Metadata *imageMetadata) uint32_t oldSt = stepCount_; if (imageMetadata->get("pdaf.regions", regions) == 0) getPhase(regions, phase, conf); - doAF(prevContrast_, phase, conf); + doAF(prevContrast_, phase, irFlag_ ? 0 : conf); updateLensPosition(); LOG(RPiAf, Debug) << std::fixed << std::setprecision(2) << static_cast(reportState_) @@ -666,7 +773,8 @@ void Af::prepare(Metadata *imageMetadata) << " ft" << oldFt << "->" << ftarget_ << " fs" << oldFs << "->" << fsmooth_ << " cont=" << (int)prevContrast_ - << " phase=" << (int)phase << " conf=" << (int)conf; + << " phase=" << (int)phase << " conf=" << (int)conf + << (irFlag_ ? " IR" : ""); } /* Report status and produce new lens setting */ @@ -690,6 +798,7 @@ void Af::process(StatisticsPtr &stats, [[maybe_unused]] Metadata *imageMetadata) { (void)imageMetadata; prevContrast_ = getContrast(stats->focusRegions); + irFlag_ = getAverageAndTestIr(stats->awbRegions, prevAverage_); } /* Controls */ diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index b06a3a16f..e1700f998 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -15,20 +15,28 @@ /* * This algorithm implements a hybrid of CDAF and PDAF, favouring PDAF. * - * Whenever PDAF is available, it is used in a continuous feedback loop. - * When triggered in auto mode, we simply enable AF for a limited number - * of frames (it may terminate early if the delta becomes small enough). + * Whenever PDAF is available (and reports sufficiently high confidence), + * it is used for continuous feedback control of the lens position. When + * triggered in Auto mode, we enable the loop for a limited number of frames + * (it may terminate sooner if the phase becomes small). In CAF mode, the + * PDAF loop runs continuously. Very small lens movements are suppressed. * * When PDAF confidence is low (due e.g. to low contrast or extreme defocus) * or PDAF data are absent, fall back to CDAF with a programmed scan pattern. - * A coarse and fine scan are performed, using ISP's CDAF focus FoM to - * estimate the lens position with peak contrast. This is slower due to - * extra latency in the ISP, and requires a settling time between steps. + * A coarse and fine scan are performed, using the ISP's CDAF contrast FoM + * to estimate the lens position with peak contrast. (This is slower due to + * extra latency in the ISP, and requires a settling time between steps.) + * The scan may terminate early if PDAF recovers and allows the zero-phase + * lens position to be interpolated. * - * Some hysteresis is applied to the switch between PDAF and CDAF, to avoid - * "nuisance" scans. During each interval where PDAF is not working, only - * ONE scan will be performed; CAF cannot track objects using CDAF alone. + * In CAF mode, the fallback to a CDAF scan is triggered when PDAF fails to + * report high confidence and a configurable number of frames have elapsed + * since the last image change since either PDAF was working or a previous + * scan found peak contrast. Image changes are detected using both contrast + * and AWB statistics (within the AF window[s]). * + * IR lighting can interfere with the correct operation of PDAF, so we + * optionally try to detect it (from AWB statistics). */ namespace RPiController { @@ -85,6 +93,8 @@ private: double stepCoarse; /* used for scans */ double stepFine; /* used for scans */ double contrastRatio; /* used for scan termination and reporting */ + double retriggerRatio; /* contrast and RGB ratio for re-triggering */ + uint32_t retriggerDelay; /* frames of stability before re-triggering */ double pdafGain; /* coefficient for PDAF feedback loop */ double pdafSquelch; /* PDAF stability parameter (device-specific) */ double maxSlew; /* limit for lens movement per frame */ @@ -103,6 +113,7 @@ private: uint32_t confThresh; /* PDAF confidence cell min (sensor-specific) */ uint32_t confClip; /* PDAF confidence cell max (sensor-specific) */ uint32_t skipFrames; /* frames to skip at start or modeswitch */ + bool checkForIR; /* Set this if PDAF is unreliable in IR light */ libcamera::ipa::Pwl map; /* converts dioptres -> lens driver position */ CfgParams(); @@ -131,6 +142,7 @@ private: void invalidateWeights(); bool getPhase(PdafRegions const ®ions, double &phase, double &conf); double getContrast(const FocusRegions &focusStats); + bool getAverageAndTestIr(const RgbyRegions &awbStats, double rgb[3]); void doPDAF(double phase, double conf); bool earlyTerminationByPhase(double phase); double findPeak(unsigned index) const; @@ -152,15 +164,18 @@ private: bool useWindows_; RegionWeights phaseWeights_; RegionWeights contrastWeights_; + RegionWeights awbWeights_; /* Working state. */ ScanState scanState_; - bool initted_; + bool initted_, irFlag_; double ftarget_, fsmooth_; - double prevContrast_; + double prevContrast_, oldSceneContrast_; + double prevAverage_[3], oldSceneAverage_[3]; double prevPhase_; unsigned skipCount_, stepCount_, dropCount_; unsigned sameSignCount_; + unsigned sceneChangeCount_; unsigned scanMaxIndex_; double scanMaxContrast_, scanMinContrast_; std::vector scanData_; From ea5f451c5660621cf1787e25c73983cec244d782 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:28 +0100 Subject: [PATCH 45/64] ipa: rpi: controller: AutoFocus bidirectional scanning To reduce unnecessary lens movements, allow the CDAF-based search procedure to start from either end of the range; or if not near an end, from the current lens position. This sometimes requires a second coarse scan, if the first one started in the middle and did not find peak contrast. Shorten the fine scan from 5 steps to 3 steps; allow fine scan to be omitted altogether when "step_fine": 0 in the tuning file. Move updateLensPosition() out of startProgrammedScan() to avoid calling it more than once per iteration. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 97 ++++++++++++++++++++----------- src/ipa/rpi/controller/rpi/af.h | 9 +-- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 4396420a0..eaaca4bc9 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -200,6 +200,7 @@ Af::Af(Controller *controller) sceneChangeCount_(0), scanMaxContrast_(0.0), scanMinContrast_(1.0e9), + scanStep_(0.0), scanData_(), reportState_(AfState::Idle) { @@ -251,13 +252,14 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met << statsRegion_.height; invalidateWeights(); - if (scanState_ >= ScanState::Coarse && scanState_ < ScanState::Settle) { + if (scanState_ >= ScanState::Coarse1 && scanState_ < ScanState::Settle) { /* * If a scan was in progress, re-start it, as CDAF statistics * may have changed. Though if the application is just about * to take a still picture, this will not help... */ startProgrammedScan(); + updateLensPosition(); } skipCount_ = cfg_.skipFrames; } @@ -543,31 +545,42 @@ void Af::doScan(double contrast, double phase, double conf) scanMinContrast_ = contrast; scanData_.emplace_back(ScanRecord{ ftarget_, contrast, phase, conf }); - if (scanState_ == ScanState::Coarse) { - if (ftarget_ >= cfg_.ranges[range_].focusMax || - contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { - /* - * Finished course scan, or termination based on contrast. - * Jump to just after max contrast and start fine scan. - */ - ftarget_ = std::min(ftarget_, findPeak(scanMaxIndex_) + - 2.0 * cfg_.speeds[speed_].stepFine); - scanState_ = ScanState::Fine; - scanData_.clear(); - } else - ftarget_ += cfg_.speeds[speed_].stepCoarse; - } else { /* ScanState::Fine */ - if (ftarget_ <= cfg_.ranges[range_].focusMin || scanData_.size() >= 5 || - contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { - /* - * Finished fine scan, or termination based on contrast. - * Use quadratic peak-finding to find best contrast position. - */ - ftarget_ = findPeak(scanMaxIndex_); + if ((scanStep_ >= 0.0 && ftarget_ >= cfg_.ranges[range_].focusMax) || + (scanStep_ <= 0.0 && ftarget_ <= cfg_.ranges[range_].focusMin) || + (scanState_ == ScanState::Fine && scanData_.size() >= 3) || + contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + double pk = findPeak(scanMaxIndex_); + /* + * Finished a scan, by hitting a limit or due to constrast dropping off. + * If this is a first coarse scan and we didn't bracket the peak, reverse! + * If this is a fine scan, or no fine step was defined, we've finished. + * Otherwise, start fine scan in opposite direction. + */ + if (scanState_ == ScanState::Coarse1 && + scanData_[0].contrast >= cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + scanStep_ = -scanStep_; + scanState_ = ScanState::Coarse2; + } else if (scanState_ == ScanState::Fine || cfg_.speeds[speed_].stepFine <= 0.0) { + ftarget_ = pk; scanState_ = ScanState::Settle; - } else - ftarget_ -= cfg_.speeds[speed_].stepFine; - } + } else if (scanState_ == ScanState::Coarse1 && + scanData_[0].contrast >= cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + scanStep_ = -scanStep_; + scanState_ = ScanState::Coarse2; + } else if (scanStep_ >= 0.0) { + ftarget_ = std::min(pk + cfg_.speeds[speed_].stepFine, + cfg_.ranges[range_].focusMax); + scanStep_ = -cfg_.speeds[speed_].stepFine; + scanState_ = ScanState::Fine; + } else { + ftarget_ = std::max(pk - cfg_.speeds[speed_].stepFine, + cfg_.ranges[range_].focusMin); + scanStep_ = cfg_.speeds[speed_].stepFine; + scanState_ = ScanState::Fine; + } + scanData_.clear(); + } else + ftarget_ += scanStep_; stepCount_ = (ftarget_ == fsmooth_) ? 0 : cfg_.speeds[speed_].stepFrames; } @@ -622,7 +635,7 @@ void Af::doAF(double contrast, double phase, double conf) /* else fall through to waiting for a scene change */ } } - if (scanState_ < ScanState::Coarse && mode_ == AfModeContinuous) { + if (scanState_ < ScanState::Coarse1 && mode_ == AfModeContinuous) { /* * In CAF mode, not in a scan, and PDAF is unavailable. * Wait for a scene change, followed by stability. @@ -642,7 +655,7 @@ void Af::doAF(double contrast, double phase, double conf) sceneChangeCount_++; if (sceneChangeCount_ >= cfg_.speeds[speed_].retriggerDelay) startProgrammedScan(); - } else if (scanState_ >= ScanState::Coarse && fsmooth_ == ftarget_) { + } else if (scanState_ >= ScanState::Coarse1 && fsmooth_ == ftarget_) { /* * CDAF-based scanning sequence. * Allow a delay between steps for CDAF FoM statistics to be @@ -714,15 +727,27 @@ void Af::startAF() oldSceneContrast_ = 0.0; sceneChangeCount_ = 0; reportState_ = AfState::Scanning; - } else + } else { startProgrammedScan(); + updateLensPosition(); + } } void Af::startProgrammedScan() { - ftarget_ = cfg_.ranges[range_].focusMin; - updateLensPosition(); - scanState_ = ScanState::Coarse; + if (!initted_ || mode_ != AfModeContinuous || + fsmooth_ <= cfg_.ranges[range_].focusMin + 2.0 * cfg_.speeds[speed_].stepCoarse) { + ftarget_ = cfg_.ranges[range_].focusMin; + scanStep_ = cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse2; + } else if (fsmooth_ >= cfg_.ranges[range_].focusMax - 2.0 * cfg_.speeds[speed_].stepCoarse) { + ftarget_ = cfg_.ranges[range_].focusMax; + scanStep_ = -cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse2; + } else { + scanStep_ = -cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse1; + } scanMaxContrast_ = 0.0; scanMinContrast_ = 1.0e9; scanMaxIndex_ = 0; @@ -785,7 +810,9 @@ void Af::prepare(Metadata *imageMetadata) else status.pauseState = AfPauseState::Running; - if (mode_ == AfModeAuto && scanState_ != ScanState::Idle) + if (scanState_ == ScanState::Idle) + status.state = AfState::Idle; + else if (mode_ == AfModeAuto) status.state = AfState::Scanning; else status.state = reportState_; @@ -907,7 +934,7 @@ void Af::setMode(AfAlgorithm::AfMode mode) pauseFlag_ = false; if (mode == AfModeContinuous) scanState_ = ScanState::Trigger; - else if (mode != AfModeAuto || scanState_ < ScanState::Coarse) + else if (mode != AfModeAuto || scanState_ < ScanState::Coarse1) goIdle(); } } @@ -923,11 +950,11 @@ void Af::pause(AfAlgorithm::AfPause pause) if (mode_ == AfModeContinuous) { if (pause == AfPauseResume && pauseFlag_) { pauseFlag_ = false; - if (scanState_ < ScanState::Coarse) + if (scanState_ < ScanState::Coarse1) scanState_ = ScanState::Trigger; } else if (pause != AfPauseResume && !pauseFlag_) { pauseFlag_ = true; - if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse) + if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse1) goIdle(); } } diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index e1700f998..d35a39d12 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -75,7 +75,8 @@ private: Idle = 0, Trigger, Pdaf, - Coarse, + Coarse1, + Coarse2, Fine, Settle }; @@ -90,8 +91,8 @@ private: }; struct SpeedDependentParams { - double stepCoarse; /* used for scans */ - double stepFine; /* used for scans */ + double stepCoarse; /* in dioptres; used for scans */ + double stepFine; /* in dioptres; used for scans */ double contrastRatio; /* used for scan termination and reporting */ double retriggerRatio; /* contrast and RGB ratio for re-triggering */ uint32_t retriggerDelay; /* frames of stability before re-triggering */ @@ -177,7 +178,7 @@ private: unsigned sameSignCount_; unsigned sceneChangeCount_; unsigned scanMaxIndex_; - double scanMaxContrast_, scanMinContrast_; + double scanMaxContrast_, scanMinContrast_, scanStep_; std::vector scanData_; AfState reportState_; }; From 619da07f733c767b3e3ce1d8c400533275fc94bb Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:29 +0100 Subject: [PATCH 46/64] ipa: rpi: Update IMX708 camera tuning files for AutoFocus changes Explicitly add new parameters: "retrigger_ratio", "retrigger_delay", "check_for_ir". Tweak other parameters to suit algorithm changes. (Though existing tuning files should still work acceptably.) Add AfSpeedFast parameters for the Raspberry Pi V3 standard lens. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/pisp/data/imx708.json | 23 ++++++++++++++++++--- src/ipa/rpi/pisp/data/imx708_noir.json | 23 ++++++++++++++++++--- src/ipa/rpi/pisp/data/imx708_wide.json | 19 ++++++++++------- src/ipa/rpi/pisp/data/imx708_wide_noir.json | 19 ++++++++++------- src/ipa/rpi/vc4/data/imx708.json | 23 ++++++++++++++++++--- src/ipa/rpi/vc4/data/imx708_noir.json | 23 ++++++++++++++++++--- src/ipa/rpi/vc4/data/imx708_wide.json | 19 ++++++++++------- src/ipa/rpi/vc4/data/imx708_wide_noir.json | 19 ++++++++++------- 8 files changed, 128 insertions(+), 40 deletions(-) diff --git a/src/ipa/rpi/pisp/data/imx708.json b/src/ipa/rpi/pisp/data/imx708.json index e8d25c216..7f2e78655 100644 --- a/src/ipa/rpi/pisp/data/imx708.json +++ b/src/ipa/rpi/pisp/data/imx708.json @@ -1139,11 +1139,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1151,6 +1167,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -1267,4 +1284,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_noir.json b/src/ipa/rpi/pisp/data/imx708_noir.json index e69afb0c6..c5e6a2652 100644 --- a/src/ipa/rpi/pisp/data/imx708_noir.json +++ b/src/ipa/rpi/pisp/data/imx708_noir.json @@ -1156,11 +1156,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1168,6 +1184,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -1230,4 +1247,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_wide.json b/src/ipa/rpi/pisp/data/imx708_wide.json index 9fff05d93..8550cdfc1 100644 --- a/src/ipa/rpi/pisp/data/imx708_wide.json +++ b/src/ipa/rpi/pisp/data/imx708_wide.json @@ -1148,23 +1148,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1172,6 +1176,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -1290,4 +1295,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_wide_noir.json b/src/ipa/rpi/pisp/data/imx708_wide_noir.json index 75d1149b6..069a06180 100644 --- a/src/ipa/rpi/pisp/data/imx708_wide_noir.json +++ b/src/ipa/rpi/pisp/data/imx708_wide_noir.json @@ -1057,23 +1057,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1081,6 +1085,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -1145,4 +1150,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708.json b/src/ipa/rpi/vc4/data/imx708.json index 4de6f0796..e54ceff49 100644 --- a/src/ipa/rpi/vc4/data/imx708.json +++ b/src/ipa/rpi/vc4/data/imx708.json @@ -638,11 +638,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -650,6 +666,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -668,4 +685,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_noir.json b/src/ipa/rpi/vc4/data/imx708_noir.json index 7b7ee874f..f351a1800 100644 --- a/src/ipa/rpi/vc4/data/imx708_noir.json +++ b/src/ipa/rpi/vc4/data/imx708_noir.json @@ -737,11 +737,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -749,6 +765,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -767,4 +784,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_wide.json b/src/ipa/rpi/vc4/data/imx708_wide.json index 6f45aafc0..bf1b122cd 100644 --- a/src/ipa/rpi/vc4/data/imx708_wide.json +++ b/src/ipa/rpi/vc4/data/imx708_wide.json @@ -637,23 +637,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -661,6 +665,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -679,4 +684,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_wide_noir.json b/src/ipa/rpi/vc4/data/imx708_wide_noir.json index b9a5227e1..ea1c8c690 100644 --- a/src/ipa/rpi/vc4/data/imx708_wide_noir.json +++ b/src/ipa/rpi/vc4/data/imx708_wide_noir.json @@ -628,23 +628,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -652,6 +656,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -670,4 +675,4 @@ } } ] -} \ No newline at end of file +} From 17eed522e8ff835fd96158aed40818346636032b Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Mon, 7 Jul 2025 20:37:46 +0200 Subject: [PATCH 47/64] subprojects: libpisp: Update to 1.2.1 Update the libpisp wrap to use the latest 1.2.1 release which silences an 'unused-parameter' warning. Bug: https://github.com/raspberrypi/libpisp/pull/43 Reviewed-by: Naushir Patuck Signed-off-by: Christian Rauch Acked-by: Kieran Bingham Acked-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- subprojects/libpisp.wrap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/libpisp.wrap b/subprojects/libpisp.wrap index 8b62c036f..0e0c7baaf 100644 --- a/subprojects/libpisp.wrap +++ b/subprojects/libpisp.wrap @@ -2,5 +2,5 @@ [wrap-git] url = https://github.com/raspberrypi/libpisp.git -revision = v1.2.0 +revision = v1.2.1 depth = 1 From 525325440b5047ac94ce96a33e03f233b8b3c59f Mon Sep 17 00:00:00 2001 From: Harvey Yang Date: Mon, 9 Dec 2024 16:34:07 +0000 Subject: [PATCH 48/64] V4L2VideoDevice: Call FrameBuffer::Private::cancel() in streamOff() At the moment `V4L2VideoDevice::streamOff()` sets `FrameBuffer::Private`'s metadata directly, while that's equivalent to calling `FrameBuffer::Private::cancel()`. To ease code tracing, this patch replace the manual modification with the function call. Signed-off-by: Harvey Yang Reviewed-by: Kieran Bingham Reviewed-by: Umang Jain Signed-off-by: Kieran Bingham --- src/libcamera/v4l2_videodevice.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp index d53aa2d3c..7822bf1ef 100644 --- a/src/libcamera/v4l2_videodevice.cpp +++ b/src/libcamera/v4l2_videodevice.cpp @@ -2031,10 +2031,9 @@ int V4L2VideoDevice::streamOff() /* Send back all queued buffers. */ for (auto it : queuedBuffers_) { FrameBuffer *buffer = it.second; - FrameMetadata &metadata = buffer->_d()->metadata(); cache_->put(it.first); - metadata.status = FrameMetadata::FrameCancelled; + buffer->_d()->cancel(); bufferReady.emit(buffer); } From e6fb24ffdb3efb66829a8b0a8ce5628e4552ee30 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Tue, 8 Jul 2025 10:22:39 +0100 Subject: [PATCH 49/64] ipa: rpi: Fix bug in AfState reporting A previous change introduced a bug in which it reported AfStateIdle when idle in Auto mode, when it should continue to report the most recent AF cycle's outcome (AfStateFocused or AfStateFailed). Also fix the Pause method so it won't reset state to AfStateIdle when paused in Continuous AF mode (to match documented behaviour). Fixes: ea5f451c5660 ("ipa: rpi: controller: AutoFocus bidirectional scanning") Signed-off-by: Nick Hollinghurst Reviewed-by: David Plowman Tested-by: David Plowman Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index eaaca4bc9..26e599303 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -810,10 +810,10 @@ void Af::prepare(Metadata *imageMetadata) else status.pauseState = AfPauseState::Running; - if (scanState_ == ScanState::Idle) - status.state = AfState::Idle; - else if (mode_ == AfModeAuto) + if (mode_ == AfModeAuto && scanState_ != ScanState::Idle) status.state = AfState::Scanning; + else if (mode_ == AfModeManual) + status.state = AfState::Idle; else status.state = reportState_; status.lensSetting = initted_ ? std::optional(cfg_.map.eval(fsmooth_)) @@ -954,8 +954,10 @@ void Af::pause(AfAlgorithm::AfPause pause) scanState_ = ScanState::Trigger; } else if (pause != AfPauseResume && !pauseFlag_) { pauseFlag_ = true; - if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse1) - goIdle(); + if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse1) { + scanState_ = ScanState::Idle; + scanData_.clear(); + } } } } From a4372127534dd9093a0ec4c3e12465ae800257e6 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Tue, 8 Jul 2025 09:49:14 +0100 Subject: [PATCH 50/64] libcamera: controls: Remove hyphenation in control description text Remove the hyphenation in "micro-seconds" in the description for the ExposureTime control to match the rest of the document. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Kieran Bingham Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index 46f51436c..566e15337 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -212,7 +212,7 @@ controls: description: | Exposure time for the frame applied in the sensor device. - This value is specified in micro-seconds. + This value is specified in microseconds. This control will only take effect if ExposureTimeMode is Manual. If this control is set when ExposureTimeMode is Auto, the value will be From 29a88d85b730baed52a2f2e5fde2a927474ce41c Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Tue, 8 Jul 2025 09:49:15 +0100 Subject: [PATCH 51/64] libcamera: controls: Use nanoseconds units for FrameWallClock Use nanoseconds for the FrameWallClock control to match the units for other timestamp controls, including SensorTimestamp. Update the RPi pipeline handlers to match the new nanoseconds units when converting from SensorTimestamp to FrameWallClock. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Kieran Bingham Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_core.yaml | 3 ++- src/libcamera/pipeline/rpi/pisp/pisp.cpp | 2 +- src/libcamera/pipeline/rpi/vc4/vc4.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index 566e15337..eec4b4f93 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -1274,7 +1274,8 @@ controls: description: | This timestamp corresponds to the same moment in time as the SensorTimestamp, but is represented as a wall clock time as measured by - the CLOCK_REALTIME clock. + the CLOCK_REALTIME clock. Like SensorTimestamp, the timestamp value is + expressed in nanoseconds. Being a wall clock measurement, it can be used to synchronise timing across different devices. diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index 2df91bacf..92b9070c1 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -1760,7 +1760,7 @@ void PiSPCameraData::cfeBufferDequeue(FrameBuffer *buffer) */ wallClockRecovery_.addSample(); uint64_t sensorTimestamp = buffer->metadata().timestamp; - uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp / 1000); + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp); ctrl.set(controls::SensorTimestamp, sensorTimestamp); ctrl.set(controls::FrameWallClock, wallClockTimestamp); diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index e99a7edf8..5cadef527 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -778,7 +778,7 @@ void Vc4CameraData::unicamBufferDequeue(FrameBuffer *buffer) */ wallClockRecovery_.addSample(); uint64_t sensorTimestamp = buffer->metadata().timestamp; - uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp / 1000); + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp); ctrl.set(controls::SensorTimestamp, sensorTimestamp); ctrl.set(controls::FrameWallClock, wallClockTimestamp); From fb7208397533b73b2b8e3bada936a83161235a4a Mon Sep 17 00:00:00 2001 From: Umang Jain Date: Tue, 1 Jul 2025 15:38:02 +0530 Subject: [PATCH 52/64] camera: Fix spell error Correct 'CameraConfigutation' spell error to 'CameraConfiguration'. Signed-off-by: Umang Jain Reviewed-by: Jacopo Mondi Reviewed-by: Kieran Bingham Signed-off-by: Kieran Bingham --- src/libcamera/camera.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/camera.cpp b/src/libcamera/camera.cpp index c180a5fdd..b4f9d2433 100644 --- a/src/libcamera/camera.cpp +++ b/src/libcamera/camera.cpp @@ -488,7 +488,7 @@ std::size_t CameraConfiguration::size() const * * \return A CameraConfiguration::Status value that describes the validation * status. - * \retval CameraConfigutation::Adjusted The configuration has been adjusted + * \retval CameraConfiguration::Adjusted The configuration has been adjusted * and is now valid. The color space of some or all of the streams may have * been changed. The caller shall check the color spaces carefully. * \retval CameraConfiguration::Valid The configuration was already valid and From afd9890b7b9a65d95ef3295f52e01e0ec9ce7b3f Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 24 Feb 2025 03:19:34 +0200 Subject: [PATCH 53/64] libcamera: delayed_controls: Inherit from Object class A second use-after-free bug related to signals staying connected after the receiver DelayedControls instance gets deleted has been found, this time in the simple pipeline handler. Fix the issue once and for all by making the DelayedControls class inherit from Object. This will disconnect signals automatically upon deletion of the receiver. Signed-off-by: Laurent Pinchart Tested-by: Stanislaw Gruszka Tested-by: Isaac Scott Reviewed-by: Isaac Scott Reviewed-by: Kieran Bingham Signed-off-by: Kieran Bingham --- include/libcamera/internal/delayed_controls.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/libcamera/internal/delayed_controls.h b/include/libcamera/internal/delayed_controls.h index e8d3014d9..b64d8bba7 100644 --- a/include/libcamera/internal/delayed_controls.h +++ b/include/libcamera/internal/delayed_controls.h @@ -10,13 +10,15 @@ #include #include +#include + #include namespace libcamera { class V4L2Device; -class DelayedControls +class DelayedControls : public Object { public: struct ControlParams { From 98921d93d06c7c6e805f6cafe8efee2dd2721793 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 5 Apr 2025 14:59:17 +0300 Subject: [PATCH 54/64] gitignore: ignore my setup Signed-off-by: Vasiliy Doylov --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 51d314408..656f622f0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ *.pyc __pycache__/ venv/ +.vscode/ +.cache/ +compile_commands.json From f7bf4c8d4f2bade02df875c311caaa36d21b9e3d Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 7 Jun 2025 02:22:30 +0300 Subject: [PATCH 55/64] CI: Add local forgejo CI Signed-off-by: Vasiliy Doylov --- .forgejo/workflows/build-alpine.yaml | 58 +++++++++ .forgejo/workflows/sync-with-upstream.yaml | 18 +++ package/alpine/APKBUILD | 132 +++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 .forgejo/workflows/build-alpine.yaml create mode 100644 .forgejo/workflows/sync-with-upstream.yaml create mode 100644 package/alpine/APKBUILD diff --git a/.forgejo/workflows/build-alpine.yaml b/.forgejo/workflows/build-alpine.yaml new file mode 100644 index 000000000..9fc5ff5ea --- /dev/null +++ b/.forgejo/workflows/build-alpine.yaml @@ -0,0 +1,58 @@ +name: PostmarketOS Build +run-name: PostmarketOS Build +on: + push: + workflow_dispatch: + +jobs: + prepare: + name: Prepare + runs-on: Pmbootstrap + outputs: + time: ${{ steps.time.outputs.time }} + steps: + - name: Set start Time + id: time + shell: sh + run: echo time=$(date +"%Y%m%d%H%M%S") >> $GITHUB_OUTPUT + - name: Update pmbootstrap + uses: actions/pmbootstrap-update@master + - name: Remove libcamera aport + run: rm -rf ${{env.PMB_PMAPORTS}}/temp/libcamera + + build: + name: Build for ${{ matrix.info.arch }} + runs-on: Pmbootstrap + strategy: + matrix: + info: + - arch: x86_64 + - arch: aarch64 + needs: prepare + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Build packages + id: build + uses: actions/pmbootstrap-build@main + with: + name: libcamera + aports: ${{github.workspace}}/package/alpine + arch: ${{ matrix.info.arch }} + src: ${{github.workspace}} + time: ${{ needs.prepare.outputs.time }} + - name: "Upload packages" + uses: actions/upload-alpine-package@main + with: + files: ${{steps.build.outputs.packages}} + secret: ${{secrets.PACKAGE_TOKEN}} + + clean: + name: "Clean" + runs-on: Pmbootstrap + needs: build + if: always() + continue-on-error: true + steps: + - name: Update pmbootstrap + uses: actions/pmbootstrap-update@master diff --git a/.forgejo/workflows/sync-with-upstream.yaml b/.forgejo/workflows/sync-with-upstream.yaml new file mode 100644 index 000000000..808cbaee8 --- /dev/null +++ b/.forgejo/workflows/sync-with-upstream.yaml @@ -0,0 +1,18 @@ +name: Sync fork with upstream +run-name: Sync fork with upstream +on: + schedule: + - cron: "@daily" + workflow_dispatch: + +jobs: + sync: + name: Sync + runs-on: Misc + steps: + - name: Sync repository with upstream + uses: actions/sync-with-mirror@main + with: + secret: ${{ secrets.PUSH_TOKEN }} + name: libcamera + branch: master diff --git a/package/alpine/APKBUILD b/package/alpine/APKBUILD new file mode 100644 index 000000000..ddfc72950 --- /dev/null +++ b/package/alpine/APKBUILD @@ -0,0 +1,132 @@ +pkgname=libcamera +pkgver=9999999 +pkgrel=0 +pkgdesc="Linux camera framework" +url="https://libcamera.org/" +arch="all" +license="LGPL-2.1-or-later AND GPL-2.0-or-later" +depends_dev=" + eudev-dev + glib-dev + gnutls-dev + gst-plugins-bad-dev + qt6-qtbase-dev + " + +makedepends="$depends_dev + coreutils + doxygen + graphviz + gtest-dev + libevent-dev + libpisp-dev + libunwind-dev + libyuv-dev + linux-headers + meson + py3-jinja2 + py3-ply + py3-sphinx + py3-yaml + qt6-qttools-dev + yaml-dev + " +subpackages=" + $pkgname-dbg + $pkgname-dev + $pkgname-doc + qcam + $pkgname-gstreamer + $pkgname-v4l2 + $pkgname-tools + " +source="" + +builddir="$srcdir/$pkgname-v$_pkgver" +# gstreamer tests fail +# manual strip because ipa .sign files depend on the file contents- have to re-sign after strip +options="!strip !check" + +case "$CARCH" in +arm*|aarch64) + subpackages="$subpackages $pkgname-raspberrypi" + ;; +esac + +case "$CARCH" in +ppc64le|s390x|riscv64|loongarch64) + # doesn't install any ipa + ;; +*) + # WIP: HACK? Don't depend on this this shit + # depends="$pkgname-ipa=$pkgver-r$pkgrel" + subpackages="$subpackages $pkgname-ipa" + ;; +esac + +build() { + abuild-meson \ + -Dtest=false \ + -Dv4l2=true \ + -Dwerror=false \ + . output + meson compile -C output +} + +package() { + DESTDIR="$pkgdir" meson install --no-rebuild -C output + + # manual strip first.. + scanelf --recursive \ + --nobanner \ + --etype "ET_DYN,ET_EXEC" \ + --format "%F" \ + "$pkgdir" \ + | while read -r file; do + strip "$file" + done +} + +ipa() { + depends="" + amove usr/lib/libcamera + # then sign ipa's + local ipa + for ipa in "$subpkgdir"/usr/lib/libcamera/ipa/ipa*.so; do + msg "signing $ipa" + "$builddir"/src/ipa/ipa-sign.sh \ + "$(find "$builddir"/output -type f -iname "*ipa-priv-key.pem")" \ + "$ipa" \ + "$ipa".sign + done +} + +qcam() { + depends="" + amove usr/bin/qcam +} + +gstreamer() { + depends="" + amove usr/lib/gstreamer-1.0 +} + +v4l2() { + depends="" + amove usr/libexec/libcamera/v4l2-compat.so +} + +raspberrypi() { + depends="" + amove usr/share/libcamera/ipa/rpi + amove usr/libexec/libcamera/raspberrypi_ipa_proxy + amove usr/share/libcamera/pipeline/rpi/vc4 +} + +tools() { + depends="" + amove usr/bin/cam + amove usr/bin/lc-compliance +} + +sha512sums="" From e3b71632540cf30b8d542c7e754327703c7e5a92 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sun, 23 Feb 2025 23:03:48 +0300 Subject: [PATCH 56/64] libcamera: software_isp: Add brightness control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 29 +++++++++++++++++++++++++++++ src/ipa/simple/algorithms/agc.h | 7 +++++++ src/ipa/simple/ipa_context.h | 2 ++ 3 files changed, 38 insertions(+) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index c46bb0ebe..249caa264 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -41,6 +41,33 @@ Agc::Agc() { } +int Agc::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + return 0; +} + +int Agc::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.brightness = std::optional(); + + return 0; +} + +void Agc::queueRequest(typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &brightness = controls.get(controls::Brightness); + if (brightness.has_value()) { + context.activeState.knobs.brightness = brightness; + LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); + } +} + void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) { /* @@ -54,6 +81,8 @@ void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, dou double next; int32_t &exposure = frameContext.sensor.exposure; double &again = frameContext.sensor.gain; + const auto brightness = context.activeState.knobs.brightness.value_or(1.0); + exposureMSV /= brightness; if (exposureMSV < kExposureOptimal - kExposureSatisfactory) { next = exposure * kExpNumeratorUp / kExpDenominator; diff --git a/src/ipa/simple/algorithms/agc.h b/src/ipa/simple/algorithms/agc.h index 112d9f5a1..00bc101e7 100644 --- a/src/ipa/simple/algorithms/agc.h +++ b/src/ipa/simple/algorithms/agc.h @@ -19,6 +19,13 @@ public: Agc(); ~Agc() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; void process(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const SwIspStats *stats, diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index a471b80ae..afc28ba22 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -64,6 +64,8 @@ struct IPAActiveState { /* 0..2 range, 1.0 = normal */ std::optional contrast; std::optional saturation; + /* 0..2 range, 1.0 = normal */ + std::optional brightness; } knobs; }; From 30e17d48c71230ca41eeafacbd99df5cd6831b09 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sun, 16 Mar 2025 22:12:10 +0300 Subject: [PATCH 57/64] libcamera: software_isp: Add AGC disable control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 11 +++++++++++ src/ipa/simple/ipa_context.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index 249caa264..de20863d8 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -44,6 +44,7 @@ Agc::Agc() int Agc::init(IPAContext &context, [[maybe_unused]] const YamlObject &tuningData) { + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); return 0; } @@ -52,6 +53,7 @@ int Agc::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { context.activeState.knobs.brightness = std::optional(); + context.activeState.knobs.ae_enabled = std::optional(); return 0; } @@ -62,10 +64,15 @@ void Agc::queueRequest(typename Module::Context &context, const ControlList &controls) { const auto &brightness = controls.get(controls::Brightness); + const auto &ae_enabled = controls.get(controls::AeEnable); if (brightness.has_value()) { context.activeState.knobs.brightness = brightness; LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); } + if (ae_enabled.has_value()) { + context.activeState.knobs.ae_enabled = ae_enabled; + LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); + } } void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) @@ -132,6 +139,10 @@ void Agc::process(IPAContext &context, const SwIspStats *stats, ControlList &metadata) { + const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); + if (!ae_enable) + return; + utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index afc28ba22..8e8add4c3 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -66,6 +66,8 @@ struct IPAActiveState { std::optional saturation; /* 0..2 range, 1.0 = normal */ std::optional brightness; + /* 0..1 range, 1 = normal */ + std::optional ae_enabled; } knobs; }; From 7d195335897e991a750c9a09229363f30b2a5100 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Mon, 17 Mar 2025 04:24:56 +0300 Subject: [PATCH 58/64] libcamera: software_isp: Add focus control Signed-off-by: Vasiliy Doylov --- .../internal/software_isp/software_isp.h | 4 +- include/libcamera/ipa/soft.mojom | 3 +- src/ipa/simple/algorithms/af.cpp | 71 +++++++++++++++++++ src/ipa/simple/algorithms/af.h | 40 +++++++++++ src/ipa/simple/algorithms/meson.build | 1 + src/ipa/simple/data/uncalibrated.yaml | 1 + src/ipa/simple/ipa_context.h | 9 +++ src/ipa/simple/soft_simple.cpp | 18 ++++- src/libcamera/pipeline/simple/simple.cpp | 30 ++++++-- src/libcamera/software_isp/software_isp.cpp | 4 +- 10 files changed, 169 insertions(+), 12 deletions(-) create mode 100644 src/ipa/simple/algorithms/af.cpp create mode 100644 src/ipa/simple/algorithms/af.h diff --git a/include/libcamera/internal/software_isp/software_isp.h b/include/libcamera/internal/software_isp/software_isp.h index 786246592..8f3a6b1b1 100644 --- a/include/libcamera/internal/software_isp/software_isp.h +++ b/include/libcamera/internal/software_isp/software_isp.h @@ -86,11 +86,11 @@ public: Signal outputBufferReady; Signal ispStatsReady; Signal metadataReady; - Signal setSensorControls; + Signal setSensorControls; private: void saveIspParams(); - void setSensorCtrls(const ControlList &sensorControls); + void setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls); void statsReady(uint32_t frame, uint32_t bufferId); void inputReady(FrameBuffer *input); void outputReady(FrameBuffer *output); diff --git a/include/libcamera/ipa/soft.mojom b/include/libcamera/ipa/soft.mojom index 77328c5fd..e5767532c 100644 --- a/include/libcamera/ipa/soft.mojom +++ b/include/libcamera/ipa/soft.mojom @@ -10,6 +10,7 @@ import "include/libcamera/ipa/core.mojom"; struct IPAConfigInfo { libcamera.ControlInfoMap sensorControls; + libcamera.ControlInfoMap lensControls; }; interface IPASoftInterface { @@ -32,7 +33,7 @@ interface IPASoftInterface { }; interface IPASoftEventInterface { - setSensorControls(libcamera.ControlList sensorControls); + setSensorControls(libcamera.ControlList sensorControls, libcamera.ControlList lensControls); setIspParams(); metadataReady(uint32 frame, libcamera.ControlList metadata); }; diff --git a/src/ipa/simple/algorithms/af.cpp b/src/ipa/simple/algorithms/af.cpp new file mode 100644 index 000000000..b51ed95e4 --- /dev/null +++ b/src/ipa/simple/algorithms/af.cpp @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#include "af.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftAutoFocus) + +namespace ipa::soft::algorithms { + +Af::Af() +{ +} + +int Af::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::LensPosition] = ControlInfo(0.0f, 100.0f, 50.0f); + return 0; +} + +int Af::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.focus_pos = std::optional(); + + return 0; +} + +void Af::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &focus_pos = controls.get(controls::LensPosition); + if (focus_pos.has_value()) { + context.activeState.knobs.focus_pos = focus_pos; + LOG(IPASoftAutoFocus, Debug) << "Setting focus position to " << focus_pos.value(); + } +} + +void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAFrameContext &frameContext, [[maybe_unused]] double exposureMSV) +{ + frameContext.lens.focus_pos = context.activeState.knobs.focus_pos.value_or(50.0) / 100.0 * (context.configuration.focus.focus_max - context.configuration.focus.focus_min); +} + +void Af::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const SwIspStats *stats, + [[maybe_unused]] ControlList &metadata) +{ + updateFocus(context, frameContext, 0); +} + +REGISTER_IPA_ALGORITHM(Af, "Af") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/af.h b/src/ipa/simple/algorithms/af.h new file mode 100644 index 000000000..a575ef102 --- /dev/null +++ b/src/ipa/simple/algorithms/af.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Af : public Algorithm +{ +public: + Af(); + ~Af() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateFocus(IPAContext &context, IPAFrameContext &frameContext, double focus); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build index 2d0adb059..dec59ee8c 100644 --- a/src/ipa/simple/algorithms/meson.build +++ b/src/ipa/simple/algorithms/meson.build @@ -6,4 +6,5 @@ soft_simple_ipa_algorithms = files([ 'blc.cpp', 'ccm.cpp', 'lut.cpp', + 'af.cpp', ]) diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index 5508e6686..d14b34294 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -16,4 +16,5 @@ algorithms: # 0, 0, 1] - Lut: - Agc: + - Af: ... diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index 8e8add4c3..71b9bb637 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -34,6 +34,9 @@ struct IPASessionConfiguration { struct { std::optional level; } black; + struct { + int32_t focus_min, focus_max; + } focus; }; struct IPAActiveState { @@ -68,6 +71,8 @@ struct IPAActiveState { std::optional brightness; /* 0..1 range, 1 = normal */ std::optional ae_enabled; + /* 0..100 range, 50.0 = normal */ + std::optional focus_pos; } knobs; }; @@ -81,6 +86,10 @@ struct IPAFrameContext : public FrameContext { double gain; } sensor; + struct { + int32_t focus_pos; + } lens; + struct { double red; double blue; diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp index c94c4cd55..53fa74c03 100644 --- a/src/ipa/simple/soft_simple.cpp +++ b/src/ipa/simple/soft_simple.cpp @@ -77,6 +77,7 @@ private: SwIspStats *stats_; std::unique_ptr camHelper_; ControlInfoMap sensorInfoMap_; + ControlInfoMap lensInfoMap_; /* Local parameter storage */ struct IPAContext context_; @@ -196,6 +197,7 @@ int IPASoftSimple::init(const IPASettings &settings, int IPASoftSimple::configure(const IPAConfigInfo &configInfo) { sensorInfoMap_ = configInfo.sensorControls; + lensInfoMap_ = configInfo.lensControls; const ControlInfo &exposureInfo = sensorInfoMap_.find(V4L2_CID_EXPOSURE)->second; const ControlInfo &gainInfo = sensorInfoMap_.find(V4L2_CID_ANALOGUE_GAIN)->second; @@ -205,6 +207,17 @@ int IPASoftSimple::configure(const IPAConfigInfo &configInfo) context_.activeState = {}; context_.frameContexts.clear(); + if (lensInfoMap_.empty()) { + LOG(IPASoft, Warning) << "No camera leans found! Focus control disabled."; + context_.configuration.focus.focus_min = 0; + context_.configuration.focus.focus_max = 0; + } else { + const ControlInfo &lensInfo = lensInfoMap_.find(V4L2_CID_FOCUS_ABSOLUTE)->second; + context_.configuration.focus.focus_min = lensInfo.min().get(); + context_.configuration.focus.focus_max = lensInfo.max().get(); + LOG(IPASoft, Warning) << "Camera leans found! Focus: " << context_.configuration.focus.focus_min << "-" << context_.configuration.focus.focus_max; + } + context_.configuration.agc.lineDuration = context_.sensorInfo.minLineLength * 1.0s / context_.sensorInfo.pixelRate; context_.configuration.agc.exposureMin = exposureInfo.min().get(); @@ -327,7 +340,10 @@ void IPASoftSimple::processStats(const uint32_t frame, ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(camHelper_ ? camHelper_->gainCode(againNew) : againNew)); - setSensorControls.emit(ctrls); + ControlList lens_ctrls(lensInfoMap_); + lens_ctrls.set(V4L2_CID_FOCUS_ABSOLUTE, frameContext.lens.focus_pos); + + setSensorControls.emit(ctrls, lens_ctrls); } std::string IPASoftSimple::logPrefix() const diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp index efb07051b..4c002ca68 100644 --- a/src/libcamera/pipeline/simple/simple.cpp +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -30,6 +30,7 @@ #include #include "libcamera/internal/camera.h" +#include "libcamera/internal/camera_lens.h" #include "libcamera/internal/camera_sensor.h" #include "libcamera/internal/camera_sensor_properties.h" #include "libcamera/internal/converter.h" @@ -41,6 +42,8 @@ #include "libcamera/internal/v4l2_subdevice.h" #include "libcamera/internal/v4l2_videodevice.h" +#include "libcamera/controls.h" + namespace libcamera { LOG_DEFINE_CATEGORY(SimplePipeline) @@ -356,7 +359,7 @@ private: void ispStatsReady(uint32_t frame, uint32_t bufferId); void metadataReady(uint32_t frame, const ControlList &metadata); - void setSensorControls(const ControlList &sensorControls); + void setSensorControls(const ControlList &sensorControls, const ControlList &lensControls); }; class SimpleCameraConfiguration : public CameraConfiguration @@ -1002,7 +1005,7 @@ void SimpleCameraData::metadataReady(uint32_t frame, const ControlList &metadata tryCompleteRequest(info->request); } -void SimpleCameraData::setSensorControls(const ControlList &sensorControls) +void SimpleCameraData::setSensorControls(const ControlList &sensorControls, const ControlList &lensControls) { delayedCtrls_->push(sensorControls); /* @@ -1013,10 +1016,21 @@ void SimpleCameraData::setSensorControls(const ControlList &sensorControls) * but it also bypasses delayedCtrls_, creating AGC regulation issues. * Both problems should be fixed. */ - if (!frameStartEmitter_) { - ControlList ctrls(sensorControls); - sensor_->setControls(&ctrls); - } + if (frameStartEmitter_) + return; + + ControlList ctrls(sensorControls); + sensor_->setControls(&ctrls); + + CameraLens *focusLens = sensor_->focusLens(); + if (!focusLens) + return; + + if (!lensControls.contains(V4L2_CID_FOCUS_ABSOLUTE)) + return; + + const ControlValue &focusValue = lensControls.get(V4L2_CID_FOCUS_ABSOLUTE); + focusLens->setFocusPosition(focusValue.get()); } /* Retrieve all source pads connected to a sink pad through active routes. */ @@ -1406,6 +1420,10 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) } else { ipa::soft::IPAConfigInfo configInfo; configInfo.sensorControls = data->sensor_->controls(); + if (data->sensor_->focusLens() != nullptr) + configInfo.lensControls = data->sensor_->focusLens()->controls(); + else + configInfo.lensControls = ControlInfoMap(); return data->swIsp_->configure(inputCfg, outputCfgs, configInfo); } } diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index 28e2a360e..8f41591c8 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -395,9 +395,9 @@ void SoftwareIsp::saveIspParams() debayerParams_ = *sharedParams_; } -void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls) +void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls) { - setSensorControls.emit(sensorControls); + setSensorControls.emit(sensorControls, lensControls); } void SoftwareIsp::statsReady(uint32_t frame, uint32_t bufferId) From d0bf6e7f8862e3e8e17bb08d998f0ad706e809e6 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 21 May 2025 21:24:09 +0300 Subject: [PATCH 59/64] libcamera: software_isp: Add manual exposure control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 14 ++++++++++++-- src/ipa/simple/ipa_context.h | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index de20863d8..35f4798cf 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -46,6 +46,8 @@ int Agc::init(IPAContext &context, { context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + context.ctrlMap[&controls::ExposureValue] = ControlInfo(0.0f, 0.5f, 1.0f); + return 0; } @@ -65,6 +67,7 @@ void Agc::queueRequest(typename Module::Context &context, { const auto &brightness = controls.get(controls::Brightness); const auto &ae_enabled = controls.get(controls::AeEnable); + const auto &exposure_value = controls.get(controls::ExposureValue); if (brightness.has_value()) { context.activeState.knobs.brightness = brightness; LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); @@ -73,6 +76,10 @@ void Agc::queueRequest(typename Module::Context &context, context.activeState.knobs.ae_enabled = ae_enabled; LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); } + if (exposure_value.has_value()) { + context.activeState.knobs.exposure_value = exposure_value.value(); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << exposure_value.value(); + } } void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) @@ -140,13 +147,16 @@ void Agc::process(IPAContext &context, ControlList &metadata) { const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); - if (!ae_enable) - return; + if (!ae_enable) + frameContext.sensor.exposure = (int32_t)( context.activeState.knobs.exposure_value.value_or(0.5) * (context.configuration.agc.exposureMax - context.configuration.agc.exposureMin)); utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << frameContext.sensor.exposure; + if (!ae_enable) + return; /* * Calculate Mean Sample Value (MSV) according to formula from: diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index 71b9bb637..c5b5527b5 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -71,6 +71,8 @@ struct IPAActiveState { std::optional brightness; /* 0..1 range, 1 = normal */ std::optional ae_enabled; + /* 0..1 range, 0.5 = normal */ + std::optional exposure_value; /* 0..100 range, 50.0 = normal */ std::optional focus_pos; } knobs; From 6e8d4d86d75624ec9a84bc5084238ef14428460e Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 21 May 2025 22:24:01 +0300 Subject: [PATCH 60/64] libcamera: software_isp: Add control to disable statistic collection Signed-off-by: Vasiliy Doylov --- .../internal/software_isp/debayer_params.h | 9 +++ src/ipa/simple/algorithms/meson.build | 1 + src/ipa/simple/algorithms/stat.cpp | 65 +++++++++++++++++++ src/ipa/simple/algorithms/stat.h | 38 +++++++++++ src/ipa/simple/data/uncalibrated.yaml | 1 + src/ipa/simple/ipa_context.h | 2 + src/libcamera/software_isp/debayer_cpu.cpp | 12 ++-- src/libcamera/software_isp/debayer_cpu.h | 1 + 8 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 src/ipa/simple/algorithms/stat.cpp create mode 100644 src/ipa/simple/algorithms/stat.h diff --git a/include/libcamera/internal/software_isp/debayer_params.h b/include/libcamera/internal/software_isp/debayer_params.h index 217cd5d92..6c36defc9 100644 --- a/include/libcamera/internal/software_isp/debayer_params.h +++ b/include/libcamera/internal/software_isp/debayer_params.h @@ -49,6 +49,15 @@ struct DebayerParams { CcmLookupTable greenCcm; CcmLookupTable blueCcm; LookupTable gammaLut; + + /* + * Statistic controls + * + * Statistic collecting are very slow. We can disable it for some actions like + * video capture or streaming. + * TODO: Add statistic window control + */ + bool collect_stats; }; } /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build index dec59ee8c..e5666b262 100644 --- a/src/ipa/simple/algorithms/meson.build +++ b/src/ipa/simple/algorithms/meson.build @@ -7,4 +7,5 @@ soft_simple_ipa_algorithms = files([ 'ccm.cpp', 'lut.cpp', 'af.cpp', + 'stat.cpp', ]) diff --git a/src/ipa/simple/algorithms/stat.cpp b/src/ipa/simple/algorithms/stat.cpp new file mode 100644 index 000000000..181a5d818 --- /dev/null +++ b/src/ipa/simple/algorithms/stat.cpp @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Debayer statistic controls + */ + +#include "stat.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftStatistic) + +namespace ipa::soft::algorithms { + +Stat::Stat() +{ +} + +int Stat::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::DebugMetadataEnable] = ControlInfo(false, true, true); + return 0; +} + +int Stat::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.stats_enabled = std::optional(); + + return 0; +} + +void Stat::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &stats_enabled = controls.get(controls::DebugMetadataEnable); + if (stats_enabled.has_value()) { + context.activeState.knobs.stats_enabled = stats_enabled; + LOG(IPASoftStatistic, Debug) << "Setting debayer enabled to " << stats_enabled.value(); + } +} + +void Stat::prepare([[maybe_unused]]IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]]IPAFrameContext &frameContext, + [[maybe_unused]] DebayerParams *params) +{ + params->collect_stats = context.activeState.knobs.stats_enabled.value_or(true); +} + +REGISTER_IPA_ALGORITHM(Stat, "Stat") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/stat.h b/src/ipa/simple/algorithms/stat.h new file mode 100644 index 000000000..dc0051cb0 --- /dev/null +++ b/src/ipa/simple/algorithms/stat.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Debayer statistic controls + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Stat : public Algorithm +{ +public: + Stat(); + ~Stat() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void prepare(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + DebayerParams *params) override; + +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index d14b34294..47eaf9c61 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -17,4 +17,5 @@ algorithms: - Lut: - Agc: - Af: + - Stat: ... diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index c5b5527b5..cfc524a13 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -75,6 +75,8 @@ struct IPAActiveState { std::optional exposure_value; /* 0..100 range, 50.0 = normal */ std::optional focus_pos; + /* 0..1 range, 1 = normal */ + std::optional stats_enabled; } knobs; }; diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 66f6038c1..c6f070225 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -668,7 +668,7 @@ void DebayerCpu::process2(const uint8_t *src, uint8_t *dst) for (unsigned int y = window_.y; y < yEnd; y += 2) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(y, linePointers); + if (this->enable_statistic) stats_->processLine0(y, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -683,7 +683,7 @@ void DebayerCpu::process2(const uint8_t *src, uint8_t *dst) if (window_.y == 0) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(yEnd, linePointers); + if (this->enable_statistic) stats_->processLine0(yEnd, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -720,7 +720,7 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) for (unsigned int y = window_.y; y < yEnd; y += 4) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(y, linePointers); + if (this->enable_statistic) stats_->processLine0(y, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -733,7 +733,7 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine2(y, linePointers); + if (this->enable_statistic) stats_->processLine2(y, linePointers); (this->*debayer2_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -771,7 +771,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output for (const FrameBuffer::Plane &plane : output->planes()) dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); - + enable_statistic = params.collect_stats; green_ = params.green; greenCcm_ = params.greenCcm; if (swapRedBlueGains_) { @@ -805,7 +805,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output return; } - stats_->startFrame(); + if(this->enable_statistic) stats_->startFrame(); if (inputConfig_.patternSize.height == 2) process2(in.planes()[0].data(), out.planes()[0].data()); diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 926195e98..54304053f 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -165,6 +165,7 @@ private: /* Skip 30 frames for things to stabilize then measure 30 frames */ static constexpr unsigned int kFramesToSkip = 30; static constexpr unsigned int kLastFrameToMeasure = 60; + bool enable_statistic = true; }; } /* namespace libcamera */ From 7fe3e610cdac6adb0de1740e1ddd1c64fd471caf Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 21 Jun 2025 01:05:32 +0300 Subject: [PATCH 61/64] HACK: WIP: Clean queued request before assertion :D Signed-off-by: Vasiliy Doylov --- src/libcamera/pipeline_handler.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index d84dff3c9..31d501a13 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -372,6 +372,8 @@ void PipelineHandler::stop(Camera *camera) /* Make sure no requests are pending. */ Camera::Private *data = camera->_d(); + // WIP: Just clean for now, idk maybe something wrong with thread sync? + data->queuedRequests_.clear(); ASSERT(data->queuedRequests_.empty()); data->requestSequence_ = 0; From a106a436326d3c9e17388ddb822c8f61e93c5dd1 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 9 Jul 2025 16:07:14 +0300 Subject: [PATCH 62/64] libcamera: software_isp: Add autofocus Signed-off-by: Vasiliy Doylov --- .../internal/software_isp/swisp_stats.h | 4 ++ src/ipa/simple/algorithms/af.cpp | 62 ++++++++++++++++++- src/ipa/simple/algorithms/af.h | 1 + src/ipa/simple/ipa_context.h | 6 ++ src/libcamera/software_isp/swstats_cpu.cpp | 16 ++++- 5 files changed, 85 insertions(+), 4 deletions(-) diff --git a/include/libcamera/internal/software_isp/swisp_stats.h b/include/libcamera/internal/software_isp/swisp_stats.h index ae11f112e..3377dd825 100644 --- a/include/libcamera/internal/software_isp/swisp_stats.h +++ b/include/libcamera/internal/software_isp/swisp_stats.h @@ -44,6 +44,10 @@ struct SwIspStats { * \brief A histogram of luminance values */ Histogram yHistogram; + /** + * \brief Holds the sharpness of an image + */ + uint64_t sharpness; }; } /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/af.cpp b/src/ipa/simple/algorithms/af.cpp index b51ed95e4..52ddf7f1a 100644 --- a/src/ipa/simple/algorithms/af.cpp +++ b/src/ipa/simple/algorithms/af.cpp @@ -27,14 +27,18 @@ int Af::init(IPAContext &context, [[maybe_unused]] const YamlObject &tuningData) { context.ctrlMap[&controls::LensPosition] = ControlInfo(0.0f, 100.0f, 50.0f); + context.ctrlMap[&controls::AfTrigger] = ControlInfo(0, 1, 0); return 0; } int Af::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { + context.activeState.knobs.focus_sweep = std::optional(); context.activeState.knobs.focus_pos = std::optional(); - + context.activeState.knobs.focus_sweep = false; + context.activeState.knobs.focus_pos = 0; + context.configuration.focus.skip = 10; return 0; } @@ -44,10 +48,24 @@ void Af::queueRequest([[maybe_unused]] typename Module::Context &context, const ControlList &controls) { const auto &focus_pos = controls.get(controls::LensPosition); + const auto &af_trigger = controls.get(controls::AfTrigger); if (focus_pos.has_value()) { context.activeState.knobs.focus_pos = focus_pos; LOG(IPASoftAutoFocus, Debug) << "Setting focus position to " << focus_pos.value(); } + if (af_trigger.has_value()) { + context.activeState.knobs.focus_sweep = af_trigger.value() == 1; + if(context.activeState.knobs.focus_sweep){ + context.activeState.knobs.focus_pos = 0; + context.configuration.focus.focus_max_pos = 0; + context.configuration.focus.sharpness_max = 0; + context.configuration.focus.start = 0; + context.configuration.focus.stop = 100; + context.configuration.focus.step = 25; + LOG(IPASoftAutoFocus, Info) << "Starting focus sweep"; + } + } + } void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAFrameContext &frameContext, [[maybe_unused]] double exposureMSV) @@ -55,12 +73,54 @@ void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAF frameContext.lens.focus_pos = context.activeState.knobs.focus_pos.value_or(50.0) / 100.0 * (context.configuration.focus.focus_max - context.configuration.focus.focus_min); } +void Af::step(uint32_t& skip, double& start, double& stop, double& step, double& focus_pos, double& max_pos, uint64_t& max_sharp, uint64_t sharp, bool& sweep){ + if(!sweep) + return; + if(skip != 0){ + skip --; + return; + } + skip = 2; + if(focus_pos < start) { + focus_pos = start; + return; + } + if(sharp > max_sharp) { + max_sharp = sharp; + max_pos = focus_pos; + } + if(focus_pos >= stop) { + LOG(IPASoftAutoFocus, Info) << "Best focus on step " <sharpness, + context.activeState.knobs.focus_sweep.value()); updateFocus(context, frameContext, 0); } diff --git a/src/ipa/simple/algorithms/af.h b/src/ipa/simple/algorithms/af.h index a575ef102..901393717 100644 --- a/src/ipa/simple/algorithms/af.h +++ b/src/ipa/simple/algorithms/af.h @@ -33,6 +33,7 @@ public: private: void updateFocus(IPAContext &context, IPAFrameContext &frameContext, double focus); + void step(uint32_t& skip, double& start, double& stop, double& step, double& focus_pos, double& max_pos, uint64_t& max_sharp, uint64_t sharp, bool& sweep); }; } /* namespace ipa::soft::algorithms */ diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index cfc524a13..f59c4006f 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -36,6 +36,10 @@ struct IPASessionConfiguration { } black; struct { int32_t focus_min, focus_max; + double focus_max_pos; + uint64_t sharpness_max; + double start, stop, step; + uint32_t skip; } focus; }; @@ -77,6 +81,8 @@ struct IPAActiveState { std::optional focus_pos; /* 0..1 range, 1 = normal */ std::optional stats_enabled; + /* 0..1 range, 0 = normal */ + std::optional focus_sweep; } knobs; }; diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index c520c806e..a6a73f483 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -147,7 +147,10 @@ static constexpr unsigned int kBlueYMul = 29; /* 0.114 * 256 */ \ uint64_t sumR = 0; \ uint64_t sumG = 0; \ - uint64_t sumB = 0; + uint64_t sumB = 0; \ + pixel_t r0 = 0, r1 = 0, b0 = 0, \ + b1 = 0, g0 = 0, g1 = 0; \ + uint64_t sharpness = 0; #define SWSTATS_ACCUMULATE_LINE_STATS(div) \ sumR += r; \ @@ -157,12 +160,18 @@ static constexpr unsigned int kBlueYMul = 29; /* 0.114 * 256 */ yVal = r * kRedYMul; \ yVal += g * kGreenYMul; \ yVal += b * kBlueYMul; \ - stats_.yHistogram[yVal * SwIspStats::kYHistogramSize / (256 * 256 * (div))]++; + stats_.yHistogram[yVal * SwIspStats::kYHistogramSize / (256 * 256 * (div))]++; \ + if (r0 != 0) \ + sharpness += abs(r - 2*r1 + r0) * kRedYMul + abs(g - 2*g1 + g0) * kGreenYMul + abs(b - 2*b1 + b0) * kBlueYMul; \ + r0 = r1; g0 = g1; b0 = b1; \ + r1 = r; g1 = g; b1 = b; \ + #define SWSTATS_FINISH_LINE_STATS() \ stats_.sumR_ += sumR; \ stats_.sumG_ += sumG; \ - stats_.sumB_ += sumB; + stats_.sumB_ += sumB; \ + stats_.sharpness += sharpness; void SwStatsCpu::statsBGGR8Line0(const uint8_t *src[]) { @@ -306,6 +315,7 @@ void SwStatsCpu::startFrame(void) stats_.sumR_ = 0; stats_.sumB_ = 0; stats_.sumG_ = 0; + stats_.sharpness = 0; stats_.yHistogram.fill(0); } From f6d95130f75473c71ccc0fbd9a6d463d53afb152 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 9 Jul 2025 17:48:32 +0300 Subject: [PATCH 63/64] gstreamer: fix crash on stream stop Signed-off-by: Vasiliy Doylov --- src/gstreamer/gstlibcamerasrc.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 3aca4eeda..da8eb4e5c 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -236,6 +236,8 @@ GstLibcameraSrcState::requestCompleted(Request *request) GLibLocker locker(&lock_); controls_.readMetadata(request); + if(queuedRequests_.empty()) + return; wrap = std::move(queuedRequests_.front()); queuedRequests_.pop(); From 2c5bda6f089cf9a8fa8a919695c33c1cf924332f Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 9 Jul 2025 18:13:29 +0300 Subject: [PATCH 64/64] gstreamer: remove dublicated property Signed-off-by: Vasiliy Doylov --- src/gstreamer/gstlibcameraprovider.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/gstreamer/gstlibcameraprovider.cpp b/src/gstreamer/gstlibcameraprovider.cpp index 5da96ea3f..08862363c 100644 --- a/src/gstreamer/gstlibcameraprovider.cpp +++ b/src/gstreamer/gstlibcameraprovider.cpp @@ -32,7 +32,7 @@ GST_DEBUG_CATEGORY_STATIC(provider_debug); */ enum { - PROP_DEVICE_NAME = 1, + PROP_DEVICE_ = 1, }; #define GST_TYPE_LIBCAMERA_DEVICE gst_libcamera_device_get_type() @@ -76,14 +76,11 @@ gst_libcamera_device_reconfigure_element(GstDevice *device, static void gst_libcamera_device_set_property(GObject *object, guint prop_id, - const GValue *value, GParamSpec *pspec) + [[maybe_unused]]const GValue *value, GParamSpec *pspec) { - GstLibcameraDevice *device = GST_LIBCAMERA_DEVICE(object); + // GstLibcameraDevice *device = GST_LIBCAMERA_DEVICE(object); switch (prop_id) { - case PROP_DEVICE_NAME: - device->name = g_value_dup_string(value); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -117,12 +114,6 @@ gst_libcamera_device_class_init(GstLibcameraDeviceClass *klass) object_class->set_property = gst_libcamera_device_set_property; object_class->finalize = gst_libcamera_device_finalize; - - GParamSpec *pspec = g_param_spec_string("name", "Name", - "The name of the camera device", "", - (GParamFlags)(G_PARAM_STATIC_STRINGS | G_PARAM_WRITABLE | - G_PARAM_CONSTRUCT_ONLY)); - g_object_class_install_property(object_class, PROP_DEVICE_NAME, pspec); } static GstDevice *