From c189fc550473e81d51bdcfc01fc99c8b06fdc647 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:13 +0200 Subject: [PATCH 001/106] libcamera: swstats_cpu: Update statsProcessFn() / processLine0() documentation Update the documentation of the statsProcessFn() / processLine0() src[] pointer argument to take into account that swstats_cpu may also be used with planar input data or with non Bayer single plane input data. The statsProcessFn typedef is private, so no documentation is generated for it. Move the new updated src[] pointer argument documentation to processLine0() so that it gets included in the generated docs. Reviewed-by: Kieran Bingham Reviewed-by: Milan Zamazal Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/swstats_cpu.cpp | 27 +++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index c520c806e..a9a3e77a8 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -71,6 +71,19 @@ namespace libcamera { * patternSize height == 1. * It'll process line 0 and 1 for input formats with patternSize height >= 2. * This function may only be called after a successful setWindow() call. + * + * This function takes an array of src pointers each pointing to a line in + * the source image. + * + * Bayer input data requires (patternSize_.height + 1) src pointers, with + * the middle element of the array pointing to the actual line being processed. + * Earlier element(s) will point to the previous line(s) and later element(s) + * to the next line(s). See the DebayerCpu::debayerFn documentation for details. + * + * Planar input data requires a src pointer for each plane, with src[0] pointing + * to the line in plane 0, etc. + * + * For non Bayer single plane input data only a single src pointer is required. */ /** @@ -89,20 +102,6 @@ namespace libcamera { * \brief Signals that the statistics are ready */ -/** - * \typedef SwStatsCpu::statsProcessFn - * \brief Called when there is data to get statistics from - * \param[in] src The input data - * - * These functions take an array of (patternSize_.height + 1) src - * pointers each pointing to a line in the source image. The middle - * element of the array will point to the actual line being processed. - * Earlier element(s) will point to the previous line(s) and later - * element(s) to the next line(s). - * - * See the documentation of DebayerCpu::debayerFn for more details. - */ - /** * \var unsigned int SwStatsCpu::ySkipMask_ * \brief Skip lines where this bitmask is set in y From 26dba2e048695eab1d11186e7921f9d79e553683 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:14 +0200 Subject: [PATCH 002/106] libcamera: swstats_cpu: Drop patternSize_ documentation patternSize_ is a private variable and its meaning is already documented in the patternSize() getter documentation. Move the list of valid sizes to the patternSize() getter documentation and drop the patternSize_ documentation. While at it also add 1x1 as valid size for use with future support of single plane non Bayer input data. Reviewed-by: Kieran Bingham Reviewed-by: Milan Zamazal Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/swstats_cpu.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index a9a3e77a8..5e4246a9c 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -58,6 +58,8 @@ namespace libcamera { * also indicates if processLine2() should be called or not. * This may only be called after a successful configure() call. * + * Valid sizes are: 1x1, 2x2, 4x2 or 4x4. + * * \return The pattern size */ @@ -112,13 +114,6 @@ namespace libcamera { * \brief Statistics window, set by setWindow(), used every line */ -/** - * \var Size SwStatsCpu::patternSize_ - * \brief The size of the bayer pattern - * - * Valid sizes are: 2x2, 4x2 or 4x4. - */ - /** * \var unsigned int SwStatsCpu::xShift_ * \brief The offset of x, applied to window_.x for bayer variants From d9ffeb0bf11436adcb609e3549926eb5921bd38c Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:15 +0200 Subject: [PATCH 003/106] libcamera: swstats_cpu: Move header to libcamera/internal/software_isp Move the swstats_cpu.h file to include/libcamera/internal/software_isp/ so that it can be used outside the src/libcamera/software_isp/ directory. Reviewed-by: Milan Zamazal Reviewed-by: Kieran Bingham Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/software_isp/meson.build | 1 + .../libcamera/internal}/software_isp/swstats_cpu.h | 0 src/libcamera/software_isp/debayer_cpu.h | 2 +- src/libcamera/software_isp/swstats_cpu.cpp | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) rename {src/libcamera => include/libcamera/internal}/software_isp/swstats_cpu.h (100%) diff --git a/include/libcamera/internal/software_isp/meson.build b/include/libcamera/internal/software_isp/meson.build index 508ddddca..ea3f3f1c1 100644 --- a/include/libcamera/internal/software_isp/meson.build +++ b/include/libcamera/internal/software_isp/meson.build @@ -4,4 +4,5 @@ libcamera_internal_headers += files([ 'debayer_params.h', 'software_isp.h', 'swisp_stats.h', + 'swstats_cpu.h', ]) diff --git a/src/libcamera/software_isp/swstats_cpu.h b/include/libcamera/internal/software_isp/swstats_cpu.h similarity index 100% rename from src/libcamera/software_isp/swstats_cpu.h rename to include/libcamera/internal/software_isp/swstats_cpu.h diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 926195e98..89a89893d 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -18,9 +18,9 @@ #include #include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" #include "debayer.h" -#include "swstats_cpu.h" namespace libcamera { diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index 5e4246a9c..aa5654dcc 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -9,7 +9,7 @@ * CPU based software statistics implementation */ -#include "swstats_cpu.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" #include From ba4218669b46fe8af63b96eaff09e9ae57793721 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:16 +0200 Subject: [PATCH 004/106] libcamera: software_isp: Move benchmark code to its own class Move the code for the builtin benchmark to its own small Benchmark class. Reviewed-by: Kieran Bingham Reviewed-by: Milan Zamazal Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- .../internal/software_isp/benchmark.h | 36 +++++++ .../internal/software_isp/meson.build | 1 + src/libcamera/software_isp/benchmark.cpp | 93 +++++++++++++++++++ src/libcamera/software_isp/debayer_cpu.cpp | 36 +------ src/libcamera/software_isp/debayer_cpu.h | 7 +- src/libcamera/software_isp/meson.build | 1 + 6 files changed, 135 insertions(+), 39 deletions(-) create mode 100644 include/libcamera/internal/software_isp/benchmark.h create mode 100644 src/libcamera/software_isp/benchmark.cpp diff --git a/include/libcamera/internal/software_isp/benchmark.h b/include/libcamera/internal/software_isp/benchmark.h new file mode 100644 index 000000000..8af250154 --- /dev/null +++ b/include/libcamera/internal/software_isp/benchmark.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Authors: + * Hans de Goede + * + * Simple builtin benchmark to measure software ISP processing times + */ + +#pragma once + +#include +#include + +namespace libcamera { + +class Benchmark +{ +public: + Benchmark(); + ~Benchmark(); + + void startFrame(void); + void finishFrame(void); + +private: + unsigned int measuredFrames_; + int64_t frameProcessTime_; + timespec frameStartTime_; + /* Skip 30 frames for things to stabilize then measure 30 frames */ + static constexpr unsigned int kFramesToSkip = 30; + static constexpr unsigned int kLastFrameToMeasure = 60; +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/software_isp/meson.build b/include/libcamera/internal/software_isp/meson.build index ea3f3f1c1..df7c3b97d 100644 --- a/include/libcamera/internal/software_isp/meson.build +++ b/include/libcamera/internal/software_isp/meson.build @@ -1,6 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 libcamera_internal_headers += files([ + 'benchmark.h', 'debayer_params.h', 'software_isp.h', 'swisp_stats.h', diff --git a/src/libcamera/software_isp/benchmark.cpp b/src/libcamera/software_isp/benchmark.cpp new file mode 100644 index 000000000..b3da3c416 --- /dev/null +++ b/src/libcamera/software_isp/benchmark.cpp @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Authors: + * Hans de Goede + * + * Simple builtin benchmark to measure software ISP processing times + */ + +#include "libcamera/internal/software_isp/benchmark.h" + +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(Benchmark) + +/** + * \class Benchmark + * \brief Simple builtin benchmark + * + * Simple builtin benchmark to measure software ISP processing times. + */ + +/** + * \brief Constructs a Benchmark object + */ +Benchmark::Benchmark() + : measuredFrames_(0), frameProcessTime_(0) +{ +} + +Benchmark::~Benchmark() +{ +} + +static inline int64_t timeDiff(timespec &after, timespec &before) +{ + return (after.tv_sec - before.tv_sec) * 1000000000LL + + (int64_t)after.tv_nsec - (int64_t)before.tv_nsec; +} + +/** + * \brief Start measuring process time for a single frame + * + * Call this function before processing frame data to start measuring + * the process time for a frame. + */ +void Benchmark::startFrame(void) +{ + if (measuredFrames_ >= Benchmark::kLastFrameToMeasure) + return; + + frameStartTime_ = {}; + clock_gettime(CLOCK_MONOTONIC_RAW, &frameStartTime_); +} + +/** + * \brief Finish measuring process time for a single frame + * + * Call this function after processing frame data to finish measuring + * the process time for a frame. + * + * This function will log frame processing time information after + * Benchmark::kLastFrameToMeasure frames have been processed. + */ +void Benchmark::finishFrame(void) +{ + if (measuredFrames_ >= Benchmark::kLastFrameToMeasure) + return; + + measuredFrames_++; + + if (measuredFrames_ <= Benchmark::kFramesToSkip) + return; + + timespec frameEndTime = {}; + clock_gettime(CLOCK_MONOTONIC_RAW, &frameEndTime); + frameProcessTime_ += timeDiff(frameEndTime, frameStartTime_); + + if (measuredFrames_ == Benchmark::kLastFrameToMeasure) { + const unsigned int measuredFrames = Benchmark::kLastFrameToMeasure - + Benchmark::kFramesToSkip; + LOG(Benchmark, Info) + << "Processed " << measuredFrames + << " frames in " << frameProcessTime_ / 1000 << "us, " + << frameProcessTime_ / (1000 * measuredFrames) + << " us/frame"; + } +} + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 66f6038c1..8d30bf4a2 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -554,9 +554,6 @@ int DebayerCpu::configure(const StreamConfiguration &inputCfg, lineBuffers_[i].resize(lineBufferLength_); } - measuredFrames_ = 0; - frameProcessTime_ = 0; - return 0; } @@ -746,24 +743,9 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) } } -namespace { - -inline int64_t timeDiff(timespec &after, timespec &before) -{ - return (after.tv_sec - before.tv_sec) * 1000000000LL + - (int64_t)after.tv_nsec - (int64_t)before.tv_nsec; -} - -} /* namespace */ - void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params) { - timespec frameStartTime; - - if (measuredFrames_ < DebayerCpu::kLastFrameToMeasure) { - frameStartTime = {}; - clock_gettime(CLOCK_MONOTONIC_RAW, &frameStartTime); - } + bench_.startFrame(); std::vector dmaSyncers; for (const FrameBuffer::Plane &plane : input->planes()) @@ -817,21 +799,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output dmaSyncers.clear(); /* Measure before emitting signals */ - if (measuredFrames_ < DebayerCpu::kLastFrameToMeasure && - ++measuredFrames_ > DebayerCpu::kFramesToSkip) { - timespec frameEndTime = {}; - clock_gettime(CLOCK_MONOTONIC_RAW, &frameEndTime); - frameProcessTime_ += timeDiff(frameEndTime, frameStartTime); - if (measuredFrames_ == DebayerCpu::kLastFrameToMeasure) { - const unsigned int measuredFrames = DebayerCpu::kLastFrameToMeasure - - DebayerCpu::kFramesToSkip; - LOG(Debayer, Info) - << "Processed " << measuredFrames - << " frames in " << frameProcessTime_ / 1000 << "us, " - << frameProcessTime_ / (1000 * measuredFrames) - << " us/frame"; - } - } + bench_.finishFrame(); /* * Buffer ids are currently not used, so pass zeros as its parameter. diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 89a89893d..182607cda 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -17,6 +17,7 @@ #include +#include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/bayer_format.h" #include "libcamera/internal/software_isp/swstats_cpu.h" @@ -160,11 +161,7 @@ private: unsigned int xShift_; /* Offset of 0/1 applied to window_.x */ bool enableInputMemcpy_; bool swapRedBlueGains_; - unsigned int measuredFrames_; - int64_t frameProcessTime_; - /* Skip 30 frames for things to stabilize then measure 30 frames */ - static constexpr unsigned int kFramesToSkip = 30; - static constexpr unsigned int kLastFrameToMeasure = 60; + Benchmark bench_; }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/meson.build b/src/libcamera/software_isp/meson.build index aac7eda7b..59fa5f02a 100644 --- a/src/libcamera/software_isp/meson.build +++ b/src/libcamera/software_isp/meson.build @@ -8,6 +8,7 @@ if not softisp_enabled endif libcamera_internal_sources += files([ + 'benchmark.cpp', 'debayer.cpp', 'debayer_cpu.cpp', 'software_isp.cpp', From d4bc61f716328d59094f730a69354835137078b0 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:17 +0200 Subject: [PATCH 005/106] libcamera: swstats_cpu: Add processFrame() method Add a method to the SwstatsCpu class to process a whole Framebuffer in one go, rather then line by line. This is useful for gathering stats when debayering is not necessary or is not done on the CPU. Reviewed-by: Milan Zamazal Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- .../internal/software_isp/swstats_cpu.h | 12 +++++ src/libcamera/software_isp/swstats_cpu.cpp | 51 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/include/libcamera/internal/software_isp/swstats_cpu.h b/include/libcamera/internal/software_isp/swstats_cpu.h index 26a2f462e..fa47cec91 100644 --- a/include/libcamera/internal/software_isp/swstats_cpu.h +++ b/include/libcamera/internal/software_isp/swstats_cpu.h @@ -18,12 +18,16 @@ #include #include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/framebuffer.h" #include "libcamera/internal/shared_mem_object.h" #include "libcamera/internal/software_isp/swisp_stats.h" +#include "benchmark.h" + namespace libcamera { class PixelFormat; +class MappedFrameBuffer; struct StreamConfiguration; class SwStatsCpu @@ -42,6 +46,7 @@ public: void setWindow(const Rectangle &window); void startFrame(); void finishFrame(uint32_t frame, uint32_t bufferId); + void processFrame(uint32_t frame, uint32_t bufferId, FrameBuffer *input); void processLine0(unsigned int y, const uint8_t *src[]) { @@ -65,6 +70,7 @@ public: private: using statsProcessFn = void (SwStatsCpu::*)(const uint8_t *src[]); + using processFrameFn = void (SwStatsCpu::*)(MappedFrameBuffer &in); int setupStandardBayerOrder(BayerFormat::Order order); /* Bayer 8 bpp unpacked */ @@ -77,6 +83,10 @@ private: void statsBGGR10PLine0(const uint8_t *src[]); void statsGBRG10PLine0(const uint8_t *src[]); + void processBayerFrame2(MappedFrameBuffer &in); + + processFrameFn processFrame_; + /* Variables set by configure(), used every line */ statsProcessFn stats0_; statsProcessFn stats2_; @@ -89,9 +99,11 @@ private: Size patternSize_; unsigned int xShift_; + unsigned int stride_; SharedMemObject sharedStats_; SwIspStats stats_; + Benchmark bench_; }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index aa5654dcc..1ff15f5b7 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -16,6 +16,7 @@ #include #include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/mapped_framebuffer.h" namespace libcamera { @@ -360,11 +361,14 @@ int SwStatsCpu::setupStandardBayerOrder(BayerFormat::Order order) */ int SwStatsCpu::configure(const StreamConfiguration &inputCfg) { + stride_ = inputCfg.stride; + BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputCfg.pixelFormat); if (bayerFormat.packing == BayerFormat::Packing::None && setupStandardBayerOrder(bayerFormat.order) == 0) { + processFrame_ = &SwStatsCpu::processBayerFrame2; switch (bayerFormat.bitDepth) { case 8: stats0_ = &SwStatsCpu::statsBGGR8Line0; @@ -385,6 +389,7 @@ int SwStatsCpu::configure(const StreamConfiguration &inputCfg) /* Skip every 3th and 4th line, sample every other 2x2 block */ ySkipMask_ = 0x02; xShift_ = 0; + processFrame_ = &SwStatsCpu::processBayerFrame2; switch (bayerFormat.order) { case BayerFormat::BGGR: @@ -425,4 +430,50 @@ void SwStatsCpu::setWindow(const Rectangle &window) window_.height &= ~(patternSize_.height - 1); } +void SwStatsCpu::processBayerFrame2(MappedFrameBuffer &in) +{ + const uint8_t *src = in.planes()[0].data(); + const uint8_t *linePointers[3]; + + /* Adjust src for starting at window_.y */ + src += window_.y * stride_; + + for (unsigned int y = 0; y < window_.height; y += 2) { + if (y & ySkipMask_) { + src += stride_ * 2; + continue; + } + + /* linePointers[0] is not used by any stats0_ functions */ + linePointers[1] = src; + linePointers[2] = src + stride_; + (this->*stats0_)(linePointers); + src += stride_ * 2; + } +} + +/** + * \brief Calculate statistics for a frame in one go + * \param[in] frame The frame number + * \param[in] bufferId ID of the statistics buffer + * \param[in] input The frame to process + * + * This may only be called after a successful setWindow() call. + */ +void SwStatsCpu::processFrame(uint32_t frame, uint32_t bufferId, FrameBuffer *input) +{ + bench_.startFrame(); + startFrame(); + + MappedFrameBuffer in(input, MappedFrameBuffer::MapFlag::Read); + if (!in.isValid()) { + LOG(SwStatsCpu, Error) << "mmap-ing buffer(s) failed"; + return; + } + + (this->*processFrame_)(in); + finishFrame(frame, bufferId); + bench_.finishFrame(); +} + } /* namespace libcamera */ From 1142f89aae2d89f6190675e5129198b187eb3f95 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 21 Sep 2024 12:41:42 +0100 Subject: [PATCH 006/106] libcamera: MappedFrameBuffer: Latch a pointer to the framebuffer Take a pointer to the given framebuffer as a private member for further use in later patches. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/mapped_framebuffer.h | 3 +++ src/libcamera/mapped_framebuffer.cpp | 2 ++ 2 files changed, 5 insertions(+) diff --git a/include/libcamera/internal/mapped_framebuffer.h b/include/libcamera/internal/mapped_framebuffer.h index 6aaabf508..75ac2c8f6 100644 --- a/include/libcamera/internal/mapped_framebuffer.h +++ b/include/libcamera/internal/mapped_framebuffer.h @@ -55,6 +55,9 @@ public: using MapFlags = Flags; MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags); + +private: + const FrameBuffer *buffer_; }; LIBCAMERA_FLAGS_ENABLE_OPERATORS(MappedFrameBuffer::MapFlag) diff --git a/src/libcamera/mapped_framebuffer.cpp b/src/libcamera/mapped_framebuffer.cpp index f54bbf21f..f5ee770c6 100644 --- a/src/libcamera/mapped_framebuffer.cpp +++ b/src/libcamera/mapped_framebuffer.cpp @@ -238,6 +238,8 @@ MappedFrameBuffer::MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags) planes_.emplace_back(info.address + plane.offset, plane.length); } + + buffer_ = buffer; } } /* namespace libcamera */ From 2929cdcc58a9730eff287a86fae0bddb3a312676 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 11 May 2025 15:16:44 +0100 Subject: [PATCH 007/106] libcamera: MappedFrameBuffer: Add MappedFrameBuffer::getPlaneFD() Add MappedFrameBuffer::getPlaneFD() which takes a plane index and returns the file descriptor associated with it. This fd will be used to feed into eglCreateImageKHR for both texture creation on upload and directly render-to-texture where the texture buffer comes from the fd given to eglCreateImageKHR. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/mapped_framebuffer.h | 1 + src/libcamera/mapped_framebuffer.cpp | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/include/libcamera/internal/mapped_framebuffer.h b/include/libcamera/internal/mapped_framebuffer.h index 75ac2c8f6..9a5355c76 100644 --- a/include/libcamera/internal/mapped_framebuffer.h +++ b/include/libcamera/internal/mapped_framebuffer.h @@ -55,6 +55,7 @@ public: using MapFlags = Flags; MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags); + int getPlaneFD(int plane); private: const FrameBuffer *buffer_; diff --git a/src/libcamera/mapped_framebuffer.cpp b/src/libcamera/mapped_framebuffer.cpp index f5ee770c6..d5f347d4d 100644 --- a/src/libcamera/mapped_framebuffer.cpp +++ b/src/libcamera/mapped_framebuffer.cpp @@ -242,4 +242,9 @@ MappedFrameBuffer::MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags) buffer_ = buffer; } +int MappedFrameBuffer::getPlaneFD(int plane) +{ + return buffer_->planes()[plane].fd.get(); +} + } /* namespace libcamera */ From 633063e099ccb0618359f9ecde9b05852c34259d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Tue, 4 Mar 2025 13:42:19 +0100 Subject: [PATCH 008/106] android: camera_device: Do not pass `nullptr` to `Request::addBuffer()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default argument already takes care of passing no fence to `addBuffer()`, so there is no reason to specify `nullptr` explicitly. Signed-off-by: Barnabás Pőcze Reviewed-by: Laurent Pinchart Reviewed-by: Kieran Bingham --- src/android/camera_device.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index a038131ae..80ff248c2 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -1079,7 +1079,7 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques buffer.internalBuffer = frameBuffer; descriptor->request_->addBuffer(sourceStream->stream(), - frameBuffer, nullptr); + frameBuffer); requestedStreams.insert(sourceStream); } From 081554db346a9ff2e610be1cff8d463ff3bc3b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Wed, 22 Jan 2025 12:27:12 +0100 Subject: [PATCH 009/106] libcamera: process: Disable copy/move MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `Process` object has address identity because a pointer to it is stored inside the `ProcessManager`. However, copy/move special methods are still generated by the compiler. So disable them to avoid potential issues and confusion. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- include/libcamera/internal/process.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/libcamera/internal/process.h b/include/libcamera/internal/process.h index b1d07a5a5..6c34aef2f 100644 --- a/include/libcamera/internal/process.h +++ b/include/libcamera/internal/process.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -42,6 +43,8 @@ public: Signal finished; private: + LIBCAMERA_DISABLE_COPY_AND_MOVE(Process) + void closeAllFdsExcept(const std::vector &fds); int isolate(); void died(int wstatus); From 0a591eaf8c36ffd34dc0d69e6221c4fe88fd4a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Mon, 24 Mar 2025 12:39:11 +0100 Subject: [PATCH 010/106] libcamera: process: Misc. cleanup around `execv()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Firstly, get the number of arguments first, and use that to determine the size of the allocation instead of retrieving it twice. Secondly, use `const_cast` instead of a C-style cast when calling `execv()`. Third, use `size_t` to match the type of `args.size()`. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/libcamera/process.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libcamera/process.cpp b/src/libcamera/process.cpp index d836fb07a..f4e4b004c 100644 --- a/src/libcamera/process.cpp +++ b/src/libcamera/process.cpp @@ -279,14 +279,15 @@ int Process::start(const std::string &path, if (file && strcmp(file, "syslog")) unsetenv("LIBCAMERA_LOG_FILE"); - const char **argv = new const char *[args.size() + 2]; - unsigned int len = args.size(); + const size_t len = args.size(); + auto argv = std::make_unique(len + 2); + argv[0] = path.c_str(); - for (unsigned int i = 0; i < len; i++) + for (size_t i = 0; i < len; i++) argv[i + 1] = args[i].c_str(); argv[len + 1] = nullptr; - execv(path.c_str(), (char **)argv); + execv(path.c_str(), const_cast(argv.get())); _exit(EXIT_FAILURE); } From fae2b506d7ce8df6149e1eedf147cb35838fbb16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Tue, 25 Mar 2025 18:02:59 +0100 Subject: [PATCH 011/106] libcamera: process: Return error if already running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Returning 0 when a running process is already managed can be confusing since the parameters might be completely different, causing the caller to mistakenly assume that the program it specified has been started. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/libcamera/process.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/process.cpp b/src/libcamera/process.cpp index f4e4b004c..edbe8664a 100644 --- a/src/libcamera/process.cpp +++ b/src/libcamera/process.cpp @@ -241,7 +241,7 @@ int Process::start(const std::string &path, int ret; if (running_) - return 0; + return -EBUSY; int childPid = fork(); if (childPid == -1) { From 8d168f33486951c7121b923bd28ce5d4a9494099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Mon, 24 Mar 2025 16:05:44 +0100 Subject: [PATCH 012/106] libcamera: process: Ensure that file descriptors are nonnegative MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Return `-EINVAL` from `Process::start()` if any of the file descriptors are negative as those most likely signal some kind of issue such as missed error checking. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/libcamera/process.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libcamera/process.cpp b/src/libcamera/process.cpp index edbe8664a..0eae68072 100644 --- a/src/libcamera/process.cpp +++ b/src/libcamera/process.cpp @@ -243,6 +243,11 @@ int Process::start(const std::string &path, if (running_) return -EBUSY; + for (int fd : fds) { + if (fd < 0) + return -EINVAL; + } + int childPid = fork(); if (childPid == -1) { ret = -errno; @@ -298,6 +303,8 @@ void Process::closeAllFdsExcept(const std::vector &fds) std::vector v(fds); sort(v.begin(), v.end()); + ASSERT(v.empty() || v.front() >= 0); + DIR *dir = opendir("/proc/self/fd"); if (!dir) return; From 831a7fefa92d323e2f75e1d62baeecb537696280 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 19 May 2024 00:10:16 +0100 Subject: [PATCH 013/106] libcamera: software_isp: Move useful items from DebayerCpu to Debayer base class The DebayerCpu class has a number of variables, embedded structures and methods which are useful to DebayerGpu implementation. Move relevant variables and methods to base class. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.h | 33 ++++++++++++++++++++++- src/libcamera/software_isp/debayer_cpu.h | 34 +----------------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index ba033d440..45da822a2 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -14,11 +14,13 @@ #include #include +#include #include #include #include +#include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/software_isp/debayer_params.h" namespace libcamera { @@ -27,7 +29,7 @@ class FrameBuffer; LOG_DECLARE_CATEGORY(Debayer) -class Debayer +class Debayer : public Object { public: virtual ~Debayer() = 0; @@ -45,9 +47,38 @@ public: virtual SizeRange sizes(PixelFormat inputFormat, const Size &inputSize) = 0; + virtual const SharedFD &getStatsFD() = 0; + + unsigned int frameSize() { return outputConfig_.frameSize; } + Signal inputBufferReady; Signal outputBufferReady; + struct DebayerInputConfig { + Size patternSize; + unsigned int bpp; /* Memory used per pixel, not precision */ + unsigned int stride; + std::vector outputFormats; + }; + + struct DebayerOutputConfig { + unsigned int bpp; /* Memory used per pixel, not precision */ + unsigned int stride; + unsigned int frameSize; + }; + + DebayerInputConfig inputConfig_; + DebayerOutputConfig outputConfig_; + DebayerParams::LookupTable red_; + DebayerParams::LookupTable green_; + DebayerParams::LookupTable blue_; + DebayerParams::CcmLookupTable redCcm_; + DebayerParams::CcmLookupTable greenCcm_; + DebayerParams::CcmLookupTable blueCcm_; + DebayerParams::LookupTable gammaLut_; + bool swapRedBlueGains_; + Benchmark bench_; + private: virtual Size patternSize(PixelFormat inputFormat) = 0; }; diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 182607cda..0b4b16e1a 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -17,7 +17,6 @@ #include -#include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/bayer_format.h" #include "libcamera/internal/software_isp/swstats_cpu.h" @@ -25,7 +24,7 @@ namespace libcamera { -class DebayerCpu : public Debayer, public Object +class DebayerCpu : public Debayer { public: DebayerCpu(std::unique_ptr stats); @@ -48,13 +47,6 @@ public: */ const SharedFD &getStatsFD() { return stats_->getStatsFD(); } - /** - * \brief Get the output frame size - * - * \return The output frame size - */ - unsigned int frameSize() { return outputConfig_.frameSize; } - private: /** * \brief Called to debayer 1 line of Bayer input data to output format @@ -111,19 +103,6 @@ private: template void debayer10P_RGRG_BGR888(uint8_t *dst, const uint8_t *src[]); - struct DebayerInputConfig { - Size patternSize; - unsigned int bpp; /* Memory used per pixel, not precision */ - unsigned int stride; - std::vector outputFormats; - }; - - struct DebayerOutputConfig { - unsigned int bpp; /* Memory used per pixel, not precision */ - unsigned int stride; - unsigned int frameSize; - }; - int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); int setupStandardBayerOrder(BayerFormat::Order order); @@ -139,20 +118,11 @@ private: /* Max. supported Bayer pattern height is 4, debayering this requires 5 lines */ static constexpr unsigned int kMaxLineBuffers = 5; - DebayerParams::LookupTable red_; - DebayerParams::LookupTable green_; - DebayerParams::LookupTable blue_; - DebayerParams::CcmLookupTable redCcm_; - DebayerParams::CcmLookupTable greenCcm_; - DebayerParams::CcmLookupTable blueCcm_; - DebayerParams::LookupTable gammaLut_; debayerFn debayer0_; debayerFn debayer1_; debayerFn debayer2_; debayerFn debayer3_; Rectangle window_; - DebayerInputConfig inputConfig_; - DebayerOutputConfig outputConfig_; std::unique_ptr stats_; std::vector lineBuffers_[kMaxLineBuffers]; unsigned int lineBufferLength_; @@ -160,8 +130,6 @@ private: unsigned int lineBufferIndex_; unsigned int xShift_; /* Offset of 0/1 applied to window_.x */ bool enableInputMemcpy_; - bool swapRedBlueGains_; - Benchmark bench_; }; } /* namespace libcamera */ From 3c0a21fd30a05556e05ff8166cf1f45bf2c12e6c Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 29 Apr 2025 12:49:49 +0100 Subject: [PATCH 014/106] libcamera: software_isp: Move Bayer parans init from DebayerCpu to Debayer Move the initialisation of Bayer params and CCM to a new constructor in the Debayer class. Ensure we call the base class constructor from DebayerCpu's constructor in the expected constructor order Debayer then DebayerCpu. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.cpp | 11 +++++++++++ src/libcamera/software_isp/debayer.h | 1 + src/libcamera/software_isp/debayer_cpu.cpp | 10 +--------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libcamera/software_isp/debayer.cpp b/src/libcamera/software_isp/debayer.cpp index e9e18c488..29fdcbbfd 100644 --- a/src/libcamera/software_isp/debayer.cpp +++ b/src/libcamera/software_isp/debayer.cpp @@ -103,6 +103,17 @@ namespace libcamera { LOG_DEFINE_CATEGORY(Debayer) +Debayer::Debayer() +{ + /* Initialize color lookup tables */ + for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { + red_[i] = green_[i] = blue_[i] = i; + redCcm_[i] = { static_cast(i), 0, 0 }; + greenCcm_[i] = { 0, static_cast(i), 0 }; + blueCcm_[i] = { 0, 0, static_cast(i) }; + } +} + Debayer::~Debayer() { } diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index 45da822a2..01b6e916f 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -32,6 +32,7 @@ LOG_DECLARE_CATEGORY(Debayer) class Debayer : public Object { public: + Debayer(); virtual ~Debayer() = 0; virtual int configure(const StreamConfiguration &inputCfg, diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 8d30bf4a2..13db8a8c3 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -40,7 +40,7 @@ namespace libcamera { * \param[in] stats Pointer to the stats object to use */ DebayerCpu::DebayerCpu(std::unique_ptr stats) - : stats_(std::move(stats)) + : Debayer(), stats_(std::move(stats)) { /* * Reading from uncached buffers may be very slow. @@ -51,14 +51,6 @@ DebayerCpu::DebayerCpu(std::unique_ptr stats) * future. */ enableInputMemcpy_ = true; - - /* Initialize color lookup tables */ - for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { - red_[i] = green_[i] = blue_[i] = i; - redCcm_[i] = { static_cast(i), 0, 0 }; - greenCcm_[i] = { 0, static_cast(i), 0 }; - blueCcm_[i] = { 0, 0, static_cast(i) }; - } } DebayerCpu::~DebayerCpu() = default; From c3a8493a12f26c59d59057ee4a0bb38337ea556f Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 20 Apr 2025 01:02:25 +0100 Subject: [PATCH 015/106] libcamera: software_isp: Move param select code to Debayer base class Move the parameter selection code into the Debayer base class in-order to facilitate reuse of the lookup tables in the eGL shaders. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.cpp | 40 ++++++++++++++++++++++ src/libcamera/software_isp/debayer.h | 5 +++ src/libcamera/software_isp/debayer_cpu.cpp | 25 ++------------ 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/libcamera/software_isp/debayer.cpp b/src/libcamera/software_isp/debayer.cpp index 29fdcbbfd..75e4bffab 100644 --- a/src/libcamera/software_isp/debayer.cpp +++ b/src/libcamera/software_isp/debayer.cpp @@ -187,4 +187,44 @@ Debayer::~Debayer() * \brief Signals when the output buffer is ready */ +/** + * \fn void Debayer::setParams(DebayerParams ¶ms) + * \brief Select the bayer params to use for the next frame debayer + * \param[in] params The parameters to be used in debayering + */ +void Debayer::setParams(DebayerParams ¶ms) +{ + green_ = params.green; + greenCcm_ = params.greenCcm; + if (swapRedBlueGains_) { + red_ = params.blue; + blue_ = params.red; + redCcm_ = params.blueCcm; + blueCcm_ = params.redCcm; + for (unsigned int i = 0; i < 256; i++) { + std::swap(redCcm_[i].r, redCcm_[i].b); + std::swap(blueCcm_[i].r, blueCcm_[i].b); + } + } else { + red_ = params.red; + blue_ = params.blue; + redCcm_ = params.redCcm; + blueCcm_ = params.blueCcm; + } + gammaLut_ = params.gammaLut; +} + +/** + * \fn void Debayer::dmaSyncBegin(DebayerParams ¶ms) + * \brief Common CPU/GPU Dma Sync Buffer begin + */ +void Debayer::dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output) +{ + for (const FrameBuffer::Plane &plane : input->planes()) + dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Read); + + for (const FrameBuffer::Plane &plane : output->planes()) + dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); +} + } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index 01b6e916f..0af66b556 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -20,6 +20,7 @@ #include #include +#include "libcamera/internal/dma_buf_allocator.h" #include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/software_isp/debayer_params.h" @@ -82,6 +83,10 @@ public: private: virtual Size patternSize(PixelFormat inputFormat) = 0; + +protected: + void setParams(DebayerParams ¶ms); + void dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output); }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 13db8a8c3..4ef573a37 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -22,7 +22,6 @@ #include #include "libcamera/internal/bayer_format.h" -#include "libcamera/internal/dma_buf_allocator.h" #include "libcamera/internal/framebuffer.h" #include "libcamera/internal/mapped_framebuffer.h" @@ -740,30 +739,10 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output bench_.startFrame(); std::vector dmaSyncers; - for (const FrameBuffer::Plane &plane : input->planes()) - dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Read); - for (const FrameBuffer::Plane &plane : output->planes()) - dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); + dmaSyncBegin(dmaSyncers, input, output); - green_ = params.green; - greenCcm_ = params.greenCcm; - if (swapRedBlueGains_) { - red_ = params.blue; - blue_ = params.red; - redCcm_ = params.blueCcm; - blueCcm_ = params.redCcm; - for (unsigned int i = 0; i < 256; i++) { - std::swap(redCcm_[i].r, redCcm_[i].b); - std::swap(blueCcm_[i].r, blueCcm_[i].b); - } - } else { - red_ = params.red; - blue_ = params.blue; - redCcm_ = params.redCcm; - blueCcm_ = params.blueCcm; - } - gammaLut_ = params.gammaLut; + setParams(params); /* Copy metadata from the input buffer */ FrameMetadata &metadata = output->_d()->metadata(); From 8ca941b7b590fd44ad4e66541eb2f3e155705485 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 27 May 2025 16:02:37 +0100 Subject: [PATCH 016/106] libcamera: software_isp: Move isStandardBayerOrder to base class isStandardBayerOrder is useful to both CPU and GPU debayer logic and reusable as-is for both. Move to shared location in base class. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.cpp | 10 ++++++++++ src/libcamera/software_isp/debayer.h | 2 ++ src/libcamera/software_isp/debayer_cpu.cpp | 6 ------ src/libcamera/software_isp/debayer_cpu.h | 1 - 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/libcamera/software_isp/debayer.cpp b/src/libcamera/software_isp/debayer.cpp index 75e4bffab..d0e17d20b 100644 --- a/src/libcamera/software_isp/debayer.cpp +++ b/src/libcamera/software_isp/debayer.cpp @@ -227,4 +227,14 @@ void Debayer::dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *inpu dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); } +/** + * \fn void Debayer::isStandardBayerOrder(BayerFormat::Order order) + * \brief Common method to validate standard Bayer order + */ +bool Debayer::isStandardBayerOrder(BayerFormat::Order order) +{ + return order == BayerFormat::BGGR || order == BayerFormat::GBRG || + order == BayerFormat::GRBG || order == BayerFormat::RGGB; +} + } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index 0af66b556..214bcdd3c 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -20,6 +20,7 @@ #include #include +#include "libcamera/internal/bayer_format.h" #include "libcamera/internal/dma_buf_allocator.h" #include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/software_isp/debayer_params.h" @@ -87,6 +88,7 @@ private: protected: void setParams(DebayerParams ¶ms); void dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output); + bool isStandardBayerOrder(BayerFormat::Order order); }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 4ef573a37..e56492848 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -282,12 +282,6 @@ void DebayerCpu::debayer10P_RGRG_BGR888(uint8_t *dst, const uint8_t *src[]) } } -static bool isStandardBayerOrder(BayerFormat::Order order) -{ - return order == BayerFormat::BGGR || order == BayerFormat::GBRG || - order == BayerFormat::GRBG || order == BayerFormat::RGGB; -} - /* * Setup the Debayer object according to the passed in parameters. * Return 0 on success, a negative errno value on failure diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 0b4b16e1a..a043a9541 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -17,7 +17,6 @@ #include -#include "libcamera/internal/bayer_format.h" #include "libcamera/internal/software_isp/swstats_cpu.h" #include "debayer.h" From 87ccbc69caf22dde18668b6e442c90c50b609f5e Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Mon, 16 Dec 2024 17:03:36 +0000 Subject: [PATCH 017/106] libcamera: software_isp: Start the ISP thread in configure OpenGL is not thread-safe and in fact associates invisible handles with the threadid of the calling context. As a result we need to make configure() and process() in SoftISP execute on the same thread. Move start thread into configure() as a first step towards this. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/software_isp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index 28e2a360e..7bee8f068 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -159,8 +159,6 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor, metadataReady.emit(frame, metadata); }); ipa_->setSensorControls.connect(this, &SoftwareIsp::setSensorCtrls); - - debayer_->moveToThread(&ispWorkerThread_); } SoftwareIsp::~SoftwareIsp() @@ -262,6 +260,9 @@ int SoftwareIsp::configure(const StreamConfiguration &inputCfg, if (ret < 0) return ret; + debayer_->moveToThread(&ispWorkerThread_); + ispWorkerThread_.start(); + return debayer_->configure(inputCfg, outputCfgs, ccmEnabled_); } @@ -343,7 +344,6 @@ int SoftwareIsp::start() if (ret) return ret; - ispWorkerThread_.start(); return 0; } From 355fd41dfdb5101388598535266292de35b95b33 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 18 Dec 2024 00:59:39 +0000 Subject: [PATCH 018/106] libcamera: software_isp: Move configure to worker thread OpenGL requires both configure() and process() to operate on the same thread. As preparation for that, move current CPU configure into the WorkerThread with a ConnectionTypeBlocking invocation of &DebayerCpu::configure. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/software_isp.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index 7bee8f068..e8fa8a17a 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -263,7 +263,15 @@ int SoftwareIsp::configure(const StreamConfiguration &inputCfg, debayer_->moveToThread(&ispWorkerThread_); ispWorkerThread_.start(); - return debayer_->configure(inputCfg, outputCfgs, ccmEnabled_); + ret = debayer_->invokeMethod(&Debayer::configure, + ConnectionTypeBlocking, inputCfg, + outputCfgs, ccmEnabled_); + if (ret) { + ispWorkerThread_.exit(); + ispWorkerThread_.wait(); + } + + return ret; } /** @@ -386,7 +394,7 @@ void SoftwareIsp::stop() void SoftwareIsp::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output) { ipa_->computeParams(frame); - debayer_->invokeMethod(&DebayerCpu::process, + debayer_->invokeMethod(&Debayer::process, ConnectionTypeQueued, frame, input, output, debayerParams_); } From 3142e3761932bc340e81ff8b335af27bd0e6590f Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 22 Apr 2025 22:25:37 +0100 Subject: [PATCH 019/106] libcamera: software_isp: debayer: Make the debayer_ object of type class Debayer not DebayerCpu Make the type of object Debayer not DebayerCpu thus allowing us to assign the object to either DebayerCpu or DebayerEGL. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/software_isp/software_isp.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/libcamera/internal/software_isp/software_isp.h b/include/libcamera/internal/software_isp/software_isp.h index 786246592..ad89c9b3c 100644 --- a/include/libcamera/internal/software_isp/software_isp.h +++ b/include/libcamera/internal/software_isp/software_isp.h @@ -37,7 +37,7 @@ namespace libcamera { -class DebayerCpu; +class Debayer; class FrameBuffer; class PixelFormat; class Stream; @@ -94,8 +94,7 @@ private: void statsReady(uint32_t frame, uint32_t bufferId); void inputReady(FrameBuffer *input); void outputReady(FrameBuffer *output); - - std::unique_ptr debayer_; + std::unique_ptr debayer_; Thread ispWorkerThread_; SharedMemObject sharedParams_; DebayerParams debayerParams_; From c2a68a2e447f1d7d411ebcbc984b62305d635b03 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 9 May 2025 14:34:27 +0100 Subject: [PATCH 020/106] libcamera: software_isp: debayer: Extend DebayerParams struct to hold a copy of per-frame CCM values Add an element to the DebayerParams structure to capture the calculated CCM. This allows us to pass the CCM into the eGL Debayer shader later on. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/software_isp/debayer_params.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/libcamera/internal/software_isp/debayer_params.h b/include/libcamera/internal/software_isp/debayer_params.h index 217cd5d92..56803195e 100644 --- a/include/libcamera/internal/software_isp/debayer_params.h +++ b/include/libcamera/internal/software_isp/debayer_params.h @@ -13,6 +13,8 @@ #include #include +#include "libcamera/internal/matrix.h" + namespace libcamera { struct DebayerParams { @@ -49,6 +51,11 @@ struct DebayerParams { CcmLookupTable greenCcm; CcmLookupTable blueCcm; LookupTable gammaLut; + + /* + * Per frame CCM values as calcualted by the IPA + */ + Matrix ccm; }; } /* namespace libcamera */ From 367b29199bef1d158dfa0ce77cf8d6ef20003a86 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 24 Jan 2024 15:06:00 +0000 Subject: [PATCH 021/106] libcamera: shaders: Move GL shader programs to src/libcamera/assets/shader Moving the GL shaders to src/libcamera/assets/shader to allow for reuse of these inside of the SoftISP. Signed-off-by: Bryan O'Donoghue --- .../libcamera/internal/shaders}/RGB.frag | 0 .../internal/shaders}/YUV_2_planes.frag | 0 .../internal/shaders}/YUV_3_planes.frag | 0 .../internal/shaders}/YUV_packed.frag | 0 .../internal/shaders}/bayer_1x_packed.frag | 0 .../libcamera/internal/shaders}/bayer_8.frag | 0 .../libcamera/internal/shaders}/bayer_8.vert | 0 .../libcamera/internal/shaders}/identity.vert | 0 src/apps/qcam/assets/shader/shaders.qrc | 16 ++--- src/apps/qcam/viewfinder_gl.cpp | 70 +++++++++---------- 10 files changed, 43 insertions(+), 43 deletions(-) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/RGB.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/YUV_2_planes.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/YUV_3_planes.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/YUV_packed.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/bayer_1x_packed.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/bayer_8.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/bayer_8.vert (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/identity.vert (100%) diff --git a/src/apps/qcam/assets/shader/RGB.frag b/include/libcamera/internal/shaders/RGB.frag similarity index 100% rename from src/apps/qcam/assets/shader/RGB.frag rename to include/libcamera/internal/shaders/RGB.frag diff --git a/src/apps/qcam/assets/shader/YUV_2_planes.frag b/include/libcamera/internal/shaders/YUV_2_planes.frag similarity index 100% rename from src/apps/qcam/assets/shader/YUV_2_planes.frag rename to include/libcamera/internal/shaders/YUV_2_planes.frag diff --git a/src/apps/qcam/assets/shader/YUV_3_planes.frag b/include/libcamera/internal/shaders/YUV_3_planes.frag similarity index 100% rename from src/apps/qcam/assets/shader/YUV_3_planes.frag rename to include/libcamera/internal/shaders/YUV_3_planes.frag diff --git a/src/apps/qcam/assets/shader/YUV_packed.frag b/include/libcamera/internal/shaders/YUV_packed.frag similarity index 100% rename from src/apps/qcam/assets/shader/YUV_packed.frag rename to include/libcamera/internal/shaders/YUV_packed.frag diff --git a/src/apps/qcam/assets/shader/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag similarity index 100% rename from src/apps/qcam/assets/shader/bayer_1x_packed.frag rename to include/libcamera/internal/shaders/bayer_1x_packed.frag diff --git a/src/apps/qcam/assets/shader/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag similarity index 100% rename from src/apps/qcam/assets/shader/bayer_8.frag rename to include/libcamera/internal/shaders/bayer_8.frag diff --git a/src/apps/qcam/assets/shader/bayer_8.vert b/include/libcamera/internal/shaders/bayer_8.vert similarity index 100% rename from src/apps/qcam/assets/shader/bayer_8.vert rename to include/libcamera/internal/shaders/bayer_8.vert diff --git a/src/apps/qcam/assets/shader/identity.vert b/include/libcamera/internal/shaders/identity.vert similarity index 100% rename from src/apps/qcam/assets/shader/identity.vert rename to include/libcamera/internal/shaders/identity.vert diff --git a/src/apps/qcam/assets/shader/shaders.qrc b/src/apps/qcam/assets/shader/shaders.qrc index 96c709f92..04f9d7061 100644 --- a/src/apps/qcam/assets/shader/shaders.qrc +++ b/src/apps/qcam/assets/shader/shaders.qrc @@ -1,13 +1,13 @@ - RGB.frag - YUV_2_planes.frag - YUV_3_planes.frag - YUV_packed.frag - bayer_1x_packed.frag - bayer_8.frag - bayer_8.vert - identity.vert + ../../../../../include/libcamera/internal/shaders/RGB.frag + ../../../../../include/libcamera/internal/shaders/YUV_2_planes.frag + ../../../../../include/libcamera/internal/shaders/YUV_3_planes.frag + ../../../../../include/libcamera/internal/shaders/YUV_packed.frag + ../../../../../include/libcamera/internal/shaders/bayer_1x_packed.frag + ../../../../../include/libcamera/internal/shaders/bayer_8.frag + ../../../../../include/libcamera/internal/shaders/bayer_8.vert + ../../../../../include/libcamera/internal/shaders/identity.vert diff --git a/src/apps/qcam/viewfinder_gl.cpp b/src/apps/qcam/viewfinder_gl.cpp index f31956ff0..70f600650 100644 --- a/src/apps/qcam/viewfinder_gl.cpp +++ b/src/apps/qcam/viewfinder_gl.cpp @@ -141,7 +141,7 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) textureMinMagFilters_ = GL_LINEAR; /* Use identity.vert as the default vertex shader. */ - vertexShaderFile_ = ":identity.vert"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/identity.vert"; fragmentShaderDefines_.clear(); @@ -150,170 +150,170 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) horzSubSample_ = 2; vertSubSample_ = 2; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV21: horzSubSample_ = 2; vertSubSample_ = 2; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV16: horzSubSample_ = 2; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV61: horzSubSample_ = 2; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV24: horzSubSample_ = 1; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV42: horzSubSample_ = 1; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::YUV420: horzSubSample_ = 2; vertSubSample_ = 2; - fragmentShaderFile_ = ":YUV_3_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_3_planes.frag"; break; case libcamera::formats::YVU420: horzSubSample_ = 2; vertSubSample_ = 2; - fragmentShaderFile_ = ":YUV_3_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_3_planes.frag"; break; case libcamera::formats::UYVY: fragmentShaderDefines_.append("#define YUV_PATTERN_UYVY"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::VYUY: fragmentShaderDefines_.append("#define YUV_PATTERN_VYUY"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::YUYV: fragmentShaderDefines_.append("#define YUV_PATTERN_YUYV"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::YVYU: fragmentShaderDefines_.append("#define YUV_PATTERN_YVYU"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::ABGR8888: fragmentShaderDefines_.append("#define RGB_PATTERN rgb"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::ARGB8888: fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::BGRA8888: fragmentShaderDefines_.append("#define RGB_PATTERN gba"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::RGBA8888: fragmentShaderDefines_.append("#define RGB_PATTERN abg"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::BGR888: fragmentShaderDefines_.append("#define RGB_PATTERN rgb"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::RGB888: fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::SBGGR8: firstRed_.setX(1.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG8: firstRed_.setX(0.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG8: firstRed_.setX(1.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB8: firstRed_.setX(0.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SBGGR12_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG12_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG12_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB12_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; default: From a6a1fbd82b83ef3258a61aa284d200559264ef9d Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 26 Jan 2024 22:10:09 +0000 Subject: [PATCH 022/106] utils: gen-shader-headers: Add a utility to generate headers from shaders Two simple script to generate a header that contains GLSL shaders translated to C arrays. Signed-off-by: Bryan O'Donoghue --- utils/gen-shader-header.py | 38 ++++++++++++++++++++++++++++++++ utils/gen-shader-headers.sh | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100755 utils/gen-shader-header.py create mode 100755 utils/gen-shader-headers.sh diff --git a/utils/gen-shader-header.py b/utils/gen-shader-header.py new file mode 100755 index 000000000..6668e648f --- /dev/null +++ b/utils/gen-shader-header.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2025, Bryan O'Donoghue. +# +# Author: Bryan O'Donoghue +# +# A python script which takes a list of shader files and converts into a C +# header. +# +import sys + +try: + with open(sys.argv[2]) as file: + data = file.read() + data_len = len(data) + + name = sys.argv[1].replace(".", "_") + name_len = name + "_len" + + j = 0 + print("unsigned char", name, "[] = {") + for ch in data: + print(f"0x{ord(ch):02x}, ", end="") + j = j + 1 + if j == 16: + print() + j = 0 + if j != 0: + print() + print("};") + + print() + print(f"const unsigned int {name_len}={data_len};") + +except FileNotFoundError: + print(f"File {sys.argv[2]} not found", file=sys.stderr) +except IOError: + print(f"Unable to read {sys.argv[2]}", file=sys.stderr) diff --git a/utils/gen-shader-headers.sh b/utils/gen-shader-headers.sh new file mode 100755 index 000000000..ca4f19f07 --- /dev/null +++ b/utils/gen-shader-headers.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -x + +if [ $# -lt 4 ]; then + echo "Invalid arg count must be >= 5" + exit 1 +fi +src_dir="$1"; shift +build_dir="$1"; shift +build_path=$build_dir/"$1"; shift + +cat < "$build_path" +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* This file is auto-generated, do not edit! */ +/* + * Copyright (C) 2025, Linaro Ltd. + * + */ + +#pragma once + +EOF + +cat <> "$build_path" +/* + * List the names of the shaders at the top of + * header for readability's sake + * +EOF + +for file in "$@"; do + echo "file is $file" + name=$(basename "$build_dir/$file" | tr '.' '_') + echo " * unsigned char $name;" >> "$build_path" +done + +echo "*/" >> "$build_path" + +echo "/* Hex encoded shader data */" >> "$build_path" +for file in "$@"; do + name=$(basename "$build_dir/$file") + "$src_dir/utils/gen-shader-header.py" "$name" "$build_dir/$file" >> "$build_path" + echo >> "$build_path" +done From 8422ee944199cba8e015ce80a52e47c4592ff0e7 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 26 Jan 2024 22:18:13 +0000 Subject: [PATCH 023/106] meson: Automatically generate glsl_shaders.h from specified shader programs Encode the bayer shader files into a header as part of the build process. Qcam already compiles the shader files down into a QT resource file which it references internally. In order to share the debayering shader programs outside of qcam create a generic header which both qcam and libcamera can operate from. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/meson.build | 10 ++++++++++ include/libcamera/internal/shaders/meson.build | 10 ++++++++++ utils/meson.build | 2 ++ 3 files changed, 22 insertions(+) create mode 100644 include/libcamera/internal/shaders/meson.build diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index 33f318b2b..c0b593bfc 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -1,6 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 subdir('tracepoints') +subdir('shaders') libcamera_internal_headers = files([ 'bayer_format.h', @@ -57,5 +58,14 @@ tracepoints_h = custom_target( libcamera_internal_headers += tracepoints_h +libcamera_shader_headers = custom_target( + 'gen-shader-headers', + input : [shader_files], + output : 'glsl_shaders.h', + command : [gen_shader_headers, meson.project_source_root(), meson.project_build_root(), '@OUTPUT@', '@INPUT@'], +) + +libcamera_internal_headers += libcamera_shader_headers + subdir('converter') subdir('software_isp') diff --git a/include/libcamera/internal/shaders/meson.build b/include/libcamera/internal/shaders/meson.build new file mode 100644 index 000000000..386b342d0 --- /dev/null +++ b/include/libcamera/internal/shaders/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 + +# List of shader files to convert to header hex +# for the purposes of inclusion in OpenGL debayering +shader_files = files([ + 'bayer_1x_packed.frag', + 'bayer_8.frag', + 'bayer_8.vert', + 'identity.vert', +]) diff --git a/utils/meson.build b/utils/meson.build index 95d657ac9..3deed8ad4 100644 --- a/utils/meson.build +++ b/utils/meson.build @@ -3,5 +3,7 @@ subdir('codegen') subdir('ipu3') +gen_shader_headers = files('gen-shader-headers.sh') + ## Module signing gen_ipa_priv_key = files('gen-ipa-priv-key.sh') From 89cf67a45fa255a8d0275dfeff15c09597633ec6 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 10 May 2025 04:58:51 +0100 Subject: [PATCH 024/106] libcamera: software_isp: ccm: Populate CCM table to Debayer params structure Populate the DebayerParams CCM table during ccm::prepare(). A copy is made of the CCM into the DebayerParams structure. Signed-off-by: Bryan O'Donoghue --- src/ipa/simple/algorithms/ccm.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ipa/simple/algorithms/ccm.cpp b/src/ipa/simple/algorithms/ccm.cpp index 0a98406c1..cb023878d 100644 --- a/src/ipa/simple/algorithms/ccm.cpp +++ b/src/ipa/simple/algorithms/ccm.cpp @@ -14,6 +14,7 @@ #include #include "libcamera/internal/matrix.h" +#include "libcamera/internal/software_isp/debayer_params.h" namespace { @@ -84,7 +85,7 @@ void Ccm::applySaturation(Matrix &ccm, float saturation) } void Ccm::prepare(IPAContext &context, const uint32_t frame, - IPAFrameContext &frameContext, [[maybe_unused]] DebayerParams *params) + IPAFrameContext &frameContext, DebayerParams *params) { auto &saturation = context.activeState.knobs.saturation; @@ -108,6 +109,7 @@ void Ccm::prepare(IPAContext &context, const uint32_t frame, context.activeState.ccm.ccm = ccm; frameContext.ccm.ccm = ccm; frameContext.saturation = saturation; + params->ccm = ccm; context.activeState.ccm.changed = true; } From 59284f4f27f29c1b1e0fd5af948a826a92bed2d3 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 10 May 2025 06:14:20 +0100 Subject: [PATCH 025/106] libcamera: software_isp: lut: Make gain corrected CCM in lut.cpp available in debayer params There is another CCM calculated in this algorithm callback, this time based on the LUT. Signed-off-by: Bryan O'Donoghue --- src/ipa/simple/algorithms/lut.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ipa/simple/algorithms/lut.cpp b/src/ipa/simple/algorithms/lut.cpp index d1d5f7271..a161adb1a 100644 --- a/src/ipa/simple/algorithms/lut.cpp +++ b/src/ipa/simple/algorithms/lut.cpp @@ -126,6 +126,7 @@ void Lut::prepare(IPAContext &context, auto &red = params->redCcm; auto &green = params->greenCcm; auto &blue = params->blueCcm; + params->ccm = ccm; for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { red[i].r = ccmValue(i, ccm[0][0]); red[i].g = ccmValue(i, ccm[1][0]); From 234849b2b9d41185124a01ed208cfa6e7ff5bbf5 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Mon, 23 Sep 2024 16:11:28 +0100 Subject: [PATCH 026/106] libcamera: software_isp: gbm: Add in a GBM helper class for GPU surface access A helper class to interact with GBM. This will allow us to specify the internal storage format of the CPU when making a texture for the Debayer vertext/fragment shaders and thus ensure we receive an uncompressed and untiled output buffer. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/gbm.h | 55 ++++++++++ include/libcamera/internal/meson.build | 1 + src/libcamera/gbm.cpp | 137 +++++++++++++++++++++++++ src/libcamera/meson.build | 11 ++ 4 files changed, 204 insertions(+) create mode 100644 include/libcamera/internal/gbm.h create mode 100644 src/libcamera/gbm.cpp diff --git a/include/libcamera/internal/gbm.h b/include/libcamera/internal/gbm.h new file mode 100644 index 000000000..a5486cc94 --- /dev/null +++ b/include/libcamera/internal/gbm.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * gbm.h - Helper class for managing GBM interactions. + */ + +#pragma once + +#include + +#include + +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(GBM) + +class GBM +{ +public: + GBM(); + ~GBM(); + + int initSurface(uint32_t width, uint32_t height); + int mapSurface(); + int getFrameBufferData(uint8_t *data_out, size_t data_len); + struct gbm_device *getDevice() { return gbm_device_; } + struct gbm_surface *getSurface() { return gbm_surface_; } + uint32_t getFrameSize() { return framesize_; } + uint32_t getStride() { return stride_; } + PixelFormat getPixelFormat() { return format_; } + +private: + int fd_; + struct gbm_device *gbm_device_; + struct gbm_surface *gbm_surface_; + + struct gbm_bo *gbm_bo_; + uint32_t width_; + uint32_t height_; + uint32_t stride_; + uint32_t offset_; + uint32_t framesize_; + void *map_; + int bo_fd_; + + PixelFormat format_; +}; + +} // namespace libcamera diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index c0b593bfc..4a2919f61 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -23,6 +23,7 @@ libcamera_internal_headers = files([ 'dma_buf_allocator.h', 'formats.h', 'framebuffer.h', + 'gbm.h', 'ipa_data_serializer.h', 'ipa_manager.h', 'ipa_module.h', diff --git a/src/libcamera/gbm.cpp b/src/libcamera/gbm.cpp new file mode 100644 index 000000000..43032093e --- /dev/null +++ b/src/libcamera/gbm.cpp @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl.cpp - Helper class for managing GBM interactions. + */ + +#include "libcamera/internal/gbm.h" + +#include +#include +#include +#include + +#include +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(GBM) + +GBM::GBM() +{ + fd_ = 0; +} + +GBM::~GBM() +{ + if (gbm_surface_) + gbm_surface_destroy(gbm_surface_); + + if (gbm_device_) + gbm_device_destroy(gbm_device_); + + if (fd_ >= 0) + close(fd_); +} + +// this should probably go into its own class to deal with the +// allocation and deletion of frambuffers attached to GBM devices/objects +int GBM::initSurface(uint32_t width, uint32_t height) +{ + const char *dri_node = "/dev/dri/renderD128"; //TODO: get from an env or config setting + + fd_ = open(dri_node, O_RDWR | O_CLOEXEC); //TODO: CLOEXEC ? + if (fd_ < 0) { + LOG(GBM, Error) << "Open " << dri_node << " fail " << fd_; + return fd_; + } + + gbm_device_ = gbm_create_device(fd_); + if (!gbm_device_) { + LOG(GBM, Error) << "gbm_crate_device fail"; + goto fail; + } + + // GBM_FORMAT_RGBA8888 is not supported mesa::src/gbm/dri/gbm_dri.c::gbm_dri_visuals_table[] + // This means we need to choose XRGB8888 or ARGB8888 as the raw buffer format + gbm_surface_ = gbm_surface_create(gbm_device_, width, height, GBM_FORMAT_ARGB8888, + GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR); + if (!gbm_surface_) { + LOG(GBM, Error) << "Unable to create linear gbm surface"; + goto fail; + } + + format_ = libcamera::formats::ARGB8888; + + return 0; +fail: + return -ENODEV; +} + +int GBM::mapSurface() +{ + gbm_bo_ = gbm_surface_lock_front_buffer(gbm_surface_); + if (!gbm_bo_) { + LOG(GBM, Error) << "GBM input buffer object create fail"; + return -ENODEV; + } + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + + bo_fd_ = gbm_bo_get_fd(gbm_bo_); + + if (!bo_fd_) { + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + LOG(GBM, Error) << "Unable to get fd for bo: " << bo_fd_; + return -ENODEV; + } + + stride_ = gbm_bo_get_stride(gbm_bo_); + width_ = gbm_bo_get_width(gbm_bo_); + height_ = gbm_bo_get_height(gbm_bo_); + offset_ = gbm_bo_get_offset(gbm_bo_, 0); + framesize_ = height_ * stride_; + + map_ = mmap(NULL, height_ * stride_, PROT_READ, MAP_SHARED, bo_fd_, 0); + if (map_ == MAP_FAILED) { + LOG(GBM, Error) << "mmap gbm_bo_ fail"; + return -ENODEV; + } + + LOG(GBM, Debug) << " stride " << stride_ + << " width " << width_ + << " height " << height_ + << " offset " << offset_ + << " framesize " << framesize_; + + return 0; +} + +int GBM::getFrameBufferData(uint8_t *data, size_t data_len) +{ + struct dma_buf_sync sync; + + gbm_surface_lock_front_buffer(gbm_surface_); + + sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ; + ioctl(bo_fd_, DMA_BUF_IOCTL_SYNC, &sync); + + if (data_len > framesize_) { + LOG(GBM, Error) << "Invalid read size " << data_len << " max is " << framesize_; + return -EINVAL; + } + + memcpy(data, map_, data_len); + + sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ; + ioctl(bo_fd_, DMA_BUF_IOCTL_SYNC, &sync); + + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + + return 0; +} +} //namespace libcamera diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 202db1efe..0d004694e 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -67,6 +67,16 @@ libcamera_deps = [] libatomic = cc.find_library('atomic', required : false) libthreads = dependency('threads') +libgbm = cc.find_library('gbm', required: false) +gbm_works = cc.check_header('gbm.h', required: false) + +if libgbm.found() and gbm_works + config_h.set('HAVE_GBM', 1) + libcamera_internal_sources += files([ + 'gbm.cpp', + ]) +endif + subdir('base') subdir('converter') subdir('ipa') @@ -188,6 +198,7 @@ libcamera_deps += [ libcamera_base_private, libcrypto, libdl, + libgbm, liblttng, libudev, libyaml, From 94f30456aa4765aa3cf19030f68a7b6696b4cf76 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 25 Feb 2024 16:23:26 +0000 Subject: [PATCH 027/106] libcamera: software_isp: egl: Introduce an eGL base helper class Introduce an eGL base helper class which provides an eGL context based on a passed width and height. The initGLContext function could be overloaded to provide an interface to a real display. A set of helper functions is provided to compile and link GLSL shaders. linkShaderProgram currently compiles vertex/fragment pairs but could be overloaded or passed a parameter to link a compute shader instead. Breaking the eGL interface away from debayering - allows to use the eGL context inside of a dma-buf heap cleanly, reuse that context inside of a debayer layer and conceivably reuse the context in a multi-stage shader pass. Small note the image_attrs[] array doesn't pass checkstyle.py however the elements of the array are in pairs. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/egl.h | 110 +++++++++ src/libcamera/egl.cpp | 369 +++++++++++++++++++++++++++++++ src/libcamera/meson.build | 23 ++ 3 files changed, 502 insertions(+) create mode 100644 include/libcamera/internal/egl.h create mode 100644 src/libcamera/egl.cpp diff --git a/include/libcamera/internal/egl.h b/include/libcamera/internal/egl.h new file mode 100644 index 000000000..04d637d88 --- /dev/null +++ b/include/libcamera/internal/egl.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl_context.cpp - Helper class for managing eGL interactions. + */ + +#pragma once + +#include + +#include + +#include "libcamera/internal/gbm.h" + +#define EGL_EGLEXT_PROTOTYPES +#include +#include +#define GL_GLEXT_PROTOTYPES +#include +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(eGL) + +class eGLImage +{ +public: + eGLImage(uint32_t width, uint32_t height, uint32_t bpp, GLenum texture_unit, uint32_t texture_unit_uniform_id) + { + image_ = EGL_NO_IMAGE_KHR; + width_ = width; + height_ = height; + bpp_ = bpp; + stride_ = width_ * bpp_ / 4; + framesize_ = stride_ * height_; + texture_unit_ = texture_unit; + texture_unit_uniform_id_ = texture_unit_uniform_id; + + glGenTextures(1, &texture_); + } + + ~eGLImage() + { + glDeleteTextures(1, &texture_); + } + + uint32_t width_; + uint32_t height_; + uint32_t stride_; + uint32_t offset_; + uint32_t framesize_; + uint32_t bpp_; + uint32_t texture_unit_uniform_id_; + GLenum texture_unit_; + GLuint texture_; + EGLImageKHR image_; +}; + +class eGL +{ +public: + eGL(); + ~eGL(); + + int initEGLContext(GBM *gbmContext); + int createDMABufTexture2D(eGLImage *eglImage, int fd); + void destroyDMABufTexture(eGLImage *eglImage); + void createTexture2D(eGLImage *eglImage, GLint format, uint32_t width, uint32_t height, void *data); + void createTexture1D(eGLImage *eglImage, GLint format, uint32_t width, void *data); + + void pushEnv(std::vector &shaderEnv, const char *str); + void makeCurrent(); + void swapBuffers(); + + int compileVertexShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + int compileFragmentShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + int linkProgram(GLuint &programIdd, GLuint fragmentshaderId, GLuint vertexshaderId); + void dumpShaderSource(GLuint shaderId); + void useProgram(GLuint programId); + +private: + int fd_; + + EGLDisplay display_; + EGLContext context_; + EGLSurface surface_; + + int compileShader(int shaderType, GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + + PFNEGLEXPORTDMABUFIMAGEMESAPROC eglExportDMABUFImageMESA; + PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOES; + + PFNEGLCREATEIMAGEKHRPROC eglCreateImageKHR; + PFNEGLDESTROYIMAGEKHRPROC eglDestroyImageKHR; + + PFNEGLCLIENTWAITSYNCKHRPROC eglClientWaitSyncKHR; + PFNEGLCREATESYNCKHRPROC eglCreateSyncKHR; +}; +} //namespace libcamera diff --git a/src/libcamera/egl.cpp b/src/libcamera/egl.cpp new file mode 100644 index 000000000..89ece148b --- /dev/null +++ b/src/libcamera/egl.cpp @@ -0,0 +1,369 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl.cpp - Helper class for managing eGL interactions. + */ + +#include "libcamera/internal/egl.h" + +#include +#include +#include +#include + +#include +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(eGL) + +eGL::eGL() +{ +} + +eGL::~eGL() +{ +} + +// Create linear image attached to previous BO object +int eGL::createDMABufTexture2D(eGLImage *eglImage, int fd) +{ + int ret = 0; + + eglImage->stride_ = eglImage->width_ * eglImage->height_; + eglImage->offset_ = 0; + eglImage->framesize_ = eglImage->height_ * eglImage->stride_; + + LOG(eGL, Info) + << " stride " << eglImage->stride_ << " width " << eglImage->width_ << " height " << eglImage->height_ << " offset " << eglImage->offset_ << " framesize " << eglImage->framesize_; + + // TODO: use the dma buf handle from udma heap here directly + // should work for both input and output with fencing + EGLint image_attrs[] = { + EGL_WIDTH, (EGLint)eglImage->width_, + EGL_HEIGHT, (EGLint)eglImage->height_, + EGL_LINUX_DRM_FOURCC_EXT, (int)GBM_FORMAT_ARGB8888, + EGL_DMA_BUF_PLANE0_FD_EXT, fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0, + EGL_DMA_BUF_PLANE0_PITCH_EXT, (EGLint)eglImage->framesize_, + EGL_NONE, EGL_NONE, /* modifier lo */ + EGL_NONE, EGL_NONE, /* modifier hi */ + EGL_NONE, + }; + + eglImage->image_ = eglCreateImageKHR(display_, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + NULL, image_attrs); + + if (eglImage->image_ == EGL_NO_IMAGE_KHR) { + LOG(eGL, Error) << "eglCreateImageKHR fail"; + ret = -ENODEV; + goto done; + } + + // Generate texture, bind, associate image to texture, configure, unbind + glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, eglImage->image_); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + +done: + return ret; +} + +void eGL::destroyDMABufTexture(eGLImage *eglImage) +{ + eglDestroyImage(display_, eglImage->image_); +} + +// +// Generate a 2D texture from an input buffer directly +void eGL::createTexture2D(eGLImage *eglImage, uint32_t width, uint32_t height, void *data) +{ + glActiveTexture(eglImage->texture_unit_); + glBindTexture(GL_TEXTURE_2D, eglImage->texture_); + + // Generate texture, bind, associate image to texture, configure, unbind + glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, data); + + // Nearest filtering + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + + // Wrap to edge to avoid edge artifacts + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +} + +int eGL::initEGLContext(GBM *gbmContext) +{ + EGLint configAttribs[] = { + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_NONE + }; + + EGLint contextAttribs[] = { + EGL_CONTEXT_MAJOR_VERSION, 2, + EGL_NONE + }; + + EGLint numConfigs; + EGLConfig config; + EGLint major; + EGLint minor; + + if (!eglBindAPI(EGL_OPENGL_ES_API)) { + LOG(eGL, Error) << "API bind fail"; + goto fail; + } + + //TODO: use optional eglGetPlatformDisplayEXT ? + display_ = eglGetDisplay(gbmContext->getDevice()); + if (display_ == EGL_NO_DISPLAY) { + LOG(eGL, Error) << "Unable to get EGL display"; + goto fail; + } + + if (eglInitialize(display_, &major, &minor) != EGL_TRUE) { + LOG(eGL, Error) << "eglInitialize fail"; + goto fail; + } + + LOG(eGL, Info) << "EGL: version " << major << "." << minor; + LOG(eGL, Info) << "EGL: EGL_VERSION: " << eglQueryString(display_, EGL_VERSION); + LOG(eGL, Info) << "EGL: EGL_VENDOR: " << eglQueryString(display_, EGL_VENDOR); + LOG(eGL, Info) << "EGL: EGL_CLIENT_APIS: " << eglQueryString(display_, EGL_CLIENT_APIS); + LOG(eGL, Info) << "EGL: EGL_EXTENSIONS: " << eglQueryString(display_, EGL_EXTENSIONS); + + //TODO: interrogate strings to make sure we aren't hooking unsupported functions + // and remember to error out if a function we depend on isn't found. + // we don't use these functions right now but expect to for DMA backed + // texture generation and render-to-texture. One thing we can do is differentiate + // between DMA and non-DMA texture generation based on the presence of these functions + // In reality most - all ? - mesa implementations have these extensions so + // probably no fallback will be required + eglCreateImageKHR = (PFNEGLCREATEIMAGEKHRPROC)eglGetProcAddress("eglCreateImageKHR"); + if (!eglCreateImageKHR) + LOG(eGL, Warning) << "eglCreateImageKHR not found"; + + eglDestroyImageKHR = (PFNEGLDESTROYIMAGEKHRPROC)eglGetProcAddress("eglDestroyImageKHR"); + if (!eglDestroyImageKHR) + LOG(eGL, Warning) << "eglDestroyImageKHR not found"; + + eglExportDMABUFImageMESA = (PFNEGLEXPORTDMABUFIMAGEMESAPROC)eglGetProcAddress("eglExportDMABUFImageMESA"); + if (!eglExportDMABUFImageMESA) + LOG(eGL, Warning) << "eglExportDMABUFImageMESA not found"; + + glEGLImageTargetTexture2DOES = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress("glEGLImageTargetTexture2DOES"); + if (!glEGLImageTargetTexture2DOES) + LOG(eGL, Warning) << "glEGLImageTargetTexture2DOES not found"; + + eglClientWaitSyncKHR = (PFNEGLCLIENTWAITSYNCKHRPROC)eglGetProcAddress("eglClientWaitSyncKHR"); + if (!eglClientWaitSyncKHR) + LOG(eGL, Warning) << "eglClientWaitSyncKHR not found"; + + eglCreateSyncKHR = (PFNEGLCREATESYNCKHRPROC)eglGetProcAddress("eglCreateSyncKHR"); + if (!eglCreateSyncKHR) + LOG(eGL, Warning) << "eglCreateSyncKHR not found"; + + if (eglChooseConfig(display_, configAttribs, &config, 1, &numConfigs) != EGL_TRUE) { + LOG(eGL, Error) << "eglChooseConfig fail"; + goto fail; + } + + context_ = eglCreateContext(display_, config, EGL_NO_CONTEXT, contextAttribs); + if (context_ == EGL_NO_CONTEXT) { + LOG(eGL, Error) << "eglContext returned EGL_NO_CONTEXT"; + goto fail; + } + + surface_ = eglCreateWindowSurface(display_, config, + (EGLNativeWindowType)gbmContext->getSurface(), + NULL); + if (surface_ == EGL_NO_SURFACE) { + LOG(eGL, Error) << "eglCreateWindowSurface fail"; + goto fail; + } + + makeCurrent(); + swapBuffers(); + + return 0; +fail: + + return -ENODEV; +} + +void eGL::makeCurrent(void) +{ + if (eglMakeCurrent(display_, surface_, surface_, context_) != EGL_TRUE) { + LOG(eGL, Error) << "eglMakeCurrent fail"; + } +} + +void eGL::swapBuffers(void) +{ + if (eglSwapBuffers(display_, surface_) != EGL_TRUE) { + LOG(eGL, Error) << "eglSwapBuffers fail"; + } +} + +void eGL::useProgram(GLuint programId) +{ + glUseProgram(programId); +} + +void eGL::pushEnv(std::vector &shaderEnv, const char *str) +{ + std::string addStr = str; + + addStr.push_back('\n'); + shaderEnv.push_back(addStr); +} + +int eGL::compileVertexShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + return compileShader(GL_VERTEX_SHADER, shaderId, shaderData, shaderDataLen, shaderEnv); +} + +int eGL::compileFragmentShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + return compileShader(GL_FRAGMENT_SHADER, shaderId, shaderData, shaderDataLen, shaderEnv); +} + +int eGL::compileShader(int shaderType, GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + GLchar **shaderSourceData; + GLint *shaderDataLengths; + GLint success; + GLsizei count; + size_t i; + + count = 1 + shaderEnv.size(); + shaderSourceData = new GLchar *[count]; + shaderDataLengths = new GLint[count]; + + // Prefix defines before main body of shader + for (i = 0; i < shaderEnv.size(); i++) { + shaderSourceData[i] = (GLchar *)shaderEnv[i].c_str(); + shaderDataLengths[i] = shaderEnv[i].length(); + } + + // Now the main body of the shader program + shaderSourceData[i] = (GLchar *)shaderData; + shaderDataLengths[i] = shaderDataLen; + + // And create the shader + shaderId = glCreateShader(shaderType); + glShaderSource(shaderId, count, shaderSourceData, shaderDataLengths); + glCompileShader(shaderId); + + // Check status + glGetShaderiv(shaderId, GL_COMPILE_STATUS, &success); + if (success == GL_FALSE) { + GLint sizeLog = 0; + GLchar *infoLog; + + glGetShaderiv(shaderId, GL_INFO_LOG_LENGTH, &sizeLog); + infoLog = new GLchar[sizeLog]; + + glGetShaderInfoLog(shaderId, sizeLog, &sizeLog, infoLog); + LOG(eGL, Error) << infoLog; + + delete[] infoLog; + } + + delete[] shaderSourceData; + delete[] shaderDataLengths; + + return !(success == GL_TRUE); +} + +void eGL::dumpShaderSource(GLuint shaderId) +{ + GLint shaderLength = 0; + GLchar *shaderSource; + + glGetShaderiv(shaderId, GL_SHADER_SOURCE_LENGTH, &shaderLength); + + LOG(eGL, Debug) << "Shader length is " << shaderLength; + + if (shaderLength > 0) { + shaderSource = new GLchar[shaderLength]; + if (!shaderSource) + return; + + glGetShaderSource(shaderId, shaderLength, &shaderLength, shaderSource); + if (shaderLength) { + LOG(eGL, Debug) << "Shader source = " << shaderSource; + } + delete[] shaderSource; + } +} + +int eGL::linkProgram(GLuint &programId, GLuint vertexshaderId, GLuint fragmentshaderId) +{ + GLint success; + GLenum err; + + programId = glCreateProgram(); + if (!programId) + goto fail; + + glAttachShader(programId, vertexshaderId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Attach compute vertex shader fail"; + goto fail; + } + + glAttachShader(programId, fragmentshaderId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Attach compute vertex shader fail"; + goto fail; + } + + glLinkProgram(programId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Link program fail"; + goto fail; + } + + glDetachShader(programId, fragmentshaderId); + glDetachShader(programId, vertexshaderId); + + // Check status + glGetProgramiv(programId, GL_LINK_STATUS, &success); + if (success == GL_FALSE) { + GLint sizeLog = 0; + GLchar *infoLog; + + glGetProgramiv(programId, GL_INFO_LOG_LENGTH, &sizeLog); + infoLog = new GLchar[sizeLog]; + + glGetProgramInfoLog(programId, sizeLog, &sizeLog, infoLog); + LOG(eGL, Error) << infoLog; + + delete[] infoLog; + goto fail; + } + + return 0; +fail: + return -ENODEV; +} +} // namespace libcamera diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 0d004694e..491eb7340 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -77,6 +77,27 @@ if libgbm.found() and gbm_works ]) endif +libegl = cc.find_library('EGL', required : false) +libglesv2 = cc.find_library('GLESv2', required : false) +mesa_works = cc.check_header('EGL/egl.h', required: false) + +if libegl.found() and mesa_works + config_h.set('HAVE_LIBEGL', 1) +endif + +if libglesv2.found() and mesa_works + config_h.set('HAVE_GLESV2', 1) +endif + +if mesa_works and gbm_works + libcamera_internal_sources += files([ + 'egl.cpp', + ]) + gles_headless_enabled = true +else + gles_headless_enabled = false +endif + subdir('base') subdir('converter') subdir('ipa') @@ -198,7 +219,9 @@ libcamera_deps += [ libcamera_base_private, libcrypto, libdl, + libegl, libgbm, + libglesv2, liblttng, libudev, libyaml, From eba39200919934ab0221224819c181fbd97b4eeb Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 31 Jan 2024 02:48:39 +0000 Subject: [PATCH 028/106] libcamera: software_isp: debayer_egl: Add an eGL debayer class Add a class to run the existing glsl debayer shaders on a GBM surface. Signed-off-by: Bryan O'Donoghue libcamera: software_isp: debayer_egl: Extend logic to enable application of softISP RGB debayer params The existing SoftISP calculates RGB gain values as a lookup table of 256 values which shifts for each frame depending on the required correction. We can pass the required tables into the debayer shaders as textures, one texture for R, G and B respectively. The debayer shader will do its debayer interpolation and then if the appropriate define is specified use the calculated R, G and B values as indexes into our bayer colour gain table. Signed-off-by: Bryan O'Donoghue --- src/libcamera/egl.cpp | 4 +- src/libcamera/software_isp/debayer_egl.cpp | 587 +++++++++++++++++++++ src/libcamera/software_isp/debayer_egl.h | 164 ++++++ src/libcamera/software_isp/meson.build | 8 + 4 files changed, 761 insertions(+), 2 deletions(-) create mode 100644 src/libcamera/software_isp/debayer_egl.cpp create mode 100644 src/libcamera/software_isp/debayer_egl.h diff --git a/src/libcamera/egl.cpp b/src/libcamera/egl.cpp index 89ece148b..c6b0f9a56 100644 --- a/src/libcamera/egl.cpp +++ b/src/libcamera/egl.cpp @@ -82,13 +82,13 @@ void eGL::destroyDMABufTexture(eGLImage *eglImage) // // Generate a 2D texture from an input buffer directly -void eGL::createTexture2D(eGLImage *eglImage, uint32_t width, uint32_t height, void *data) +void eGL::createTexture2D(eGLImage *eglImage, GLint format, uint32_t width, uint32_t height, void *data) { glActiveTexture(eglImage->texture_unit_); glBindTexture(GL_TEXTURE_2D, eglImage->texture_); // Generate texture, bind, associate image to texture, configure, unbind - glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, data); + glTexImage2D(GL_TEXTURE_2D, 0, format, width, height, 0, format, GL_UNSIGNED_BYTE, data); // Nearest filtering glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp new file mode 100644 index 000000000..3fb155119 --- /dev/null +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -0,0 +1,587 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * debayer_cpu.cpp - EGL based debayering class + */ + +#include +#include +#include + +#include + +#include "libcamera/internal/glsl_shaders.h" +#include "debayer_egl.h" + +namespace libcamera { + +DebayerEGL::DebayerEGL(std::unique_ptr stats) + : Debayer(), stats_(std::move(stats)) +{ + eglImageBayerIn_ = eglImageRedLookup_ = eglImageBlueLookup_ = eglImageGreenLookup_ = NULL; +} + +DebayerEGL::~DebayerEGL() +{ + if (eglImageBlueLookup_) + delete eglImageBlueLookup_; + + if (eglImageGreenLookup_) + delete eglImageGreenLookup_; + + if (eglImageRedLookup_) + delete eglImageRedLookup_; + + if (eglImageBayerIn_) + delete eglImageBayerIn_; +} + +int DebayerEGL::getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config) +{ + BayerFormat bayerFormat = + BayerFormat::fromPixelFormat(inputFormat); + + if ((bayerFormat.bitDepth == 8 || bayerFormat.bitDepth == 10) && + bayerFormat.packing == BayerFormat::Packing::None && + isStandardBayerOrder(bayerFormat.order)) { + config.bpp = (bayerFormat.bitDepth + 7) & ~7; + config.patternSize.width = 2; + config.patternSize.height = 2; + config.outputFormats = std::vector({ formats::XRGB8888, + formats::ARGB8888, + formats::XBGR8888, + formats::ABGR8888 }); + return 0; + } + + if (bayerFormat.bitDepth == 10 && + bayerFormat.packing == BayerFormat::Packing::CSI2 && + isStandardBayerOrder(bayerFormat.order)) { + config.bpp = 10; + config.patternSize.width = 4; /* 5 bytes per *4* pixels */ + config.patternSize.height = 2; + config.outputFormats = std::vector({ formats::XRGB8888, + formats::ARGB8888, + formats::XBGR8888, + formats::ABGR8888 }); + return 0; + } + + LOG(Debayer, Info) + << "Unsupported input format " << inputFormat.toString(); + return -EINVAL; +} + +int DebayerEGL::getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config) +{ + if (outputFormat == formats::XRGB8888 || outputFormat == formats::ARGB8888 || + outputFormat == formats::XBGR8888 || outputFormat == formats::ABGR8888) { + config.bpp = 32; + return 0; + } + + LOG(Debayer, Error) + << "Unsupported output format " << outputFormat.toString(); + + return -EINVAL; +} + +int DebayerEGL::getShaderVariableLocations(void) +{ + attributeVertex_ = glGetAttribLocation(programId_, "vertexIn"); + attributeTexture_ = glGetAttribLocation(programId_, "textureIn"); + + textureUniformBayerDataIn_ = glGetUniformLocation(programId_, "tex_y"); + textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); + textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); + textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); + + textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); + textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); + textureUniformStrideFactor_ = glGetUniformLocation(programId_, "stride_factor"); + textureUniformBayerFirstRed_ = glGetUniformLocation(programId_, "tex_bayer_first_red"); + textureUniformProjMatrix_ = glGetUniformLocation(programId_, "proj_matrix"); + + LOG(Debayer, Debug) << "vertexIn " << attributeVertex_ << " textureIn " << attributeTexture_ + << " tex_y " << textureUniformBayerDataIn_ + << " red_param " << textureUniformRedLookupDataIn_ + << " green_param " << textureUniformGreenLookupDataIn_ + << " blue_param " << textureUniformBlueLookupDataIn_ + << " tex_step " << textureUniformStep_ + << " tex_size " << textureUniformSize_ + << " stride_factor " << textureUniformStrideFactor_ + << " tex_bayer_first_red " << textureUniformBayerFirstRed_ + << " proj_matrix " << textureUniformProjMatrix_; + return 0; +} + +int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputFormat) +{ + std::vector shaderEnv; + unsigned int fragmentShaderDataLen; + unsigned char *fragmentShaderData; + unsigned int vertexShaderDataLen; + unsigned char *vertexShaderData; + GLenum err; + + // Target gles 100 glsl requires "#version x" as first directive in shader + egl_.pushEnv(shaderEnv, "#version 100"); + + // Specify GL_OES_EGL_image_external + egl_.pushEnv(shaderEnv, "#extension GL_OES_EGL_image_external: enable"); + + // Tell shaders how to re-order output taking account of how the + // pixels are actually stored by GBM + switch (outputFormat) { + case formats::ARGB8888: + case formats::XRGB8888: + break; + case formats::ABGR8888: + case formats::XBGR8888: + egl_.pushEnv(shaderEnv, "#define SWAP_BLUE"); + break; + default: + goto invalid_fmt; + } + + // Pixel location parameters + switch (inputFormat) { + case libcamera::formats::SBGGR8: + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SBGGR12_CSI2P: + firstRed_x_ = 1.0; + firstRed_y_ = 1.0; + break; + case libcamera::formats::SGBRG8: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + firstRed_x_ = 0.0; + firstRed_y_ = 1.0; + break; + case libcamera::formats::SGRBG8: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + firstRed_x_ = 1.0; + firstRed_y_ = 0.0; + break; + case libcamera::formats::SRGGB8: + case libcamera::formats::SRGGB10_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: + firstRed_x_ = 0.0; + firstRed_y_ = 0.0; + break; + default: + goto invalid_fmt; + break; + }; + + // Shader selection + switch (inputFormat) { + case libcamera::formats::SBGGR8: + case libcamera::formats::SGBRG8: + case libcamera::formats::SGRBG8: + case libcamera::formats::SRGGB8: + fragmentShaderData = bayer_8_frag; + fragmentShaderDataLen = bayer_8_frag_len; + vertexShaderData = bayer_8_vert; + vertexShaderDataLen = bayer_8_vert_len; + break; + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SRGGB10_CSI2P: + egl_.pushEnv(shaderEnv, "#define RAW10P"); + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + break; + case libcamera::formats::SBGGR12_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: + egl_.pushEnv(shaderEnv, "#define RAW12P"); + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + break; + default: + goto invalid_fmt; + break; + }; + + // Flag to shaders that we have parameter gain tables + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + + if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) + goto compile_fail; + + if (egl_.compileFragmentShader(fragmentShaderId_, fragmentShaderData, fragmentShaderDataLen, shaderEnv)) + goto compile_fail; + + if (egl_.linkProgram(programId_, vertexShaderId_, fragmentShaderId_)) + goto link_fail; + + egl_.dumpShaderSource(vertexShaderId_); + egl_.dumpShaderSource(fragmentShaderId_); + + /* Ensure we set the programId_ */ + egl_.useProgram(programId_); + err = glGetError(); + if (err != GL_NO_ERROR) + goto program_fail; + + if (getShaderVariableLocations()) + goto parameters_fail; + + return 0; + +parameters_fail: + LOG(Debayer, Error) << "Program parameters fail"; + return -ENODEV; + +program_fail: + LOG(Debayer, Error) << "Use program error " << err; + return -ENODEV; + +link_fail: + LOG(Debayer, Error) << "Linking program fail"; + return -ENODEV; + +compile_fail: + LOG(Debayer, Error) << "Compile debayer shaders fail"; + return -ENODEV; + +invalid_fmt: + LOG(Debayer, Error) << "Unsupported input output format combination"; + return -EINVAL; +} + +int DebayerEGL::configure(const StreamConfiguration &inputCfg, + const std::vector> &outputCfgs, + bool ccmEnabled) +{ + if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) + return -EINVAL; + + if (stats_->configure(inputCfg) != 0) + return -EINVAL; + + const Size &stats_pattern_size = stats_->patternSize(); + if (inputConfig_.patternSize.width != stats_pattern_size.width || + inputConfig_.patternSize.height != stats_pattern_size.height) { + LOG(Debayer, Error) + << "mismatching stats and debayer pattern sizes for " + << inputCfg.pixelFormat.toString(); + return -EINVAL; + } + + inputConfig_.stride = inputCfg.stride; + width_ = inputCfg.size.width; + height_ = inputCfg.size.height; + ccmEnabled_ = ccmEnabled = false; + + if (outputCfgs.size() != 1) { + LOG(Debayer, Error) + << "Unsupported number of output streams: " + << outputCfgs.size(); + return -EINVAL; + } + + LOG(Debayer, Info) << "Input size " << inputCfg.size << " stride " << inputCfg.stride; + + if (gbmSurface_.initSurface(inputCfg.size.width, inputCfg.size.height)) + return -ENODEV; + + if (egl_.initEGLContext(&gbmSurface_)) + return -ENODEV; + + glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); + LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; + + if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { + LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits + << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS + << " try using an identity CCM "; + return -ENODEV; + } + // Raw bayer input as texture + eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); + if (!eglImageBayerIn_) + return -ENOMEM; + + /// RGB correction tables as 2d textures + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); + if (!eglImageRedLookup_) + return -ENOMEM; + + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); + if (!eglImageGreenLookup_) + return -ENOMEM; + + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); + if (!eglImageBlueLookup_) + return -ENOMEM; + + // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) + if (gbmSurface_.mapSurface()) + return -ENODEV; + + StreamConfiguration &outputCfg = outputCfgs[0]; + SizeRange outSizeRange = sizes(inputCfg.pixelFormat, inputCfg.size); + + outputConfig_.stride = gbmSurface_.getStride(); + outputConfig_.frameSize = gbmSurface_.getFrameSize(); + + LOG(Debayer, Debug) << "Overriding stream config stride " + << outputCfg.stride << " with GBM surface stride " + << outputConfig_.stride; + outputCfg.stride = outputConfig_.stride; + + if (!outSizeRange.contains(outputCfg.size) || outputConfig_.stride != outputCfg.stride) { + LOG(Debayer, Error) + << "Invalid output size/stride: " + << "\n " << outputCfg.size << " (" << outSizeRange << ")" + << "\n " << outputCfg.stride << " (" << outputConfig_.stride << ")"; + return -EINVAL; + } + + window_.x = ((inputCfg.size.width - outputCfg.size.width) / 2) & + ~(inputConfig_.patternSize.width - 1); + window_.y = ((inputCfg.size.height - outputCfg.size.height) / 2) & + ~(inputConfig_.patternSize.height - 1); + window_.width = outputCfg.size.width; + window_.height = outputCfg.size.height; + + /* Don't pass x,y since process() already adjusts src before passing it */ + stats_->setWindow(Rectangle(window_.size())); + + LOG(Debayer, Debug) << "Input width " << inputCfg.size.width << " height " << inputCfg.size.height; + LOG(Debayer, Debug) << "Output width " << outputCfg.size.width << " height " << outputCfg.size.height; + LOG(Debayer, Debug) << "Output stride " << outputCfg.size.width << " height " << outputCfg.size.height; + + if (initBayerShaders(inputCfg.pixelFormat, outputCfg.pixelFormat)) + return -EINVAL; + + return 0; +} + +Size DebayerEGL::patternSize(PixelFormat inputFormat) +{ + DebayerEGL::DebayerInputConfig config; + + if (getInputConfig(inputFormat, config) != 0) + return {}; + + return config.patternSize; +} + +std::vector DebayerEGL::formats(PixelFormat inputFormat) +{ + DebayerEGL::DebayerInputConfig config; + + if (getInputConfig(inputFormat, config) != 0) + return std::vector(); + + return config.outputFormats; +} + +std::tuple +DebayerEGL::strideAndFrameSize(const PixelFormat &outputFormat, const Size &size) +{ + DebayerEGL::DebayerOutputConfig config; + + if (getOutputConfig(outputFormat, config) != 0) + return std::make_tuple(0, 0); + + /* round up to multiple of 8 for 64 bits alignment */ + unsigned int stride = (size.width * config.bpp / 8 + 7) & ~7; + + return std::make_tuple(stride, stride * size.height); +} + +void DebayerEGL::setShaderVariableValues(void) +{ + /* + * Raw Bayer 8-bit, and packed raw Bayer 10-bit/12-bit formats + * are stored in a GL_LUMINANCE texture. The texture width is + * equal to the stride. + */ + GLfloat firstRed[] = { firstRed_x_, firstRed_y_ }; + GLfloat imgSize[] = { (GLfloat)width_, + (GLfloat)height_ }; + GLfloat Step[] = { 1.0f / (inputConfig_.stride - 1), + 1.0f / (height_ - 1) }; + GLfloat Stride = 1.0f; + GLfloat projIdentityMatrix[] = { + 1, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0, + 0, 0, 0, 1 + }; + + // vertexIn - bayer_8.vert + glEnableVertexAttribArray(attributeVertex_); + glVertexAttribPointer(attributeVertex_, 2, GL_FLOAT, GL_TRUE, + 2 * sizeof(GLfloat), vcoordinates); + + // textureIn - bayer_8.vert + glEnableVertexAttribArray(attributeTexture_); + glVertexAttribPointer(attributeTexture_, 2, GL_FLOAT, GL_TRUE, + 2 * sizeof(GLfloat), tcoordinates); + + // Set the sampler2D to the respective texture unit for each texutre + // To simultaneously sample multiple textures we need to use multiple + // texture units + glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + + // These values are: + // firstRed = tex_bayer_first_red - bayer_8.vert + // imgSize = tex_size - bayer_8.vert + // step = tex_step - bayer_8.vert + // Stride = stride_factor identity.vert + // textureUniformProjMatri = No scaling + glUniform2fv(textureUniformBayerFirstRed_, 1, firstRed); + glUniform2fv(textureUniformSize_, 1, imgSize); + glUniform2fv(textureUniformStep_, 1, Step); + glUniform1f(textureUniformStrideFactor_, Stride); + glUniformMatrix4fv(textureUniformProjMatrix_, 1, + GL_FALSE, projIdentityMatrix); + + LOG(Debayer, Debug) << "vertexIn " << attributeVertex_ << " textureIn " << attributeTexture_ + << " tex_y " << textureUniformBayerDataIn_ + << " red_param " << textureUniformRedLookupDataIn_ + << " green_param " << textureUniformGreenLookupDataIn_ + << " blue_param " << textureUniformBlueLookupDataIn_ + << " tex_step " << textureUniformStep_ + << " tex_size " << textureUniformSize_ + << " stride_factor " << textureUniformStrideFactor_ + << " tex_bayer_first_red " << textureUniformBayerFirstRed_; + + LOG (Debayer, Debug) << "textureUniformY_ = 0 " << + " firstRed.x " << firstRed[0] << + " firstRed.y " << firstRed[1] << + " textureUniformSize_.width " << imgSize[0] << " " + " textureUniformSize_.height " << imgSize[1] << + " textureUniformStep_.x " << Step[0] << + " textureUniformStep_.y " << Step[1] << + " textureUniformStrideFactor_ " << Stride << + " textureUniformProjMatrix_ " << textureUniformProjMatrix_; + return; +} + +void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, DebayerParams ¶ms) +{ + LOG(Debayer, Debug) + << "Input height " << height_ + << " width " << width_ + << " fd " << in.getPlaneFD(0); + + // eGL context switch + egl_.makeCurrent(); + + // Greate a standard texture + // we will replace this with the DMA version at some point + egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); + + // Populate bayer parameters + egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + + // Setup the scene + setShaderVariableValues(); + glViewport(0, 0, width_, height_); + glClear(GL_COLOR_BUFFER_BIT); + glDisable(GL_BLEND); + + // Draw the scene + glDrawArrays(GL_TRIANGLE_FAN, 0, DEBAYER_OPENGL_COORDS); + + // eglclientWaitScynKhr / eglwaitsynckr ? + egl_.swapBuffers(); + + // Copy from the output GBM buffer to our output plane + // once we get render to texture working the + // explicit lock ioctl, memcpy and unlock ioctl won't be required + gbmSurface_.getFrameBufferData(out.planes()[0].data(), out.planes()[0].size()); +} + +void DebayerEGL::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params) +{ + bench_.startFrame(); + + std::vector dmaSyncers; + + dmaSyncBegin(dmaSyncers, input, output); + + setParams(params); + + /* Copy metadata from the input buffer */ + FrameMetadata &metadata = output->_d()->metadata(); + metadata.status = input->metadata().status; + metadata.sequence = input->metadata().sequence; + metadata.timestamp = input->metadata().timestamp; + + MappedFrameBuffer in(input, MappedFrameBuffer::MapFlag::Read); + MappedFrameBuffer out(output, MappedFrameBuffer::MapFlag::Write); + if (!in.isValid() || !out.isValid()) { + LOG(Debayer, Error) << "mmap-ing buffer(s) failed"; + metadata.status = FrameMetadata::FrameError; + return; + } + + debayerGPU(in, out, params); + + dmaSyncers.clear(); + + bench_.finishFrame(); + + metadata.planes()[0].bytesused = out.planes()[0].size(); + + // Calculate stats for the whole frame + stats_->processFrame(frame, 0, input); + + outputBufferReady.emit(output); + inputBufferReady.emit(input); +} + +SizeRange DebayerEGL::sizes(PixelFormat inputFormat, const Size &inputSize) +{ + Size patternSize = this->patternSize(inputFormat); + unsigned int borderHeight = patternSize.height; + + if (patternSize.isNull()) + return {}; + + /* No need for top/bottom border with a pattern height of 2 */ + if (patternSize.height == 2) + borderHeight = 0; + + /* + * For debayer interpolation a border is kept around the entire image + * and the minimum output size is pattern-height x pattern-width. + */ + if (inputSize.width < (3 * patternSize.width) || + inputSize.height < (2 * borderHeight + patternSize.height)) { + LOG(Debayer, Warning) + << "Input format size too small: " << inputSize.toString(); + return {}; + } + + return SizeRange(Size(patternSize.width, patternSize.height), + Size((inputSize.width - 2 * patternSize.width) & ~(patternSize.width - 1), + (inputSize.height - 2 * borderHeight) & ~(patternSize.height - 1)), + patternSize.width, patternSize.height); +} + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h new file mode 100644 index 000000000..c0fc220f8 --- /dev/null +++ b/src/libcamera/software_isp/debayer_egl.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025, Bryan O'Donoghue. + * + * Authors: + * Bryan O'Donoghue + * + * debayer_opengl.h - EGL debayer header + */ + +#pragma once + +#include +#include +#include + +#define GL_GLEXT_PROTOTYPES +#define EGL_EGLEXT_PROTOTYPES +#include +#include +#include + +#include + +#include "debayer.h" + +#include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/egl.h" +#include "libcamera/internal/framebuffer.h" +#include "libcamera/internal/mapped_framebuffer.h" +#include "libcamera/internal/software_isp/benchmark.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" + +namespace libcamera { + +#define DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS 4 +#define DEBAYER_OPENGL_COORDS 4 + +/** + * \class DebayerEGL + * \brief Class for debayering using an EGL Shader + * + * Implements an EGL shader based debayering solution. + */ +class DebayerEGL : public Debayer +{ +public: + /** + * \brief Constructs a DebayerEGL object. + * \param[in] stats Pointer to the stats object to use. + */ + DebayerEGL(std::unique_ptr stats); + ~DebayerEGL(); + + /* + * Setup the Debayer object according to the passed in parameters. + * Return 0 on success, a negative errno value on failure + * (unsupported parameters). + */ + int configure(const StreamConfiguration &inputCfg, + const std::vector> &outputCfgs, + bool ccmEnabled); + + /* + * Get width and height at which the bayer-pattern repeats. + * Return pattern-size or an empty Size for an unsupported inputFormat. + */ + Size patternSize(PixelFormat inputFormat); + + std::vector formats(PixelFormat input); + std::tuple strideAndFrameSize(const PixelFormat &outputFormat, const Size &size); + + void process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params); + + /** + * \brief Get the file descriptor for the statistics. + * + * \return the file descriptor pointing to the statistics. + */ + const SharedFD &getStatsFD() { return stats_->getStatsFD(); } + + /** + * \brief Get the output frame size. + * + * \return The output frame size. + */ + unsigned int frameSize() { return outputConfig_.frameSize; } + + SizeRange sizes(PixelFormat inputFormat, const Size &inputSize); + +private: + static int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); + static int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); + int setupStandardBayerOrder(BayerFormat::Order order); + void pushEnv(std::vector &shaderEnv, const char *str); + int initBayerShaders(PixelFormat inputFormat, PixelFormat outputFormat); + int initEGLContext(); + int generateTextures(); + int compileShaderProgram(GLuint &shaderId, GLenum shaderType, + unsigned char *shaderData, int shaderDataLen, + std::vector shaderEnv); + int linkShaderProgram(void); + int getShaderVariableLocations(); + void setShaderVariableValues(void); + void configureTexture(GLuint &texture); + void debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, DebayerParams ¶ms); + + // Shader program identifiers + GLuint vertexShaderId_; + GLuint fragmentShaderId_; + GLuint programId_; + enum { + BAYER_INPUT_INDEX = 0, + BAYER_OUTPUT_INDEX, + BAYER_BUF_NUM, + }; + + // Pointer to object representing input texture + eGLImage *eglImageBayerIn_; + + eGLImage *eglImageRedLookup_; + eGLImage *eglImageGreenLookup_; + eGLImage *eglImageBlueLookup_; + + // Shader parameters + float firstRed_x_; + float firstRed_y_; + GLint attributeVertex_; + GLint attributeTexture_; + GLint textureUniformStep_; + GLint textureUniformSize_; + GLint textureUniformStrideFactor_; + GLint textureUniformBayerFirstRed_; + GLint textureUniformProjMatrix_; + + GLint textureUniformBayerDataIn_; + GLint textureUniformRedLookupDataIn_; + GLint textureUniformGreenLookupDataIn_; + GLint textureUniformBlueLookupDataIn_; + + Rectangle window_; + std::unique_ptr stats_; + eGL egl_; + GBM gbmSurface_; + uint32_t width_; + uint32_t height_; + bool ccmEnabled_; + + GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { + { -1.0f, -1.0f }, + { -1.0f, +1.0f }, + { +1.0f, +1.0f }, + { +1.0f, -1.0f }, + }; + + GLfloat tcoordinates[DEBAYER_OPENGL_COORDS][2] = { + { 0.0f, 1.0f }, + { 0.0f, 0.0f }, + { 1.0f, 0.0f }, + { 1.0f, 1.0f }, + }; +}; + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/meson.build b/src/libcamera/software_isp/meson.build index 59fa5f02a..c61ac7d59 100644 --- a/src/libcamera/software_isp/meson.build +++ b/src/libcamera/software_isp/meson.build @@ -2,6 +2,7 @@ softisp_enabled = pipelines.contains('simple') summary({'SoftISP support' : softisp_enabled}, section : 'Configuration') +summary({'SoftISP GPU acceleration' : gles_headless_enabled}, section : 'Configuration') if not softisp_enabled subdir_done() @@ -14,3 +15,10 @@ libcamera_internal_sources += files([ 'software_isp.cpp', 'swstats_cpu.cpp', ]) + +if softisp_enabled and gles_headless_enabled + config_h.set('HAVE_DEBAYER_EGL', 1) + libcamera_internal_sources += files([ + 'debayer_egl.cpp', + ]) +endif From a9fa1ff3c65f9e7f8a1ac201a2befb0f49008885 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 12 Jun 2024 02:08:02 +0100 Subject: [PATCH 029/106] libcamera: software_isp: debayer_egl: Make DebayerEGL an environment option If GPUISP support is available make it so an environment variable can switch it on. Given we don't have full feature parity with CPUISP just yet on pixel format output, we should default to CPUISP mode giving the user the option to switch on GPUISP by setting LIBCAMERA_SOFTISP_MODE=gpu Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/software_isp.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index e8fa8a17a..ff4471c05 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -25,6 +26,9 @@ #include "libcamera/internal/software_isp/debayer_params.h" #include "debayer_cpu.h" +#if HAVE_DEBAYER_EGL +#include "debayer_egl.h" +#endif /** * \file software_isp.cpp @@ -114,7 +118,20 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor, } stats->statsReady.connect(this, &SoftwareIsp::statsReady); - debayer_ = std::make_unique(std::move(stats)); +#if HAVE_DEBAYER_EGL + const char *softISPMode = utils::secure_getenv("LIBCAMERA_SOFTISP_MODE"); + + if (softISPMode && !strcmp(softISPMode, "gpu")) + debayer_ = std::make_unique(std::move(stats)); +#endif + if (!debayer_) + debayer_ = std::make_unique(std::move(stats)); + + if (!debayer_) { + LOG(SoftwareIsp, Error) << "Failed to create Debayer object"; + return; + } + debayer_->inputBufferReady.connect(this, &SoftwareIsp::inputReady); debayer_->outputBufferReady.connect(this, &SoftwareIsp::outputReady); From 60082dd56fd6f0d56ccf8852c8459a40420a9396 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 1 Dec 2024 13:27:12 +0000 Subject: [PATCH 030/106] libcamera: shaders: Use highp not mediump for float precision We get better sample resolution with highp instead of mediump. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/shaders/RGB.frag | 2 +- include/libcamera/internal/shaders/YUV_2_planes.frag | 2 +- include/libcamera/internal/shaders/YUV_3_planes.frag | 2 +- include/libcamera/internal/shaders/YUV_packed.frag | 2 +- include/libcamera/internal/shaders/bayer_1x_packed.frag | 2 +- include/libcamera/internal/shaders/bayer_8.frag | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libcamera/internal/shaders/RGB.frag b/include/libcamera/internal/shaders/RGB.frag index 4c374ac98..724395894 100644 --- a/include/libcamera/internal/shaders/RGB.frag +++ b/include/libcamera/internal/shaders/RGB.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/include/libcamera/internal/shaders/YUV_2_planes.frag b/include/libcamera/internal/shaders/YUV_2_planes.frag index 1d5d12062..d286f1179 100644 --- a/include/libcamera/internal/shaders/YUV_2_planes.frag +++ b/include/libcamera/internal/shaders/YUV_2_planes.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/include/libcamera/internal/shaders/YUV_3_planes.frag b/include/libcamera/internal/shaders/YUV_3_planes.frag index 8f788e90a..8e3e0b4a5 100644 --- a/include/libcamera/internal/shaders/YUV_3_planes.frag +++ b/include/libcamera/internal/shaders/YUV_3_planes.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/include/libcamera/internal/shaders/YUV_packed.frag b/include/libcamera/internal/shaders/YUV_packed.frag index b9ef9d41b..3c9e3e397 100644 --- a/include/libcamera/internal/shaders/YUV_packed.frag +++ b/include/libcamera/internal/shaders/YUV_packed.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index f53f55758..19b13ad08 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -20,7 +20,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif /* diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index 7e35ca88e..aa7a1b004 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -16,7 +16,7 @@ Copyright (C) 2021, Linaro //Pixel Shader #ifdef GL_ES -precision mediump float; +precision highp float; #endif /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ From 60394c45dc6d8a226b537ba2020b8b25e1cb7cbb Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 22 Apr 2025 02:08:59 +0100 Subject: [PATCH 031/106] libcamera: shaders: Extend debayer shaders to apply RGB gain values on output Extend out the bayer fragment shaders to take 3 x 256 byte inputs as textures from the CPU. We then use an index to the table to recover the colour-corrected values provided by the SoftIPA thread. Signed-off-by: Bryan O'Donoghue --- .../internal/shaders/bayer_1x_packed.frag | 56 ++++++++++++++++ .../libcamera/internal/shaders/bayer_8.frag | 62 ++++++++++++++++- src/libcamera/software_isp/debayer_egl.cpp | 67 +++++++++++++------ src/libcamera/software_isp/debayer_egl.h | 7 +- 4 files changed, 169 insertions(+), 23 deletions(-) diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index 19b13ad08..90bd64570 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -65,6 +65,10 @@ uniform vec2 tex_step; uniform vec2 tex_bayer_first_red; uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; +uniform mat3 ccm; void main(void) { @@ -212,5 +216,57 @@ void main(void) vec3(patterns.y, C, patterns.x) : vec3(patterns.wz, C)); +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; +#endif + gl_FragColor = vec4(rgb, 1.0); } diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index aa7a1b004..5955c2eaf 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -21,11 +21,17 @@ precision highp float; /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; varying vec4 center; varying vec4 yCoord; varying vec4 xCoord; +uniform mat3 ccm; void main(void) { + vec3 rgb; + #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r float C = texture2D(tex_y, center.xy).r; // ( 0, 0) @@ -97,11 +103,65 @@ void main(void) { PATTERN.xw += kB.xw * B; PATTERN.xz += kF.xz * F; - gl_FragColor.rgb = (alternate.y == 0.0) ? + rgb = (alternate.y == 0.0) ? ((alternate.x == 0.0) ? vec3(C, PATTERN.xy) : vec3(PATTERN.z, C, PATTERN.w)) : ((alternate.x == 0.0) ? vec3(PATTERN.w, C, PATTERN.z) : vec3(PATTERN.yx, C)); + +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; +#endif + + gl_FragColor.rgb = rgb; } diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 3fb155119..824cd6d3f 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void) textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); + ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm"); textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); @@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void) << " red_param " << textureUniformRedLookupDataIn_ << " green_param " << textureUniformGreenLookupDataIn_ << " blue_param " << textureUniformBlueLookupDataIn_ + << " ccm " << ccmUniformDataIn_ << " tex_step " << textureUniformStep_ << " tex_size " << textureUniformSize_ << " stride_factor " << textureUniformStrideFactor_ @@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm break; }; - // Flag to shaders that we have parameter gain tables - egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + if (ccmEnabled_) { + // Run the CCM if available + egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS"); + } else { + // Flag to shaders that we have parameter gain tables + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + } if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) goto compile_fail; @@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, const std::vector> &outputCfgs, bool ccmEnabled) { + GLint maxTextureImageUnits; + if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) return -EINVAL; @@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, inputConfig_.stride = inputCfg.stride; width_ = inputCfg.size.width; height_ = inputCfg.size.height; - ccmEnabled_ = ccmEnabled = false; + ccmEnabled_ = ccmEnabled = true; if (outputCfgs.size() != 1) { LOG(Debayer, Error) @@ -304,30 +313,35 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; - if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { + if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS << " try using an identity CCM "; return -ENODEV; } + // Raw bayer input as texture eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); if (!eglImageBayerIn_) return -ENOMEM; - /// RGB correction tables as 2d textures - // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate - eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); - if (!eglImageRedLookup_) - return -ENOMEM; + // Only do the RGB lookup table textures if CCM is disabled + if (!ccmEnabled_) { - eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); - if (!eglImageGreenLookup_) - return -ENOMEM; + /// RGB correction tables as 2d textures + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); + if (!eglImageRedLookup_) + return -ENOMEM; - eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); - if (!eglImageBlueLookup_) - return -ENOMEM; + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); + if (!eglImageGreenLookup_) + return -ENOMEM; + + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); + if (!eglImageBlueLookup_) + return -ENOMEM; + } // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) if (gbmSurface_.mapSurface()) @@ -440,9 +454,11 @@ void DebayerEGL::setShaderVariableValues(void) // To simultaneously sample multiple textures we need to use multiple // texture units glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); - glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + if (!ccmEnabled_) { + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + } // These values are: // firstRed = tex_bayer_first_red - bayer_8.vert @@ -494,9 +510,18 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); // Populate bayer parameters - egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); - egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); - egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + if (ccmEnabled_) { + GLfloat ccm[] = { + 1, 0, 0, + 0, 1, 0, + 0, 0, 1, + }; + glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); + } else { + egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + } // Setup the scene setShaderVariableValues(); diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h index c0fc220f8..94bc6fc41 100644 --- a/src/libcamera/software_isp/debayer_egl.h +++ b/src/libcamera/software_isp/debayer_egl.h @@ -134,17 +134,22 @@ private: GLint textureUniformProjMatrix_; GLint textureUniformBayerDataIn_; + + // These textures will either point to simple RGB gains or to CCM lookup tables GLint textureUniformRedLookupDataIn_; GLint textureUniformGreenLookupDataIn_; GLint textureUniformBlueLookupDataIn_; + // Represent per-frame CCM as a uniform vector of floats 3 x 3 + GLint ccmUniformDataIn_; + bool ccmEnabled_; + Rectangle window_; std::unique_ptr stats_; eGL egl_; GBM gbmSurface_; uint32_t width_; uint32_t height_; - bool ccmEnabled_; GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { { -1.0f, -1.0f }, From 2b7bca4a149bcb03a487dcb4ebb46c4c17f241ca Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 10 May 2025 06:25:46 +0100 Subject: [PATCH 032/106] libcamera: software_isp: debayer_egl: Convert from identity CCM to CCM calculated by SoftIPA Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer_egl.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 824cd6d3f..b30d21077 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -293,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, inputConfig_.stride = inputCfg.stride; width_ = inputCfg.size.width; height_ = inputCfg.size.height; - ccmEnabled_ = ccmEnabled = true; + ccmEnabled_ = ccmEnabled; if (outputCfgs.size() != 1) { LOG(Debayer, Error) @@ -511,10 +511,10 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay // Populate bayer parameters if (ccmEnabled_) { - GLfloat ccm[] = { - 1, 0, 0, - 0, 1, 0, - 0, 0, 1, + GLfloat ccm[9] = { + params.ccm[0][0], params.ccm[0][1], params.ccm[0][2], + params.ccm[1][0], params.ccm[1][1], params.ccm[1][2], + params.ccm[2][0], params.ccm[2][1], params.ccm[2][2], }; glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); } else { From c7346319bd90fdb4aabf2674c1afdfa5985a6ac3 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 10 May 2025 06:26:30 +0100 Subject: [PATCH 033/106] libcamera: software_isp: Switch on uncalibrated CCM to validate eGLDebayer Signed-off-by: Bryan O'Donoghue --- src/ipa/simple/data/uncalibrated.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index 5508e6686..8b6df9afc 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -8,12 +8,12 @@ algorithms: # Color correction matrices can be defined here. The CCM algorithm # has a significant performance impact, and should only be enabled # if tuned. - # - Ccm: - # ccms: - # - ct: 6500 - # ccm: [ 1, 0, 0, - # 0, 1, 0, - # 0, 0, 1] + - Ccm: + ccms: + - ct: 6500 + ccm: [ 1, 0, 0, + 0, 1, 0, + 0, 0, 1] - Lut: - Agc: ... From 53930ee1d723ca1ba9ea8bcd7b211662fb71670d Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 30 May 2025 11:43:28 +0100 Subject: [PATCH 034/106] libcamera: software_isp: Make isStandardBayerOrder static Make this member function static so that it may be called from static members of derived classes. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index 214bcdd3c..3893318b2 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -88,7 +88,7 @@ private: protected: void setParams(DebayerParams ¶ms); void dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output); - bool isStandardBayerOrder(BayerFormat::Order order); + static bool isStandardBayerOrder(BayerFormat::Order order); }; } /* namespace libcamera */ From 4a9a1dff88e13d3b547082c61047709d317d7d27 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 30 May 2025 11:50:47 +0100 Subject: [PATCH 035/106] libcamera: software_isp: debayer_cpu: Make getInputConfig and getOutputConfig static Make getInputConfig and getOutputConfig static so as to allow for interrogation of the supported pixel formats prior to object instantiation. Do this so as to allow the higher level logic make an informed choice between CPU and GPU ISP based on which pixel formats are supported. Curretnly CPU ISP supports more diverse input and output schemes. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer_cpu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index a043a9541..999e3421c 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -102,8 +102,8 @@ private: template void debayer10P_RGRG_BGR888(uint8_t *dst, const uint8_t *src[]); - int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); - int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); + static int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); + static int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); int setupStandardBayerOrder(BayerFormat::Order order); int setDebayerFunctions(PixelFormat inputFormat, PixelFormat outputFormat, From 6637b468b70be2827b648b4b1bb6156c92384bda Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 4 Jun 2025 16:51:16 +0100 Subject: [PATCH 036/106] libcamera: shaders: Extend bayer shaders to support swapping R and B on output We can easily facilitate swapping R and B on output. Pivot on an environment define for this purpose. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/shaders/bayer_1x_packed.frag | 4 ++++ include/libcamera/internal/shaders/bayer_8.frag | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index 90bd64570..c0632eb1f 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -268,5 +268,9 @@ void main(void) rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; #endif +#if defined (SWAP_BLUE) + gl_FragColor = vec4(rgb.bgr, 1.0); +#else gl_FragColor = vec4(rgb, 1.0); +#endif } diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index 5955c2eaf..74ce15096 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -163,5 +163,9 @@ void main(void) { rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; #endif - gl_FragColor.rgb = rgb; +#if defined (SWAP_BLUE) + gl_FragColor = vec4(rgb.bgr, 1.0); +#else + gl_FragColor = vec4(rgb, 1.0); +#endif } From 9b66144aad4088e99cc5fc3ad215f8f1620c78cb Mon Sep 17 00:00:00 2001 From: Milan Zamazal Date: Tue, 29 Apr 2025 13:56:25 +0200 Subject: [PATCH 037/106] libcamera: shaders: Fix neighbouring positions in 8-bit debayering When accessing a texture position in a shader, the pixel with the nearest centre to the specified texture coordinates (as mandated by specifying GL_NEAREST parameter) is taken. The current vertex shader determines the positions of the neighbouring pixels by adding the provided texture steps to the exact centre pixel coordinates. But this places the computed coordinates, from the point of view of GL_NEAREST, exactly between the pixels and is thus prone to floating point inaccuracies. Wrong neighbouring pixel coordinates may be used, resulting in artefacts in the output image. Let's fix the problem by shifting the initial coordinates a bit from the pixel border. Signed-off-by: Milan Zamazal --- include/libcamera/internal/shaders/bayer_8.vert | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/libcamera/internal/shaders/bayer_8.vert b/include/libcamera/internal/shaders/bayer_8.vert index fb5109eee..fc1cf89f2 100644 --- a/include/libcamera/internal/shaders/bayer_8.vert +++ b/include/libcamera/internal/shaders/bayer_8.vert @@ -44,10 +44,10 @@ void main(void) { center.xy = textureIn; center.zw = textureIn * tex_size + tex_bayer_first_red; - xCoord = center.x + vec4(-2.0 * tex_step.x, - -tex_step.x, tex_step.x, 2.0 * tex_step.x); - yCoord = center.y + vec4(-2.0 * tex_step.y, - -tex_step.y, tex_step.y, 2.0 * tex_step.y); + xCoord = center.x + 0.1 * tex_step.x + + vec4(-2.0 * tex_step.x, -tex_step.x, tex_step.x, 2.0 * tex_step.x); + yCoord = center.y + 0.1 * tex_step.y + + vec4(-2.0 * tex_step.y, -tex_step.y, tex_step.y, 2.0 * tex_step.y); gl_Position = proj_matrix * vertexIn; } From f28498a2fb878d1786de5cc5ce1dd16af0ee597e Mon Sep 17 00:00:00 2001 From: Milan Zamazal Date: Tue, 29 Apr 2025 14:19:59 +0200 Subject: [PATCH 038/106] libcamera: software_isp: GPU support for unpacked 10/12-bit formats The GPU processing supports 8-bit sensor formats and 10/12-bit packed formats. Support for 10/12-bit unpacked formats is missing, let's add it. 10/12-bit unpacked formats use two adjacent bytes to store the value. This means the 8-bit shaders can be used if we can modify them for additional support of 16-bit addressing. This requires the following modifications: - Using GL_RG (two bytes per pixel) instead of GL_LUMINANCE (one byte per pixel) as the texture format for the given input formats. - Setting the texture width to the number of pixels rather than the number of bytes. - Making the definition of `fetch' macro variable, according to the pixel format. - Using only `fetch' for accessing the texture. Signed-off-by: Milan Zamazal Signed-off-by: Bryan O'Donoghue --- .../libcamera/internal/shaders/bayer_8.frag | 10 +++- src/libcamera/software_isp/debayer_egl.cpp | 46 +++++++++++++------ src/libcamera/software_isp/debayer_egl.h | 2 + 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index 74ce15096..78c2609c2 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -32,9 +32,17 @@ uniform mat3 ccm; void main(void) { vec3 rgb; + #if defined(RAW10P) + #define pixel(p) p.r / 4.0 + p.g * 64.0 + #define fetch(x, y) pixel(texture2D(tex_y, vec2(x, y))) + #elif defined(RAW12P) + #define pixel(p) p.r / 16.0 + p.g * 16.0 + #define fetch(x, y) pixel(texture2D(tex_y, vec2(x, y))) + #else #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r + #endif - float C = texture2D(tex_y, center.xy).r; // ( 0, 0) + float C = fetch(center.x, center.y); // ( 0, 0) const vec4 kC = vec4( 4.0, 6.0, 5.0, 5.0) / 8.0; // Determine which of four types of pixels we are on. diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index b30d21077..71742d849 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -151,6 +151,8 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm } // Pixel location parameters + glFormat_ = GL_LUMINANCE; + bytesPerPixel_ = 1; switch (inputFormat) { case libcamera::formats::SBGGR8: case libcamera::formats::SBGGR10_CSI2P: @@ -197,20 +199,38 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm case libcamera::formats::SGRBG10_CSI2P: case libcamera::formats::SRGGB10_CSI2P: egl_.pushEnv(shaderEnv, "#define RAW10P"); - fragmentShaderData = bayer_1x_packed_frag; - fragmentShaderDataLen = bayer_1x_packed_frag_len; - vertexShaderData = identity_vert; - vertexShaderDataLen = identity_vert_len; + if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { + fragmentShaderData = bayer_8_frag; + fragmentShaderDataLen = bayer_8_frag_len; + vertexShaderData = bayer_8_vert; + vertexShaderDataLen = bayer_8_vert_len; + glFormat_ = GL_RG; + bytesPerPixel_ = 2; + } else { + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + } break; case libcamera::formats::SBGGR12_CSI2P: case libcamera::formats::SGBRG12_CSI2P: case libcamera::formats::SGRBG12_CSI2P: case libcamera::formats::SRGGB12_CSI2P: egl_.pushEnv(shaderEnv, "#define RAW12P"); - fragmentShaderData = bayer_1x_packed_frag; - fragmentShaderDataLen = bayer_1x_packed_frag_len; - vertexShaderData = identity_vert; - vertexShaderDataLen = identity_vert_len; + if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { + fragmentShaderData = bayer_8_frag; + fragmentShaderDataLen = bayer_8_frag_len; + vertexShaderData = bayer_8_vert; + vertexShaderDataLen = bayer_8_vert_len; + glFormat_ = GL_RG; + bytesPerPixel_ = 2; + } else { + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + } break; default: goto invalid_fmt; @@ -430,7 +450,7 @@ void DebayerEGL::setShaderVariableValues(void) GLfloat firstRed[] = { firstRed_x_, firstRed_y_ }; GLfloat imgSize[] = { (GLfloat)width_, (GLfloat)height_ }; - GLfloat Step[] = { 1.0f / (inputConfig_.stride - 1), + GLfloat Step[] = { static_cast(bytesPerPixel_) / (inputConfig_.stride - 1), 1.0f / (height_ - 1) }; GLfloat Stride = 1.0f; GLfloat projIdentityMatrix[] = { @@ -507,7 +527,7 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay // Greate a standard texture // we will replace this with the DMA version at some point - egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); + egl_.createTexture2D(eglImageBayerIn_, glFormat_, inputConfig_.stride / bytesPerPixel_, height_, in.planes()[0].data()); // Populate bayer parameters if (ccmEnabled_) { @@ -518,9 +538,9 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay }; glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); } else { - egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); - egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); - egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + egl_.createTexture2D(eglImageRedLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); } // Setup the scene diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h index 94bc6fc41..56f5434ac 100644 --- a/src/libcamera/software_isp/debayer_egl.h +++ b/src/libcamera/software_isp/debayer_egl.h @@ -150,6 +150,8 @@ private: GBM gbmSurface_; uint32_t width_; uint32_t height_; + GLint glFormat_; + unsigned int bytesPerPixel_; GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { { -1.0f, -1.0f }, From a2b7c63532670b3817b0719c1723ecc1d3f08daa Mon Sep 17 00:00:00 2001 From: Milan Zamazal Date: Tue, 29 Apr 2025 14:37:05 +0200 Subject: [PATCH 039/106] libcamera: shaders: Rename bayer_8 to bayer_unpacked bayer_8.* shaders are now used for all unpacked sensor data formats, regardless of the pixel bit width. Let's rename the "8-bit" shaders to avoid confusion. Signed-off-by: Milan Zamazal Signed-off-by: Bryan O'Donoghue --- .../{bayer_8.frag => bayer_unpacked.frag} | 0 .../{bayer_8.vert => bayer_unpacked.vert} | 0 .../libcamera/internal/shaders/meson.build | 4 ++-- src/apps/qcam/assets/shader/shaders.qrc | 4 ++-- src/apps/qcam/viewfinder_gl.cpp | 16 ++++++------- src/libcamera/software_isp/debayer_egl.cpp | 24 +++++++++---------- 6 files changed, 24 insertions(+), 24 deletions(-) rename include/libcamera/internal/shaders/{bayer_8.frag => bayer_unpacked.frag} (100%) rename include/libcamera/internal/shaders/{bayer_8.vert => bayer_unpacked.vert} (100%) diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_unpacked.frag similarity index 100% rename from include/libcamera/internal/shaders/bayer_8.frag rename to include/libcamera/internal/shaders/bayer_unpacked.frag diff --git a/include/libcamera/internal/shaders/bayer_8.vert b/include/libcamera/internal/shaders/bayer_unpacked.vert similarity index 100% rename from include/libcamera/internal/shaders/bayer_8.vert rename to include/libcamera/internal/shaders/bayer_unpacked.vert diff --git a/include/libcamera/internal/shaders/meson.build b/include/libcamera/internal/shaders/meson.build index 386b342d0..dd441a577 100644 --- a/include/libcamera/internal/shaders/meson.build +++ b/include/libcamera/internal/shaders/meson.build @@ -4,7 +4,7 @@ # for the purposes of inclusion in OpenGL debayering shader_files = files([ 'bayer_1x_packed.frag', - 'bayer_8.frag', - 'bayer_8.vert', + 'bayer_unpacked.frag', + 'bayer_unpacked.vert', 'identity.vert', ]) diff --git a/src/apps/qcam/assets/shader/shaders.qrc b/src/apps/qcam/assets/shader/shaders.qrc index 04f9d7061..32dfa51bf 100644 --- a/src/apps/qcam/assets/shader/shaders.qrc +++ b/src/apps/qcam/assets/shader/shaders.qrc @@ -6,8 +6,8 @@ ../../../../../include/libcamera/internal/shaders/YUV_3_planes.frag ../../../../../include/libcamera/internal/shaders/YUV_packed.frag ../../../../../include/libcamera/internal/shaders/bayer_1x_packed.frag - ../../../../../include/libcamera/internal/shaders/bayer_8.frag - ../../../../../include/libcamera/internal/shaders/bayer_8.vert + ../../../../../include/libcamera/internal/shaders/bayer_unpacked.frag + ../../../../../include/libcamera/internal/shaders/bayer_unpacked.vert ../../../../../include/libcamera/internal/shaders/identity.vert diff --git a/src/apps/qcam/viewfinder_gl.cpp b/src/apps/qcam/viewfinder_gl.cpp index 70f600650..95965ab71 100644 --- a/src/apps/qcam/viewfinder_gl.cpp +++ b/src/apps/qcam/viewfinder_gl.cpp @@ -235,29 +235,29 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) case libcamera::formats::SBGGR8: firstRed_.setX(1.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; - fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG8: firstRed_.setX(0.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; - fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG8: firstRed_.setX(1.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; - fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB8: firstRed_.setX(0.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; - fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SBGGR10_CSI2P: diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 71742d849..9ec966608 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -189,10 +189,10 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm case libcamera::formats::SGBRG8: case libcamera::formats::SGRBG8: case libcamera::formats::SRGGB8: - fragmentShaderData = bayer_8_frag; - fragmentShaderDataLen = bayer_8_frag_len; - vertexShaderData = bayer_8_vert; - vertexShaderDataLen = bayer_8_vert_len; + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; break; case libcamera::formats::SBGGR10_CSI2P: case libcamera::formats::SGBRG10_CSI2P: @@ -200,10 +200,10 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm case libcamera::formats::SRGGB10_CSI2P: egl_.pushEnv(shaderEnv, "#define RAW10P"); if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { - fragmentShaderData = bayer_8_frag; - fragmentShaderDataLen = bayer_8_frag_len; - vertexShaderData = bayer_8_vert; - vertexShaderDataLen = bayer_8_vert_len; + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; glFormat_ = GL_RG; bytesPerPixel_ = 2; } else { @@ -219,10 +219,10 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm case libcamera::formats::SRGGB12_CSI2P: egl_.pushEnv(shaderEnv, "#define RAW12P"); if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { - fragmentShaderData = bayer_8_frag; - fragmentShaderDataLen = bayer_8_frag_len; - vertexShaderData = bayer_8_vert; - vertexShaderDataLen = bayer_8_vert_len; + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; glFormat_ = GL_RG; bytesPerPixel_ = 2; } else { From e0c8eb0913ccc716d0f06596f9929e16ae285e69 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 10 Jun 2025 16:06:01 +0100 Subject: [PATCH 040/106] libcamera: software_isp: Add a gpuisp todo list List the series of things to do in GPU ISP in perceived order of difficulty. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/gpuisp-todo.txt | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/libcamera/software_isp/gpuisp-todo.txt diff --git a/src/libcamera/software_isp/gpuisp-todo.txt b/src/libcamera/software_isp/gpuisp-todo.txt new file mode 100644 index 000000000..0ff82f81e --- /dev/null +++ b/src/libcamera/software_isp/gpuisp-todo.txt @@ -0,0 +1,42 @@ +List the TODOs in perceived order of ease. + +24 bit output support: + - Take the BPP we already capture and get a 24 bit GBM surface + - Pass a compile-time parameter to the shaders to tell them to do + gl_FragColor = rgb not gl_FragColor = rgba + +Make GPUISP default: + - Right now the environment variable allows over-riding to swtich + from CPU to GPU. + - Once we support 24 BPP output on GPUISP we will have the same + pixel format support as CPU and can set the default to GPU without + regressing functionality + +glTexture1D: + - Initial code was developed for < GLES 2.O but since we have fixed + on GLES >= 2.0 this means we can use glTexture1D + - Provided this is so amend the shaders to do val = texture(x, y, 0); + not texture(x, y, 0.5) the 0.5 is because of using glTexture2D + +Surfaceless GBM: + - We get a GBM surface and then have to swap buffers + If we rework for surfaceless GBM and EGL then the swap buffer can + be dropped. + +dma-buf texture upload: + - Currently we pass the input buffer to glCreateTexture2D. + We should be able to make the upload of the input buffer go faster + by using eglCreateImageKHR and enumerated the dma-buf contents. + +Render-to-texture: + - Right now we render to the GBM provided surface framebuffer + and then memcpy from that buffer to the target output buffer. + This necessitates flushing the cache on the target buffer in + addition to the memcpy(). + - Render-to-texture where we generate the target framebuffer + directly from a dma-buf handle will mitigate the memcpy() phase. + - It should be the case then that the consumer of the output buffer + i.e. the thing that's not libcamera is responsible to flush the cache + if-and-only-if that user writes to the buffer. + - We need to flush the cache on the buffer because we are memcpying() to it. + From 9613459d30c1504c8c4ebd6b7241fa169f2eefd1 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sun, 23 Feb 2025 23:03:48 +0300 Subject: [PATCH 041/106] libcamera: software_isp: Add brightness control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 29 +++++++++++++++++++++++++++++ src/ipa/simple/algorithms/agc.h | 7 +++++++ src/ipa/simple/ipa_context.h | 2 ++ 3 files changed, 38 insertions(+) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index c46bb0ebe..249caa264 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -41,6 +41,33 @@ Agc::Agc() { } +int Agc::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + return 0; +} + +int Agc::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.brightness = std::optional(); + + return 0; +} + +void Agc::queueRequest(typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &brightness = controls.get(controls::Brightness); + if (brightness.has_value()) { + context.activeState.knobs.brightness = brightness; + LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); + } +} + void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) { /* @@ -54,6 +81,8 @@ void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, dou double next; int32_t &exposure = frameContext.sensor.exposure; double &again = frameContext.sensor.gain; + const auto brightness = context.activeState.knobs.brightness.value_or(1.0); + exposureMSV /= brightness; if (exposureMSV < kExposureOptimal - kExposureSatisfactory) { next = exposure * kExpNumeratorUp / kExpDenominator; diff --git a/src/ipa/simple/algorithms/agc.h b/src/ipa/simple/algorithms/agc.h index 112d9f5a1..00bc101e7 100644 --- a/src/ipa/simple/algorithms/agc.h +++ b/src/ipa/simple/algorithms/agc.h @@ -19,6 +19,13 @@ public: Agc(); ~Agc() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; void process(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const SwIspStats *stats, diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index a471b80ae..afc28ba22 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -64,6 +64,8 @@ struct IPAActiveState { /* 0..2 range, 1.0 = normal */ std::optional contrast; std::optional saturation; + /* 0..2 range, 1.0 = normal */ + std::optional brightness; } knobs; }; From 9f2edfa764612db4b8a6afb42ba8344ab78f24a1 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sun, 16 Mar 2025 22:12:10 +0300 Subject: [PATCH 042/106] libcamera: software_isp: Add AGC disable control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 11 +++++++++++ src/ipa/simple/ipa_context.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index 249caa264..de20863d8 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -44,6 +44,7 @@ Agc::Agc() int Agc::init(IPAContext &context, [[maybe_unused]] const YamlObject &tuningData) { + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); return 0; } @@ -52,6 +53,7 @@ int Agc::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { context.activeState.knobs.brightness = std::optional(); + context.activeState.knobs.ae_enabled = std::optional(); return 0; } @@ -62,10 +64,15 @@ void Agc::queueRequest(typename Module::Context &context, const ControlList &controls) { const auto &brightness = controls.get(controls::Brightness); + const auto &ae_enabled = controls.get(controls::AeEnable); if (brightness.has_value()) { context.activeState.knobs.brightness = brightness; LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); } + if (ae_enabled.has_value()) { + context.activeState.knobs.ae_enabled = ae_enabled; + LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); + } } void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) @@ -132,6 +139,10 @@ void Agc::process(IPAContext &context, const SwIspStats *stats, ControlList &metadata) { + const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); + if (!ae_enable) + return; + utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index afc28ba22..8e8add4c3 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -66,6 +66,8 @@ struct IPAActiveState { std::optional saturation; /* 0..2 range, 1.0 = normal */ std::optional brightness; + /* 0..1 range, 1 = normal */ + std::optional ae_enabled; } knobs; }; From 37aa6edbf3be46a718dc215061d75e3cf79bd4c5 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Mon, 17 Mar 2025 04:24:56 +0300 Subject: [PATCH 043/106] libcamera: software_isp: Add focus control Signed-off-by: Vasiliy Doylov --- .../internal/software_isp/software_isp.h | 4 +- include/libcamera/ipa/soft.mojom | 3 +- src/ipa/simple/algorithms/af.cpp | 71 +++++++++++++++++++ src/ipa/simple/algorithms/af.h | 40 +++++++++++ src/ipa/simple/algorithms/meson.build | 1 + src/ipa/simple/data/uncalibrated.yaml | 1 + src/ipa/simple/ipa_context.h | 9 +++ src/ipa/simple/soft_simple.cpp | 18 ++++- src/libcamera/pipeline/simple/simple.cpp | 30 ++++++-- src/libcamera/software_isp/software_isp.cpp | 4 +- 10 files changed, 169 insertions(+), 12 deletions(-) create mode 100644 src/ipa/simple/algorithms/af.cpp create mode 100644 src/ipa/simple/algorithms/af.h diff --git a/include/libcamera/internal/software_isp/software_isp.h b/include/libcamera/internal/software_isp/software_isp.h index ad89c9b3c..9e5e05fc0 100644 --- a/include/libcamera/internal/software_isp/software_isp.h +++ b/include/libcamera/internal/software_isp/software_isp.h @@ -86,11 +86,11 @@ public: Signal outputBufferReady; Signal ispStatsReady; Signal metadataReady; - Signal setSensorControls; + Signal setSensorControls; private: void saveIspParams(); - void setSensorCtrls(const ControlList &sensorControls); + void setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls); void statsReady(uint32_t frame, uint32_t bufferId); void inputReady(FrameBuffer *input); void outputReady(FrameBuffer *output); diff --git a/include/libcamera/ipa/soft.mojom b/include/libcamera/ipa/soft.mojom index 77328c5fd..e5767532c 100644 --- a/include/libcamera/ipa/soft.mojom +++ b/include/libcamera/ipa/soft.mojom @@ -10,6 +10,7 @@ import "include/libcamera/ipa/core.mojom"; struct IPAConfigInfo { libcamera.ControlInfoMap sensorControls; + libcamera.ControlInfoMap lensControls; }; interface IPASoftInterface { @@ -32,7 +33,7 @@ interface IPASoftInterface { }; interface IPASoftEventInterface { - setSensorControls(libcamera.ControlList sensorControls); + setSensorControls(libcamera.ControlList sensorControls, libcamera.ControlList lensControls); setIspParams(); metadataReady(uint32 frame, libcamera.ControlList metadata); }; diff --git a/src/ipa/simple/algorithms/af.cpp b/src/ipa/simple/algorithms/af.cpp new file mode 100644 index 000000000..b51ed95e4 --- /dev/null +++ b/src/ipa/simple/algorithms/af.cpp @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#include "af.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftAutoFocus) + +namespace ipa::soft::algorithms { + +Af::Af() +{ +} + +int Af::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::LensPosition] = ControlInfo(0.0f, 100.0f, 50.0f); + return 0; +} + +int Af::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.focus_pos = std::optional(); + + return 0; +} + +void Af::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &focus_pos = controls.get(controls::LensPosition); + if (focus_pos.has_value()) { + context.activeState.knobs.focus_pos = focus_pos; + LOG(IPASoftAutoFocus, Debug) << "Setting focus position to " << focus_pos.value(); + } +} + +void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAFrameContext &frameContext, [[maybe_unused]] double exposureMSV) +{ + frameContext.lens.focus_pos = context.activeState.knobs.focus_pos.value_or(50.0) / 100.0 * (context.configuration.focus.focus_max - context.configuration.focus.focus_min); +} + +void Af::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const SwIspStats *stats, + [[maybe_unused]] ControlList &metadata) +{ + updateFocus(context, frameContext, 0); +} + +REGISTER_IPA_ALGORITHM(Af, "Af") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/af.h b/src/ipa/simple/algorithms/af.h new file mode 100644 index 000000000..a575ef102 --- /dev/null +++ b/src/ipa/simple/algorithms/af.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Af : public Algorithm +{ +public: + Af(); + ~Af() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateFocus(IPAContext &context, IPAFrameContext &frameContext, double focus); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build index 2d0adb059..dec59ee8c 100644 --- a/src/ipa/simple/algorithms/meson.build +++ b/src/ipa/simple/algorithms/meson.build @@ -6,4 +6,5 @@ soft_simple_ipa_algorithms = files([ 'blc.cpp', 'ccm.cpp', 'lut.cpp', + 'af.cpp', ]) diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index 8b6df9afc..9274c11e1 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -16,4 +16,5 @@ algorithms: 0, 0, 1] - Lut: - Agc: + - Af: ... diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index 8e8add4c3..71b9bb637 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -34,6 +34,9 @@ struct IPASessionConfiguration { struct { std::optional level; } black; + struct { + int32_t focus_min, focus_max; + } focus; }; struct IPAActiveState { @@ -68,6 +71,8 @@ struct IPAActiveState { std::optional brightness; /* 0..1 range, 1 = normal */ std::optional ae_enabled; + /* 0..100 range, 50.0 = normal */ + std::optional focus_pos; } knobs; }; @@ -81,6 +86,10 @@ struct IPAFrameContext : public FrameContext { double gain; } sensor; + struct { + int32_t focus_pos; + } lens; + struct { double red; double blue; diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp index c94c4cd55..53fa74c03 100644 --- a/src/ipa/simple/soft_simple.cpp +++ b/src/ipa/simple/soft_simple.cpp @@ -77,6 +77,7 @@ private: SwIspStats *stats_; std::unique_ptr camHelper_; ControlInfoMap sensorInfoMap_; + ControlInfoMap lensInfoMap_; /* Local parameter storage */ struct IPAContext context_; @@ -196,6 +197,7 @@ int IPASoftSimple::init(const IPASettings &settings, int IPASoftSimple::configure(const IPAConfigInfo &configInfo) { sensorInfoMap_ = configInfo.sensorControls; + lensInfoMap_ = configInfo.lensControls; const ControlInfo &exposureInfo = sensorInfoMap_.find(V4L2_CID_EXPOSURE)->second; const ControlInfo &gainInfo = sensorInfoMap_.find(V4L2_CID_ANALOGUE_GAIN)->second; @@ -205,6 +207,17 @@ int IPASoftSimple::configure(const IPAConfigInfo &configInfo) context_.activeState = {}; context_.frameContexts.clear(); + if (lensInfoMap_.empty()) { + LOG(IPASoft, Warning) << "No camera leans found! Focus control disabled."; + context_.configuration.focus.focus_min = 0; + context_.configuration.focus.focus_max = 0; + } else { + const ControlInfo &lensInfo = lensInfoMap_.find(V4L2_CID_FOCUS_ABSOLUTE)->second; + context_.configuration.focus.focus_min = lensInfo.min().get(); + context_.configuration.focus.focus_max = lensInfo.max().get(); + LOG(IPASoft, Warning) << "Camera leans found! Focus: " << context_.configuration.focus.focus_min << "-" << context_.configuration.focus.focus_max; + } + context_.configuration.agc.lineDuration = context_.sensorInfo.minLineLength * 1.0s / context_.sensorInfo.pixelRate; context_.configuration.agc.exposureMin = exposureInfo.min().get(); @@ -327,7 +340,10 @@ void IPASoftSimple::processStats(const uint32_t frame, ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(camHelper_ ? camHelper_->gainCode(againNew) : againNew)); - setSensorControls.emit(ctrls); + ControlList lens_ctrls(lensInfoMap_); + lens_ctrls.set(V4L2_CID_FOCUS_ABSOLUTE, frameContext.lens.focus_pos); + + setSensorControls.emit(ctrls, lens_ctrls); } std::string IPASoftSimple::logPrefix() const diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp index efb07051b..4c002ca68 100644 --- a/src/libcamera/pipeline/simple/simple.cpp +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -30,6 +30,7 @@ #include #include "libcamera/internal/camera.h" +#include "libcamera/internal/camera_lens.h" #include "libcamera/internal/camera_sensor.h" #include "libcamera/internal/camera_sensor_properties.h" #include "libcamera/internal/converter.h" @@ -41,6 +42,8 @@ #include "libcamera/internal/v4l2_subdevice.h" #include "libcamera/internal/v4l2_videodevice.h" +#include "libcamera/controls.h" + namespace libcamera { LOG_DEFINE_CATEGORY(SimplePipeline) @@ -356,7 +359,7 @@ private: void ispStatsReady(uint32_t frame, uint32_t bufferId); void metadataReady(uint32_t frame, const ControlList &metadata); - void setSensorControls(const ControlList &sensorControls); + void setSensorControls(const ControlList &sensorControls, const ControlList &lensControls); }; class SimpleCameraConfiguration : public CameraConfiguration @@ -1002,7 +1005,7 @@ void SimpleCameraData::metadataReady(uint32_t frame, const ControlList &metadata tryCompleteRequest(info->request); } -void SimpleCameraData::setSensorControls(const ControlList &sensorControls) +void SimpleCameraData::setSensorControls(const ControlList &sensorControls, const ControlList &lensControls) { delayedCtrls_->push(sensorControls); /* @@ -1013,10 +1016,21 @@ void SimpleCameraData::setSensorControls(const ControlList &sensorControls) * but it also bypasses delayedCtrls_, creating AGC regulation issues. * Both problems should be fixed. */ - if (!frameStartEmitter_) { - ControlList ctrls(sensorControls); - sensor_->setControls(&ctrls); - } + if (frameStartEmitter_) + return; + + ControlList ctrls(sensorControls); + sensor_->setControls(&ctrls); + + CameraLens *focusLens = sensor_->focusLens(); + if (!focusLens) + return; + + if (!lensControls.contains(V4L2_CID_FOCUS_ABSOLUTE)) + return; + + const ControlValue &focusValue = lensControls.get(V4L2_CID_FOCUS_ABSOLUTE); + focusLens->setFocusPosition(focusValue.get()); } /* Retrieve all source pads connected to a sink pad through active routes. */ @@ -1406,6 +1420,10 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) } else { ipa::soft::IPAConfigInfo configInfo; configInfo.sensorControls = data->sensor_->controls(); + if (data->sensor_->focusLens() != nullptr) + configInfo.lensControls = data->sensor_->focusLens()->controls(); + else + configInfo.lensControls = ControlInfoMap(); return data->swIsp_->configure(inputCfg, outputCfgs, configInfo); } } diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index ff4471c05..0e4693b77 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -420,9 +420,9 @@ void SoftwareIsp::saveIspParams() debayerParams_ = *sharedParams_; } -void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls) +void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls) { - setSensorControls.emit(sensorControls); + setSensorControls.emit(sensorControls, lensControls); } void SoftwareIsp::statsReady(uint32_t frame, uint32_t bufferId) From 21501e6e538809d95c2b315f0645b733c21e5da1 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 21 May 2025 21:24:09 +0300 Subject: [PATCH 044/106] libcamera: software_isp: Add manual exposure control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 14 ++++++++++++-- src/ipa/simple/ipa_context.h | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index de20863d8..35f4798cf 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -46,6 +46,8 @@ int Agc::init(IPAContext &context, { context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + context.ctrlMap[&controls::ExposureValue] = ControlInfo(0.0f, 0.5f, 1.0f); + return 0; } @@ -65,6 +67,7 @@ void Agc::queueRequest(typename Module::Context &context, { const auto &brightness = controls.get(controls::Brightness); const auto &ae_enabled = controls.get(controls::AeEnable); + const auto &exposure_value = controls.get(controls::ExposureValue); if (brightness.has_value()) { context.activeState.knobs.brightness = brightness; LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); @@ -73,6 +76,10 @@ void Agc::queueRequest(typename Module::Context &context, context.activeState.knobs.ae_enabled = ae_enabled; LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); } + if (exposure_value.has_value()) { + context.activeState.knobs.exposure_value = exposure_value.value(); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << exposure_value.value(); + } } void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) @@ -140,13 +147,16 @@ void Agc::process(IPAContext &context, ControlList &metadata) { const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); - if (!ae_enable) - return; + if (!ae_enable) + frameContext.sensor.exposure = (int32_t)( context.activeState.knobs.exposure_value.value_or(0.5) * (context.configuration.agc.exposureMax - context.configuration.agc.exposureMin)); utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << frameContext.sensor.exposure; + if (!ae_enable) + return; /* * Calculate Mean Sample Value (MSV) according to formula from: diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index 71b9bb637..c5b5527b5 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -71,6 +71,8 @@ struct IPAActiveState { std::optional brightness; /* 0..1 range, 1 = normal */ std::optional ae_enabled; + /* 0..1 range, 0.5 = normal */ + std::optional exposure_value; /* 0..100 range, 50.0 = normal */ std::optional focus_pos; } knobs; From 8cf5d8fa3aadcd231211e2ede2827315964be508 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 5 Apr 2025 14:59:17 +0300 Subject: [PATCH 045/106] gitignore: ignore my setup Signed-off-by: Vasiliy Doylov --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 51d314408..656f622f0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ *.pyc __pycache__/ venv/ +.vscode/ +.cache/ +compile_commands.json From 434c9d2a4b11b912ec7e5b0acc3494469d90063a Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 7 Jun 2025 02:22:30 +0300 Subject: [PATCH 046/106] CI: Add local forgejo CI Signed-off-by: Vasiliy Doylov --- .forgejo/workflows/build-alpine.yaml | 37 +++++++++ package/alpine/APKBUILD | 117 +++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 .forgejo/workflows/build-alpine.yaml create mode 100644 package/alpine/APKBUILD diff --git a/.forgejo/workflows/build-alpine.yaml b/.forgejo/workflows/build-alpine.yaml new file mode 100644 index 000000000..335b29ad6 --- /dev/null +++ b/.forgejo/workflows/build-alpine.yaml @@ -0,0 +1,37 @@ +name: PostmarketOS Build +run-name: PostmarketOS Build +on: + push: + workflow_dispatch: + +jobs: + build: + name: Build for ${{ matrix.info.arch }} + runs-on: Pmbootstrap + strategy: + matrix: + info: + - arch: x86_64 + - arch: aarch64 + steps: + - name: Remove libcamera aport + run: rm -rf ${{env.PMB_PMAPORTS}}/temp/libcamera + - name: Check out repository code + uses: actions/checkout@v4 + - name: Build packages + id: build + uses: actions/pmbootstrap-build@main + with: + name: libcamera + aports: ${{github.workspace}}/package/alpine + arch: ${{ matrix.info.arch }} + src: ${{github.workspace}} + - name: "Upload packages" + uses: actions/upload-alpine-package@main + with: + files: ${{steps.build.outputs.packages}} + secret: ${{secrets.PACKAGE_TOKEN}} + - name: Reset pmaports changes + if: always() + continue-on-error: true + run: git -C ${{env.PMB_PMAPORTS}} reset --hard diff --git a/package/alpine/APKBUILD b/package/alpine/APKBUILD new file mode 100644 index 000000000..93ede5b0f --- /dev/null +++ b/package/alpine/APKBUILD @@ -0,0 +1,117 @@ +pkgname=libcamera +pkgver=9999999 +pkgrel=0 +pkgdesc="Linux camera framework" +url="https://libcamera.org/" +arch="all" +license="LGPL-2.1-or-later AND GPL-2.0-or-later" +depends_dev=" + eudev-dev + glib-dev + gnutls-dev + gst-plugins-bad-dev + qt6-qtbase-dev + " + +makedepends="$depends_dev + coreutils + doxygen + graphviz + gtest-dev + libevent-dev + libpisp-dev + libunwind-dev + libyuv-dev + linux-headers + meson + py3-jinja2 + py3-ply + py3-sphinx + py3-yaml + qt6-qttools-dev + yaml-dev + " +subpackages=" + $pkgname-dbg + $pkgname-dev + $pkgname-doc + $pkgname-gstreamer + $pkgname-tools + " +source="" +builddir="$srcdir/$pkgname" +# gstreamer tests fail +# manual strip because ipa .sign files depend on the file contents- have to re-sign after strip +options="!strip !check" + +case "$CARCH" in +arm*|aarch64) + subpackages="$subpackages" + ;; +esac + +case "$CARCH" in +ppc64le|s390x|riscv64|loongarch64) + # doesn't install any ipa + ;; +*) + depends="$pkgname-ipa" + subpackages="$subpackages $pkgname-ipa" + ;; +esac + +build() { + abuild-meson \ + -Dtest=false \ + -Dv4l2=false \ + -Dwerror=false \ + -Dpipelines=simple \ + -Dipas=simple \ + . output + meson compile -C output +} + +check() { + meson test -C output --print-errorlogs +} + +package() { + DESTDIR="$pkgdir" meson install --no-rebuild -C output + + # manual strip first.. + scanelf --recursive \ + --nobanner \ + --etype "ET_DYN,ET_EXEC" \ + --format "%F" \ + "$pkgdir" \ + | while read -r file; do + strip "$file" + done +} + +ipa() { + depends="" + amove usr/lib/libcamera + # then sign ipa's + local ipa + for ipa in "$subpkgdir"/usr/lib/libcamera/ipa/ipa*.so; do + msg "signing $ipa" + "$builddir"/src/ipa/ipa-sign.sh \ + "$(find "$builddir"/output -type f -iname "*ipa-priv-key.pem")" \ + "$ipa" \ + "$ipa".sign + done +} + +gstreamer() { + depends="" + amove usr/lib/gstreamer-1.0 +} + +tools() { + depends="" + amove usr/bin/cam + amove usr/bin/lc-compliance +} + +sha512sums="" From a450911b98888abea6596e6595ae2e46b671506a Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 11 Jun 2025 22:43:24 +0300 Subject: [PATCH 047/106] CI: change pkgname Signed-off-by: Vasiliy Doylov --- .forgejo/workflows/build-alpine.yaml | 2 +- package/alpine/APKBUILD | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/build-alpine.yaml b/.forgejo/workflows/build-alpine.yaml index 335b29ad6..5f779d3a9 100644 --- a/.forgejo/workflows/build-alpine.yaml +++ b/.forgejo/workflows/build-alpine.yaml @@ -22,7 +22,7 @@ jobs: id: build uses: actions/pmbootstrap-build@main with: - name: libcamera + name: libcamera-neko-gpu aports: ${{github.workspace}}/package/alpine arch: ${{ matrix.info.arch }} src: ${{github.workspace}} diff --git a/package/alpine/APKBUILD b/package/alpine/APKBUILD index 93ede5b0f..1f31843a2 100644 --- a/package/alpine/APKBUILD +++ b/package/alpine/APKBUILD @@ -1,4 +1,5 @@ -pkgname=libcamera +basepkgname=libcamera +pkgname=$basepkgname-neko-gpu pkgver=9999999 pkgrel=0 pkgdesc="Linux camera framework" @@ -38,6 +39,14 @@ subpackages=" $pkgname-gstreamer $pkgname-tools " +provides=" + $basepkgname + $basepkgname-dbg + $basepkgname-dev + $basepkgname-doc + $basepkgname-gstreamer + $basepkgname-tools +" source="" builddir="$srcdir/$pkgname" # gstreamer tests fail From c50eb1f04a644cf82cbbc83b1446436b2220efac Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:19 +0100 Subject: [PATCH 048/106] libcamera: framebuffer: Add FrameMetadata::Status::FrameStartup Add a new status enum, FrameStartup, used to denote that even though the frame has been successfully captured, the IQ parameters set by the IPA will cause the frame to be unusable and applications are advised to not consume this frame. An example of this would be on a cold-start of the 3A algorithms, and there will be large oscillations to converge to a stable state quickly. Additional, update the definition of the FrameError state to cover the usage when the sensor is known to produce a number of invalid/error frames after stream-on. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- include/libcamera/framebuffer.h | 1 + src/libcamera/framebuffer.cpp | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/libcamera/framebuffer.h b/include/libcamera/framebuffer.h index ff8392430..e83825b46 100644 --- a/include/libcamera/framebuffer.h +++ b/include/libcamera/framebuffer.h @@ -26,6 +26,7 @@ struct FrameMetadata { FrameSuccess, FrameError, FrameCancelled, + FrameStartup, }; struct Plane { diff --git a/src/libcamera/framebuffer.cpp b/src/libcamera/framebuffer.cpp index 826848f75..219db50d6 100644 --- a/src/libcamera/framebuffer.cpp +++ b/src/libcamera/framebuffer.cpp @@ -43,12 +43,19 @@ LOG_DEFINE_CATEGORY(Buffer) * The frame has been captured with success and contains valid data. All fields * of the FrameMetadata structure are valid. * \var FrameMetadata::FrameError - * An error occurred during capture of the frame. The frame data may be partly - * or fully invalid. The sequence and timestamp fields of the FrameMetadata - * structure is valid, the other fields may be invalid. + * The frame data is partly or fully corrupted, missing or otherwise invalid. + * This can for instance indicate a hardware transmission error, or invalid data + * produced by the sensor during its startup phase. The sequence and timestamp + * fields of the FrameMetadata structure is valid, all the other fields may be + * invalid. * \var FrameMetadata::FrameCancelled * Capture stopped before the frame completed. The frame data is not valid. All * fields of the FrameMetadata structure but the status field are invalid. + * \var FrameMetadata::FrameStartup + * The frame has been successfully captured. However, the IPA is in a + * cold-start or reset phase and will result in image quality parameters + * producing unusable images. Applications are recommended to not consume these + * frames. All other fields of the FrameMetadata structure are valid. */ /** From b114c155a71660964c1947daacc5100fa047c9f3 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:20 +0100 Subject: [PATCH 049/106] ipa: rpi: Replace dropFrameCount in the IPA -> PH interface Replace the dropFrameCount parameter returned from ipa::start() to the pipeline handler by startupFrameCount and invalidFrameCount. The former counts the number of frames required for AWB/AGC to converge, and the latter counts the number of invalid frames produced by the sensor when starting up. In the pipeline handler, use the sum of these 2 values to replicate the existing dropFrameCount behaviour. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- include/libcamera/ipa/raspberrypi.mojom | 3 ++- src/ipa/rpi/common/ipa_base.cpp | 14 ++++++++------ .../pipeline/rpi/common/pipeline_base.cpp | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/include/libcamera/ipa/raspberrypi.mojom b/include/libcamera/ipa/raspberrypi.mojom index e30c70bde..12b083e9d 100644 --- a/include/libcamera/ipa/raspberrypi.mojom +++ b/include/libcamera/ipa/raspberrypi.mojom @@ -52,7 +52,8 @@ struct ConfigResult { struct StartResult { libcamera.ControlList controls; - int32 dropFrameCount; + int32 startupFrameCount; + int32 invalidFrameCount; }; struct PrepareParams { diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index e0f8b7e78..eee81e064 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -324,6 +324,7 @@ void IpaBase::start(const ControlList &controls, StartResult *result) * "mistrusted", which depends on whether this is a startup from cold, * or merely a mode switch in a running system. */ + unsigned int agcConvergenceFrames = 0, awbConvergenceFrames = 0; frameCount_ = 0; if (firstStart_) { dropFrameCount_ = helper_->hideFramesStartup(); @@ -336,7 +337,6 @@ void IpaBase::start(const ControlList &controls, StartResult *result) * (mistrustCount_) that they won't see. But if zero (i.e. * no convergence necessary), no frames need to be dropped. */ - unsigned int agcConvergenceFrames = 0; RPiController::AgcAlgorithm *agc = dynamic_cast( controller_.getAlgorithm("agc")); if (agc) { @@ -345,7 +345,6 @@ void IpaBase::start(const ControlList &controls, StartResult *result) agcConvergenceFrames += mistrustCount_; } - unsigned int awbConvergenceFrames = 0; RPiController::AwbAlgorithm *awb = dynamic_cast( controller_.getAlgorithm("awb")); if (awb) { @@ -353,15 +352,18 @@ void IpaBase::start(const ControlList &controls, StartResult *result) if (awbConvergenceFrames) awbConvergenceFrames += mistrustCount_; } - - dropFrameCount_ = std::max({ dropFrameCount_, agcConvergenceFrames, awbConvergenceFrames }); - LOG(IPARPI, Debug) << "Drop " << dropFrameCount_ << " frames on startup"; } else { dropFrameCount_ = helper_->hideFramesModeSwitch(); mistrustCount_ = helper_->mistrustFramesModeSwitch(); } - result->dropFrameCount = dropFrameCount_; + result->startupFrameCount = std::max({ agcConvergenceFrames, awbConvergenceFrames }); + result->invalidFrameCount = dropFrameCount_; + + dropFrameCount_ = std::max({ dropFrameCount_, agcConvergenceFrames, awbConvergenceFrames }); + + LOG(IPARPI, Debug) << "Startup frames: " << result->startupFrameCount + << " Invalid frames: " << result->invalidFrameCount; firstStart_ = false; lastRunTimestamp_ = 0; diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 1f13e5230..595648595 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -660,8 +660,8 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) data->setSensorControls(result.controls); /* Configure the number of dropped frames required on startup. */ - data->dropFrameCount_ = data->config_.disableStartupFrameDrops - ? 0 : result.dropFrameCount; + data->dropFrameCount_ = data->config_.disableStartupFrameDrops ? + 0 : result.startupFrameCount + result.invalidFrameCount; for (auto const stream : data->streams_) stream->resetBuffers(); From 6cf9c4d34fbda570f61c09fa44e6bc430600b84c Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:21 +0100 Subject: [PATCH 050/106] pipeline: ipa: rpi: Split RPiCameraData::dropFrameCount_ Split the pipeline handler drop frame tracking into startup frames and invalid frames, as reported by the IPA. Remove the drop buffer handling logic in the pipeline handler. Now all image buffers are returned out with the appropriate FrameStatus set for startup or invalid frames. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- .../pipeline/rpi/common/pipeline_base.cpp | 98 ++++++++----------- .../pipeline/rpi/common/pipeline_base.h | 5 +- 2 files changed, 42 insertions(+), 61 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 595648595..3f0b7abdc 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -659,9 +659,9 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) if (!result.controls.empty()) data->setSensorControls(result.controls); - /* Configure the number of dropped frames required on startup. */ - data->dropFrameCount_ = data->config_.disableStartupFrameDrops ? - 0 : result.startupFrameCount + result.invalidFrameCount; + /* Configure the number of startup and invalid frames reported by the IPA. */ + data->startupFrameCount_ = result.startupFrameCount; + data->invalidFrameCount_ = result.invalidFrameCount; for (auto const stream : data->streams_) stream->resetBuffers(); @@ -678,7 +678,6 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) data->buffersAllocated_ = true; } - /* We need to set the dropFrameCount_ before queueing buffers. */ ret = queueAllBuffers(camera); if (ret) { LOG(RPI, Error) << "Failed to queue buffers"; @@ -894,28 +893,12 @@ int PipelineHandlerBase::queueAllBuffers(Camera *camera) int ret; for (auto const stream : data->streams_) { - if (!(stream->getFlags() & StreamFlag::External)) { - ret = stream->queueAllBuffers(); - if (ret < 0) - return ret; - } else { - /* - * For external streams, we must queue up a set of internal - * buffers to handle the number of drop frames requested by - * the IPA. This is done by passing nullptr in queueBuffer(). - * - * The below queueBuffer() call will do nothing if there - * are not enough internal buffers allocated, but this will - * be handled by queuing the request for buffers in the - * RPiStream object. - */ - unsigned int i; - for (i = 0; i < data->dropFrameCount_; i++) { - ret = stream->queueBuffer(nullptr); - if (ret) - return ret; - } - } + if (stream->getFlags() & StreamFlag::External) + continue; + + ret = stream->queueAllBuffers(); + if (ret < 0) + return ret; } return 0; @@ -1412,7 +1395,15 @@ void CameraData::handleStreamBuffer(FrameBuffer *buffer, RPi::Stream *stream) * buffer back to the stream. */ Request *request = requestQueue_.empty() ? nullptr : requestQueue_.front(); - if (!dropFrameCount_ && request && request->findBuffer(stream) == buffer) { + if (request && request->findBuffer(stream) == buffer) { + FrameMetadata &md = buffer->_d()->metadata(); + + /* Mark the non-converged and invalid frames in the metadata. */ + if (invalidFrameCount_) + md.status = FrameMetadata::Status::FrameError; + else if (startupFrameCount_) + md.status = FrameMetadata::Status::FrameStartup; + /* * Tag the buffer as completed, returning it to the * application. @@ -1458,42 +1449,31 @@ void CameraData::handleState() void CameraData::checkRequestCompleted() { - bool requestCompleted = false; - /* - * If we are dropping this frame, do not touch the request, simply - * change the state to IDLE when ready. - */ - if (!dropFrameCount_) { - Request *request = requestQueue_.front(); - if (request->hasPendingBuffers()) - return; + Request *request = requestQueue_.front(); + if (request->hasPendingBuffers()) + return; - /* Must wait for metadata to be filled in before completing. */ - if (state_ != State::IpaComplete) - return; + /* Must wait for metadata to be filled in before completing. */ + if (state_ != State::IpaComplete) + return; - LOG(RPI, Debug) << "Completing request sequence: " - << request->sequence(); + LOG(RPI, Debug) << "Completing request sequence: " + << request->sequence(); - pipe()->completeRequest(request); - requestQueue_.pop(); - requestCompleted = true; - } + pipe()->completeRequest(request); + requestQueue_.pop(); - /* - * Make sure we have three outputs completed in the case of a dropped - * frame. - */ - if (state_ == State::IpaComplete && - ((ispOutputCount_ == ispOutputTotal_ && dropFrameCount_) || - requestCompleted)) { - LOG(RPI, Debug) << "Going into Idle state"; - state_ = State::Idle; - if (dropFrameCount_) { - dropFrameCount_--; - LOG(RPI, Debug) << "Dropping frame at the request of the IPA (" - << dropFrameCount_ << " left)"; - } + LOG(RPI, Debug) << "Going into Idle state"; + state_ = State::Idle; + + if (invalidFrameCount_) { + invalidFrameCount_--; + LOG(RPI, Debug) << "Decrementing invalid frames to " + << invalidFrameCount_; + } else if (startupFrameCount_) { + startupFrameCount_--; + LOG(RPI, Debug) << "Decrementing startup frames to " + << startupFrameCount_; } } diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index aae0c2f35..6023f9f9d 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -48,7 +48,7 @@ class CameraData : public Camera::Private public: CameraData(PipelineHandler *pipe) : Camera::Private(pipe), state_(State::Stopped), - dropFrameCount_(0), buffersAllocated_(false), + startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false), ispOutputCount_(0), ispOutputTotal_(0) { } @@ -151,7 +151,8 @@ public: /* Mapping of CropParams keyed by the output stream order in CameraConfiguration */ std::map cropParams_; - unsigned int dropFrameCount_; + unsigned int startupFrameCount_; + unsigned int invalidFrameCount_; /* * If set, this stores the value that represets a gain of one for From 98d144fef39877d79df8a948ced0b2ce4377a6be Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:22 +0100 Subject: [PATCH 051/106] pipeline: rpi: Remove disable_startup_frame_drops config option With the previous change to not drop frames in the pipeline handler, the "disable_startup_frame_drops" pipeline config option is not used. Remove it, and throw a warning if the option is present in the YAML config file. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/libcamera/pipeline/rpi/common/pipeline_base.cpp | 7 ++++--- src/libcamera/pipeline/rpi/common/pipeline_base.h | 5 ----- src/libcamera/pipeline/rpi/pisp/data/example.yaml | 5 ----- src/libcamera/pipeline/rpi/vc4/data/example.yaml | 5 ----- 4 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 3f0b7abdc..d8c7ca935 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -1078,7 +1078,6 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front int CameraData::loadPipelineConfiguration() { config_ = { - .disableStartupFrameDrops = false, .cameraTimeoutValue = 0, }; @@ -1115,8 +1114,10 @@ int CameraData::loadPipelineConfiguration() const YamlObject &phConfig = (*root)["pipeline_handler"]; - config_.disableStartupFrameDrops = - phConfig["disable_startup_frame_drops"].get(config_.disableStartupFrameDrops); + if (phConfig.contains("disable_startup_frame_drops")) + LOG(RPI, Warning) + << "The disable_startup_frame_drops key is now deprecated, " + << "startup frames are now identified by the FrameMetadata::Status::FrameStartup flag"; config_.cameraTimeoutValue = phConfig["camera_timeout_value_ms"].get(config_.cameraTimeoutValue); diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 6023f9f9d..e27c4f860 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -164,11 +164,6 @@ public: bool buffersAllocated_; struct Config { - /* - * Override any request from the IPA to drop a number of startup - * frames. - */ - bool disableStartupFrameDrops; /* * Override the camera timeout value calculated by the IPA based * on frame durations. diff --git a/src/libcamera/pipeline/rpi/pisp/data/example.yaml b/src/libcamera/pipeline/rpi/pisp/data/example.yaml index d67e654a8..baf03be79 100644 --- a/src/libcamera/pipeline/rpi/pisp/data/example.yaml +++ b/src/libcamera/pipeline/rpi/pisp/data/example.yaml @@ -16,11 +16,6 @@ # # "num_cfe_config_queue": 2, - # Override any request from the IPA to drop a number of startup - # frames. - # - # "disable_startup_frame_drops": false, - # Custom timeout value (in ms) for camera to use. This overrides # the value computed by the pipeline handler based on frame # durations. diff --git a/src/libcamera/pipeline/rpi/vc4/data/example.yaml b/src/libcamera/pipeline/rpi/vc4/data/example.yaml index b8e01adea..27e543488 100644 --- a/src/libcamera/pipeline/rpi/vc4/data/example.yaml +++ b/src/libcamera/pipeline/rpi/vc4/data/example.yaml @@ -29,11 +29,6 @@ # # "min_total_unicam_buffers": 4, - # Override any request from the IPA to drop a number of startup - # frames. - # - # "disable_startup_frame_drops": false, - # Custom timeout value (in ms) for camera to use. This overrides # the value computed by the pipeline handler based on frame # durations. From a402f9ebc1225b684300fcc664fd37e0ddf95f30 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:23 +0100 Subject: [PATCH 052/106] pipeline: rpi: Remove ispOutputCount_ and ispOutputTotal_ With the drop frame logic removed from the pipeline handler, these member variables and not used, so remove them. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/libcamera/pipeline/rpi/common/pipeline_base.h | 6 +----- src/libcamera/pipeline/rpi/pisp/pisp.cpp | 12 ------------ src/libcamera/pipeline/rpi/vc4/vc4.cpp | 15 --------------- 3 files changed, 1 insertion(+), 32 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index e27c4f860..898f31577 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -48,8 +48,7 @@ class CameraData : public Camera::Private public: CameraData(PipelineHandler *pipe) : Camera::Private(pipe), state_(State::Stopped), - startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false), - ispOutputCount_(0), ispOutputTotal_(0) + startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false) { } @@ -179,9 +178,6 @@ protected: virtual void tryRunPipeline() = 0; - unsigned int ispOutputCount_; - unsigned int ispOutputTotal_; - private: void checkRequestCompleted(); }; diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index 91e7f4c94..ccf135c3d 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -1834,12 +1834,6 @@ void PiSPCameraData::beOutputDequeue(FrameBuffer *buffer) dmabufSyncEnd(buffer->planes()[0].fd); handleStreamBuffer(buffer, stream); - - /* - * Increment the number of ISP outputs generated. - * This is needed to track dropped frames. - */ - ispOutputCount_++; handleState(); } @@ -1885,7 +1879,6 @@ void PiSPCameraData::prepareIspComplete(const ipa::RPi::BufferIds &buffers, bool * If there is no need to run the Backend, just signal that the * input buffer is completed and all Backend outputs are ready. */ - ispOutputCount_ = ispOutputTotal_; buffer = cfe_[Cfe::Output0].getBuffers().at(bayerId).buffer; handleStreamBuffer(buffer, &cfe_[Cfe::Output0]); } else @@ -1994,7 +1987,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) global.bayer_enables |= PISP_BE_BAYER_ENABLE_INPUT; global.bayer_order = toPiSPBayerOrder(cfeFormat.fourcc); - ispOutputTotal_ = 1; /* Config buffer */ if (PISP_IMAGE_FORMAT_COMPRESSED(inputFormat.format)) { pisp_decompress_config decompress; decompress.offset = DefaultCompressionOffset; @@ -2025,7 +2017,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) setupOutputClipping(ispFormat0, outputFormat0); be_->SetOutputFormat(0, outputFormat0); - ispOutputTotal_++; } if (global.rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT1) { @@ -2049,7 +2040,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) setupOutputClipping(ispFormat1, outputFormat1); be_->SetOutputFormat(1, outputFormat1); - ispOutputTotal_++; } /* Setup the TDN I/O blocks in case TDN gets turned on later. */ @@ -2256,8 +2246,6 @@ void PiSPCameraData::prepareCfe() void PiSPCameraData::prepareBe(uint32_t bufferId, bool stitchSwapBuffers) { - ispOutputCount_ = 0; - FrameBuffer *buffer = cfe_[Cfe::Output0].getBuffers().at(bufferId).buffer; LOG(RPI, Debug) << "Input re-queue to ISP, buffer id " << bufferId diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index fe910bdf2..ac6dab814 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -597,8 +597,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi stream->setFlags(StreamFlag::External); } - ispOutputTotal_ = outStreams.size(); - /* * If ISP::Output0 stream has not been configured by the application, * we must allow the hardware to generate an output so that the data @@ -625,8 +623,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi return -EINVAL; } - ispOutputTotal_++; - LOG(RPI, Debug) << "Defaulting ISP Output0 format to " << format; } @@ -662,8 +658,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi << ret; return -EINVAL; } - - ispOutputTotal_++; } /* ISP statistics output format. */ @@ -676,8 +670,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi return ret; } - ispOutputTotal_++; - /* * Configure the Unicam embedded data output format only if the sensor * supports it. @@ -843,12 +835,6 @@ void Vc4CameraData::ispOutputDequeue(FrameBuffer *buffer) handleStreamBuffer(buffer, stream); } - /* - * Increment the number of ISP outputs generated. - * This is needed to track dropped frames. - */ - ispOutputCount_++; - handleState(); } @@ -880,7 +866,6 @@ void Vc4CameraData::prepareIspComplete(const ipa::RPi::BufferIds &buffers, << ", timestamp: " << buffer->metadata().timestamp; isp_[Isp::Input].queueBuffer(buffer); - ispOutputCount_ = 0; if (sensorMetadata_ && embeddedId) { buffer = unicam_[Unicam::Embedded].getBuffers().at(embeddedId & RPi::MaskID).buffer; From 8d2cd0b5b813bf5d26918df2218f2c0493992de6 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 6 Jun 2025 11:55:24 +0100 Subject: [PATCH 053/106] ipa: rpi: Rename dropFrameCount_ to invalidCount_ Rename dropFrameCount_ to invalidCount_ to better reflect its use as frames are no longer dropped by the pipeline handler. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Jacopo Mondi Signed-off-by: Kieran Bingham --- src/ipa/rpi/common/ipa_base.cpp | 10 +++++----- src/ipa/rpi/common/ipa_base.h | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index eee81e064..50e25f057 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -327,7 +327,7 @@ void IpaBase::start(const ControlList &controls, StartResult *result) unsigned int agcConvergenceFrames = 0, awbConvergenceFrames = 0; frameCount_ = 0; if (firstStart_) { - dropFrameCount_ = helper_->hideFramesStartup(); + invalidCount_ = helper_->hideFramesStartup(); mistrustCount_ = helper_->mistrustFramesStartup(); /* @@ -353,14 +353,14 @@ void IpaBase::start(const ControlList &controls, StartResult *result) awbConvergenceFrames += mistrustCount_; } } else { - dropFrameCount_ = helper_->hideFramesModeSwitch(); + invalidCount_ = helper_->hideFramesModeSwitch(); mistrustCount_ = helper_->mistrustFramesModeSwitch(); } result->startupFrameCount = std::max({ agcConvergenceFrames, awbConvergenceFrames }); - result->invalidFrameCount = dropFrameCount_; + result->invalidFrameCount = invalidCount_; - dropFrameCount_ = std::max({ dropFrameCount_, agcConvergenceFrames, awbConvergenceFrames }); + invalidCount_ = std::max({ invalidCount_, agcConvergenceFrames, awbConvergenceFrames }); LOG(IPARPI, Debug) << "Startup frames: " << result->startupFrameCount << " Invalid frames: " << result->invalidFrameCount; @@ -443,7 +443,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) /* Allow a 10% margin on the comparison below. */ Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns; - if (lastRunTimestamp_ && frameCount_ > dropFrameCount_ && + if (lastRunTimestamp_ && frameCount_ > invalidCount_ && delta < controllerMinFrameDuration * 0.9 && !hdrChange) { /* * Ensure we merge the previous frame's metadata with the current diff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h index 1a811beb3..e818104ba 100644 --- a/src/ipa/rpi/common/ipa_base.h +++ b/src/ipa/rpi/common/ipa_base.h @@ -115,8 +115,8 @@ private: /* How many frames we should avoid running control algos on. */ unsigned int mistrustCount_; - /* Number of frames that need to be dropped on startup. */ - unsigned int dropFrameCount_; + /* Number of frames that need to be marked as dropped on startup. */ + unsigned int invalidCount_; /* Frame timestamp for the last run of the controller. */ uint64_t lastRunTimestamp_; From b544ce1c19c96917b207c0ba2efbe5f186b48212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Fri, 13 Jun 2025 16:31:53 +0200 Subject: [PATCH 054/106] apps: common: image: Fix assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `plane` must be strictly less than the vector's size, it cannot be equal to it. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart --- src/apps/common/image.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/common/image.cpp b/src/apps/common/image.cpp index a2a0f58f3..9a67238aa 100644 --- a/src/apps/common/image.cpp +++ b/src/apps/common/image.cpp @@ -98,12 +98,12 @@ unsigned int Image::numPlanes() const Span Image::data(unsigned int plane) { - assert(plane <= planes_.size()); + assert(plane < planes_.size()); return planes_[plane]; } Span Image::data(unsigned int plane) const { - assert(plane <= planes_.size()); + assert(plane < planes_.size()); return planes_[plane]; } From 5c8de8a08e0af64c7666b00d022e9f1fe352bee5 Mon Sep 17 00:00:00 2001 From: Antoine Bouyer Date: Wed, 4 Jun 2025 15:14:02 +0200 Subject: [PATCH 055/106] pipeline: imx8-isi: Cosmetic changes Change indentation to pass checkstyle script. Fixes: 680cde600509 ("libcamera: imx8-isi: Split Bayer/YUV config generation") Signed-off-by: Antoine Bouyer Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/libcamera/pipeline/imx8-isi/imx8-isi.cpp | 42 ++++++++++---------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp index ecda426a6..c393319de 100644 --- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp +++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp @@ -761,30 +761,28 @@ PipelineHandlerISI::generateConfiguration(Camera *camera, */ StreamConfiguration cfg; - switch (role) { - case StreamRole::StillCapture: - case StreamRole::Viewfinder: - case StreamRole::VideoRecording: { - Size size = role == StreamRole::StillCapture - ? data->sensor_->resolution() - : PipelineHandlerISI::kPreviewSize; - cfg = generateYUVConfiguration(camera, size); - if (cfg.pixelFormat.isValid()) - break; + switch (role) { + case StreamRole::StillCapture: + case StreamRole::Viewfinder: + case StreamRole::VideoRecording: { + Size size = role == StreamRole::StillCapture + ? data->sensor_->resolution() + : PipelineHandlerISI::kPreviewSize; + cfg = generateYUVConfiguration(camera, size); + if (cfg.pixelFormat.isValid()) + break; + /* + * Fallback to use a Bayer format if that's what the + * sensor supports. + */ + [[fallthrough]]; + } - /* - * Fallback to use a Bayer format if that's what the - * sensor supports. - */ - [[fallthrough]]; - - } - - case StreamRole::Raw: { - cfg = generateRawConfiguration(camera); - break; - } + case StreamRole::Raw: { + cfg = generateRawConfiguration(camera); + break; + } default: LOG(ISI, Error) << "Requested stream role not supported: " << role; From 5621ac27a2c639c8e84f8c838486c8eaf72f3d74 Mon Sep 17 00:00:00 2001 From: Antoine Bouyer Date: Wed, 4 Jun 2025 15:14:03 +0200 Subject: [PATCH 056/106] pipeline: imx8-isi: Fix match returned value in error case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The match() function returns a boolean type, while it could return int in case of error when opening the capture file. Fixes: 0ec982d21086 ("libcamera: pipeline: Add IMX8 ISI pipeline") Signed-off-by: Antoine Bouyer Reviewed-by: Barnabás Pőcze Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/libcamera/pipeline/imx8-isi/imx8-isi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp index c393319de..b81e436b2 100644 --- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp +++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp @@ -1003,7 +1003,7 @@ bool PipelineHandlerISI::match(DeviceEnumerator *enumerator) ret = capture->open(); if (ret) - return ret; + return false; pipes_.push_back({ std::move(isi), std::move(capture) }); } From d3f3b95b64d2239adca7e0c6e9eb35db690ca98d Mon Sep 17 00:00:00 2001 From: Antoine Bouyer Date: Fri, 6 Jun 2025 09:34:59 +0200 Subject: [PATCH 057/106] pipeline: imx8-isi: Dynamically compute crossbar subdevice's first source. So far, imx8-isi pipeline supports _symetrical_ crossbar, with same amount of sink and source pads. But for some other imx SoCs, such as i.MX8QM or i.MX95, crossbar is not symetric anymore. Since each crossbar source is already captured as a pipes_ vector entry, we use pipes_ vector's size to compute 1st source index. "1st source index" = "total number of crossbar pads" - pipes_.count() Signed-off-by: Antoine Bouyer Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/libcamera/pipeline/imx8-isi/imx8-isi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp index b81e436b2..f4014b95d 100644 --- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp +++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp @@ -820,7 +820,7 @@ int PipelineHandlerISI::configure(Camera *camera, CameraConfiguration *c) * routing table instead of resetting it. */ V4L2Subdevice::Routing routing = {}; - unsigned int xbarFirstSource = crossbar_->entity()->pads().size() / 2 + 1; + unsigned int xbarFirstSource = crossbar_->entity()->pads().size() - pipes_.size(); for (const auto &[idx, config] : utils::enumerate(*c)) { uint32_t sourcePad = xbarFirstSource + idx; From 04e7823eb24bbc665f98fcb4dc5c3d494b39a356 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 22 May 2025 11:47:29 +0200 Subject: [PATCH 058/106] gstreamer: Document improvements when updating minimum GStreamer version A const_cast<> was recently added to fix a compilation issue with older GStreamer versions. Add a comment to indicate it can be removed when bumping the minimum GStreamer version requirement. While at it, also document a possible future improvement in the same function, and wrap long lines. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index b34f08977..6ede43b06 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -285,10 +285,19 @@ gst_libcamera_extrapolate_info(GstVideoInfo *info, guint32 stride) } static GstFlowReturn -gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, const GstVideoInfo *dest_info, guint32 stride) +gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, + const GstVideoInfo *dest_info, guint32 stride) { - GstVideoInfo src_info = *dest_info; + /* + * When dropping support for versions earlier than v1.22.0, use + * + * g_auto (GstVideoFrame) src_frame = GST_VIDEO_FRAME_INIT; + * g_auto (GstVideoFrame) dest_frame = GST_VIDEO_FRAME_INIT; + * + * and drop the gst_video_frame_unmap() calls. + */ GstVideoFrame src_frame, dest_frame; + GstVideoInfo src_info = *dest_info; gst_libcamera_extrapolate_info(&src_info, stride); src_info.size = gst_buffer_get_size(src); @@ -298,7 +307,12 @@ gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, const GstVideoIn return GST_FLOW_ERROR; } - if (!gst_video_frame_map(&dest_frame, const_cast(dest_info), dest, GST_MAP_WRITE)) { + /* + * When dropping support for versions earlier than 1.20.0, drop the + * const_cast<>(). + */ + if (!gst_video_frame_map(&dest_frame, const_cast(dest_info), + dest, GST_MAP_WRITE)) { GST_ERROR("Could not map dest buffer"); gst_video_frame_unmap(&src_frame); return GST_FLOW_ERROR; From 3b6820778918a9971c6b9b39385d408682d6193e Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 22 May 2025 11:47:29 +0200 Subject: [PATCH 059/106] gstreamer: Factor out video pool creation The gst_libcamera_src_negotiate() function uses 5 indentation levels, causing long lines. Move video pool creation to a separate function to increase readability. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 113 +++++++++++++++++++----------- 1 file changed, 71 insertions(+), 42 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 6ede43b06..34a08f8ea 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -29,6 +29,7 @@ #include #include +#include #include #include @@ -521,6 +522,71 @@ gst_libcamera_src_open(GstLibcameraSrc *self) return true; } +/** + * \brief Create a video pool for a pad + * \param[in] self The libcamerasrc instance + * \param[in] srcpad The pad + * \param[in] caps The pad caps + * \param[in] info The video info for the pad + * + * This function creates and returns a video buffer pool for the given pad if + * needed to accommodate stride mismatch. If the peer element supports stride + * negotiation through the meta API, no pool is needed and the function will + * return a null pool. + * + * \return A tuple containing the video buffers pool pointer and an error code + */ +static std::tuple +gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, + GstCaps *caps, const GstVideoInfo *info) +{ + GstQuery *query = NULL; + const gboolean need_pool = true; + gboolean has_video_meta = false; + GstBufferPool *video_pool = NULL; + + query = gst_query_new_allocation(caps, need_pool); + if (!gst_pad_peer_query(srcpad, query)) + GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); + else + has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + + if (!has_video_meta) { + GstBufferPool *pool = NULL; + + if (gst_query_get_n_allocation_pools(query) > 0) + gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); + + if (pool) + video_pool = pool; + else { + GstStructure *config; + guint min_buffers = 3; + video_pool = gst_video_buffer_pool_new(); + + config = gst_buffer_pool_get_config(video_pool); + gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); + + GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); + + gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); + } + + GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); + + if (!gst_buffer_pool_set_active(video_pool, true)) { + gst_caps_unref(caps); + GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, + ("Failed to active buffer pool"), + ("gst_libcamera_src_negotiate() failed.")); + return { NULL, -EINVAL }; + } + } + + gst_query_unref(query); + return { video_pool, 0 }; +} + /* Must be called with stream_lock held. */ static bool gst_libcamera_src_negotiate(GstLibcameraSrc *self) @@ -603,50 +669,13 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self) /* Stride mismatch between camera stride and that calculated by video-info. */ if (static_cast(info.stride[0]) != stream_cfg.stride && GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) { - GstQuery *query = NULL; - const gboolean need_pool = true; - gboolean has_video_meta = false; - gst_libcamera_extrapolate_info(&info, stream_cfg.stride); - query = gst_query_new_allocation(caps, need_pool); - if (!gst_pad_peer_query(srcpad, query)) - GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); - else - has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); - - if (!has_video_meta) { - GstBufferPool *pool = NULL; - - if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); - - if (pool) - video_pool = pool; - else { - GstStructure *config; - guint min_buffers = 3; - video_pool = gst_video_buffer_pool_new(); - - config = gst_buffer_pool_get_config(video_pool); - gst_buffer_pool_config_set_params(config, caps, info.size, min_buffers, 0); - - GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); - - gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); - } - - GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); - - if (!gst_buffer_pool_set_active(video_pool, true)) { - gst_caps_unref(caps); - GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, - ("Failed to active buffer pool"), - ("gst_libcamera_src_negotiate() failed.")); - return false; - } - } - gst_query_unref(query); + std::tie(video_pool, ret) = + gst_libcamera_create_video_pool(self, srcpad, + caps, &info); + if (ret) + return false; } GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator, From 613202b80963c8290964f93c1e5090af8c2957f2 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 22 May 2025 11:47:29 +0200 Subject: [PATCH 060/106] gstreamer: Reduce indentation in gst_libcamera_create_video_pool() Now that video pool creation is handled by a dedicated function, the logic can be simplified by returning early instead of nesting scopes. Do so to decrease indentation and improve readability, and document the implementation of the function with comments. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 57 ++++++++++++++++--------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 34a08f8ea..1294c9a09 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -542,45 +542,48 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, { GstQuery *query = NULL; const gboolean need_pool = true; - gboolean has_video_meta = false; GstBufferPool *video_pool = NULL; + /* + * Get the peer allocation hints to check if it supports the meta API. + * If so, the stride will be negotiated, and there's no need to create a + * video pool. + */ query = gst_query_new_allocation(caps, need_pool); + if (!gst_pad_peer_query(srcpad, query)) GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); - else - has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + else if (gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL)) + return { NULL, 0 }; - if (!has_video_meta) { - GstBufferPool *pool = NULL; + GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); - if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); + /* + * If the allocation query has pools, use the first one. Otherwise, + * create a new pool. + */ + if (gst_query_get_n_allocation_pools(query) > 0) + gst_query_parse_nth_allocation_pool(query, 0, &video_pool, NULL, NULL, NULL); - if (pool) - video_pool = pool; - else { - GstStructure *config; - guint min_buffers = 3; - video_pool = gst_video_buffer_pool_new(); + if (!video_pool) { + GstStructure *config; + guint min_buffers = 3; - config = gst_buffer_pool_get_config(video_pool); - gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); + video_pool = gst_video_buffer_pool_new(); + config = gst_buffer_pool_get_config(video_pool); + gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); - GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); + GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); - gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); - } + gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); + } - GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); - - if (!gst_buffer_pool_set_active(video_pool, true)) { - gst_caps_unref(caps); - GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, - ("Failed to active buffer pool"), - ("gst_libcamera_src_negotiate() failed.")); - return { NULL, -EINVAL }; - } + if (!gst_buffer_pool_set_active(video_pool, true)) { + gst_caps_unref(caps); + GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, + ("Failed to active buffer pool"), + ("gst_libcamera_src_negotiate() failed.")); + return { NULL, -EINVAL }; } gst_query_unref(query); From f7c4fcd30191bc87fa00f7886df32bf285898195 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 4 Jun 2025 04:08:20 +0300 Subject: [PATCH 061/106] gstreamer: Rename variable in gst_libcamera_create_video_pool() Now that the code is isolated in a function, the video_pool variable in gst_libcamera_create_video_pool() can be renamed to pool without clashing with another local variable. Do so to reduce line length. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 1294c9a09..9cc8d9e3b 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -542,7 +542,7 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, { GstQuery *query = NULL; const gboolean need_pool = true; - GstBufferPool *video_pool = NULL; + GstBufferPool *pool = NULL; /* * Get the peer allocation hints to check if it supports the meta API. @@ -563,22 +563,22 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, * create a new pool. */ if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &video_pool, NULL, NULL, NULL); + gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); - if (!video_pool) { + if (!pool) { GstStructure *config; guint min_buffers = 3; - video_pool = gst_video_buffer_pool_new(); - config = gst_buffer_pool_get_config(video_pool); + pool = gst_video_buffer_pool_new(); + config = gst_buffer_pool_get_config(pool); gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); - gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); + gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(pool), config); } - if (!gst_buffer_pool_set_active(video_pool, true)) { + if (!gst_buffer_pool_set_active(pool, true)) { gst_caps_unref(caps); GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, ("Failed to active buffer pool"), @@ -587,7 +587,7 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, } gst_query_unref(query); - return { video_pool, 0 }; + return { pool, 0 }; } /* Must be called with stream_lock held. */ From 772b06bd8c7e989c7fc041e7ad982cbec87e0635 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Thu, 22 May 2025 11:47:29 +0200 Subject: [PATCH 062/106] gstreamer: Fix leak of GstQuery and GstBufferPool in error path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gst_libcamera_create_video_pool() function leaks a GstQuery instance and a GstBufferPool instance in an error path. Fix the leaks with g_autoptr(). Signed-off-by: Laurent Pinchart Reviewed-by: Barnabás Pőcze --- src/gstreamer/gstlibcamerasrc.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 9cc8d9e3b..05b47ece5 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -540,9 +541,9 @@ static std::tuple gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, GstCaps *caps, const GstVideoInfo *info) { - GstQuery *query = NULL; + g_autoptr(GstQuery) query = NULL; + g_autoptr(GstBufferPool) pool = NULL; const gboolean need_pool = true; - GstBufferPool *pool = NULL; /* * Get the peer allocation hints to check if it supports the meta API. @@ -586,8 +587,7 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, return { NULL, -EINVAL }; } - gst_query_unref(query); - return { pool, 0 }; + return { std::exchange(pool, nullptr), 0 }; } /* Must be called with stream_lock held. */ From a8f90517e0f752148b2b26dfc2771df32962ec45 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 4 Jun 2025 02:36:01 +0300 Subject: [PATCH 063/106] gstreamer: Drop incorrect unref on caps The caps object passeed to the gst_libcamera_create_video_pool() function is managed as a g_autoptr() in the caller. The function doesn't acquire any new reference, so it shouldn't call gst_caps_unref(). Fix it. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamerasrc.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 05b47ece5..d86b423a9 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -580,7 +580,6 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, } if (!gst_buffer_pool_set_active(pool, true)) { - gst_caps_unref(caps); GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, ("Failed to active buffer pool"), ("gst_libcamera_src_negotiate() failed.")); From b3ff75d7589a263412ad63008b3c8518d40e6316 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 4 Jun 2025 04:17:10 +0300 Subject: [PATCH 064/106] gstreamer: Replace NULL with nullptr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Usage of NULL has slowly crept in the libcamerasrc sources. Replace it with nullptr. Reported-by: Nicolas Dufresne Signed-off-by: Laurent Pinchart Reviewed-by: Barnabás Pőcze Reviewed-by: Nicolas Dufresne --- src/gstreamer/gstlibcamera-controls.cpp.in | 2 +- src/gstreamer/gstlibcamerapad.cpp | 2 +- src/gstreamer/gstlibcamerasrc.cpp | 23 +++++++++++----------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/gstreamer/gstlibcamera-controls.cpp.in b/src/gstreamer/gstlibcamera-controls.cpp.in index 89c530da0..2a16b39a9 100644 --- a/src/gstreamer/gstlibcamera-controls.cpp.in +++ b/src/gstreamer/gstlibcamera-controls.cpp.in @@ -68,7 +68,7 @@ static const GEnumValue {{ ctrl.name|snake_case }}_types[] = { "{{ enum.gst_name }}" }, {%- endfor %} - {0, NULL, NULL} + {0, nullptr, nullptr} }; #define TYPE_{{ ctrl.name|snake_case|upper }} \ diff --git a/src/gstreamer/gstlibcamerapad.cpp b/src/gstreamer/gstlibcamerapad.cpp index 3bc2bc87e..81a0ef44c 100644 --- a/src/gstreamer/gstlibcamerapad.cpp +++ b/src/gstreamer/gstlibcamerapad.cpp @@ -102,7 +102,7 @@ gst_libcamera_stream_role_get_type() "libcamera::Viewfinder", "view-finder", }, - { 0, NULL, NULL } + { 0, nullptr, nullptr } }; if (!type) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index d86b423a9..ebea34e47 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -368,10 +368,10 @@ int GstLibcameraSrcState::processRequest() if (video_pool) { /* Only set video pool when a copy is needed. */ - GstBuffer *copy = NULL; + GstBuffer *copy = nullptr; const GstVideoInfo info = gst_libcamera_pad_get_video_info(srcpad); - ret = gst_buffer_pool_acquire_buffer(video_pool, ©, NULL); + ret = gst_buffer_pool_acquire_buffer(video_pool, ©, nullptr); if (ret != GST_FLOW_OK) { gst_buffer_unref(buffer); GST_ELEMENT_ERROR(src_, RESOURCE, SETTINGS, @@ -541,8 +541,8 @@ static std::tuple gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, GstCaps *caps, const GstVideoInfo *info) { - g_autoptr(GstQuery) query = NULL; - g_autoptr(GstBufferPool) pool = NULL; + g_autoptr(GstQuery) query = nullptr; + g_autoptr(GstBufferPool) pool = nullptr; const gboolean need_pool = true; /* @@ -554,8 +554,8 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, if (!gst_pad_peer_query(srcpad, query)) GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); - else if (gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL)) - return { NULL, 0 }; + else if (gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, nullptr)) + return { nullptr, 0 }; GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); @@ -564,7 +564,8 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, * create a new pool. */ if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); + gst_query_parse_nth_allocation_pool(query, 0, &pool, nullptr, + nullptr, nullptr); if (!pool) { GstStructure *config; @@ -583,7 +584,7 @@ gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, ("Failed to active buffer pool"), ("gst_libcamera_src_negotiate() failed.")); - return { NULL, -EINVAL }; + return { nullptr, -EINVAL }; } return { std::exchange(pool, nullptr), 0 }; @@ -660,7 +661,7 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self) for (gsize i = 0; i < state->srcpads_.size(); i++) { GstPad *srcpad = state->srcpads_[i]; const StreamConfiguration &stream_cfg = state->config_->at(i); - GstBufferPool *video_pool = NULL; + GstBufferPool *video_pool = nullptr; GstVideoInfo info; g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg, transfer[i]); @@ -1065,7 +1066,7 @@ gst_libcamera_src_request_new_pad(GstElement *element, GstPadTemplate *templ, const gchar *name, [[maybe_unused]] const GstCaps *caps) { GstLibcameraSrc *self = GST_LIBCAMERA_SRC(element); - g_autoptr(GstPad) pad = NULL; + g_autoptr(GstPad) pad = nullptr; GST_DEBUG_OBJECT(self, "new request pad created"); @@ -1079,7 +1080,7 @@ gst_libcamera_src_request_new_pad(GstElement *element, GstPadTemplate *templ, GST_ELEMENT_ERROR(element, STREAM, FAILED, ("Internal data stream error."), ("Could not add pad to element")); - return NULL; + return nullptr; } gst_child_proxy_child_added(GST_CHILD_PROXY(self), G_OBJECT(pad), GST_OBJECT_NAME(pad)); From b4c92a61bf6bafd3f9fd55f377c274a257814b13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Wed, 4 Jun 2025 17:00:46 +0200 Subject: [PATCH 065/106] ipa: rpi: Initialize enum controls with a list of values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is how uvcvideo and rkisp1 do it. See ee918b370a08b ("ipa: rkisp1: agc: Initialize enum controls with a list of values") for the motivation. In summary, having a list of values is used as a sign that the control is an enum in multiple places (e.g. `cam`, `camshark`). Signed-off-by: Barnabás Pőcze Reviewed-by: Naushir Patuck Reviewed-by: Laurent Pinchart --- src/ipa/rpi/common/ipa_base.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index 50e25f057..6565f5366 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -58,23 +58,24 @@ const ControlInfoMap::Map ipaControls{ /* \todo Move this to the Camera class */ { &controls::AeEnable, ControlInfo(false, true, true) }, { &controls::ExposureTimeMode, - ControlInfo(static_cast(controls::ExposureTimeModeAuto), - static_cast(controls::ExposureTimeModeManual), - static_cast(controls::ExposureTimeModeAuto)) }, + ControlInfo({ { ControlValue(controls::ExposureTimeModeAuto), + ControlValue(controls::ExposureTimeModeManual) } }, + ControlValue(controls::ExposureTimeModeAuto)) }, { &controls::ExposureTime, ControlInfo(1, 66666, static_cast(defaultExposureTime.get())) }, { &controls::AnalogueGainMode, - ControlInfo(static_cast(controls::AnalogueGainModeAuto), - static_cast(controls::AnalogueGainModeManual), - static_cast(controls::AnalogueGainModeAuto)) }, + ControlInfo({ { ControlValue(controls::AnalogueGainModeAuto), + ControlValue(controls::AnalogueGainModeManual) } }, + ControlValue(controls::AnalogueGainModeAuto)) }, { &controls::AnalogueGain, ControlInfo(1.0f, 16.0f, 1.0f) }, { &controls::AeMeteringMode, ControlInfo(controls::AeMeteringModeValues) }, { &controls::AeConstraintMode, ControlInfo(controls::AeConstraintModeValues) }, { &controls::AeExposureMode, ControlInfo(controls::AeExposureModeValues) }, { &controls::ExposureValue, ControlInfo(-8.0f, 8.0f, 0.0f) }, - { &controls::AeFlickerMode, ControlInfo(static_cast(controls::FlickerOff), - static_cast(controls::FlickerManual), - static_cast(controls::FlickerOff)) }, + { &controls::AeFlickerMode, + ControlInfo({ { ControlValue(controls::FlickerOff), + ControlValue(controls::FlickerManual) } }, + ControlValue(controls::FlickerOff)) }, { &controls::AeFlickerPeriod, ControlInfo(100, 1000000) }, { &controls::Brightness, ControlInfo(-1.0f, 1.0f, 0.0f) }, { &controls::Contrast, ControlInfo(0.0f, 32.0f, 1.0f) }, From 4a277906a41e73b4e35e1c528f906e3c70938b16 Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Thu, 5 Jun 2025 15:37:41 +0900 Subject: [PATCH 066/106] gstreamer: Fix libcamerasrc responding latency before setting caps Whenever a downstream element queries latency, libcamerasrc will always reply, even though it has not yet determined the latency. However some downstream elements (e.g. glvideomixer/aggregator) will query the latency before libcamerasrc sets the caps. When these elements get the latency, they will start the caps negotiation. Since libcamerasrc has not yet determined caps, invalid negotiation is performed and workflow is disrupted. So, set latency to 'GST_CLOCK_TIME_NONE' during initialization, and reply to the query after libcamerasrc confirms the latency. At this time, libcamerasrc has also completed caps negotiation and downstream elements work fine. In addition, every time the src pad task stops, we reset the latency to GST_CLOCK_TIME_NONE to ensure that when next time task starts, the downstream elements can generate out buffers after receiving the effective latency. Signed-off-by: Hou Qi Reviewed-by: Nicolas Dufresne Signed-off-by: Kieran Bingham --- src/gstreamer/gstlibcamerapad.cpp | 5 +++++ src/gstreamer/gstlibcamerasrc.cpp | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/gstreamer/gstlibcamerapad.cpp b/src/gstreamer/gstlibcamerapad.cpp index 81a0ef44c..22b967198 100644 --- a/src/gstreamer/gstlibcamerapad.cpp +++ b/src/gstreamer/gstlibcamerapad.cpp @@ -72,6 +72,10 @@ gst_libcamera_pad_query(GstPad *pad, GstObject *parent, GstQuery *query) if (query->type != GST_QUERY_LATENCY) return gst_pad_query_default(pad, parent, query); + GLibLocker lock(GST_OBJECT(self)); + if (self->latency == GST_CLOCK_TIME_NONE) + return FALSE; + /* TRUE here means live, we assumes that max latency is the same as min * as we have no idea that duration of frames. */ gst_query_set_latency(query, TRUE, self->latency, self->latency); @@ -81,6 +85,7 @@ gst_libcamera_pad_query(GstPad *pad, GstObject *parent, GstQuery *query) static void gst_libcamera_pad_init(GstLibcameraPad *self) { + self->latency = GST_CLOCK_TIME_NONE; GST_PAD_QUERYFUNC(self) = gst_libcamera_pad_query; } diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index ebea34e47..51ba8b67a 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -881,8 +881,10 @@ gst_libcamera_src_task_leave([[maybe_unused]] GstTask *task, { GLibRecLocker locker(&self->stream_lock); - for (GstPad *srcpad : state->srcpads_) + for (GstPad *srcpad : state->srcpads_) { + gst_libcamera_pad_set_latency(srcpad, GST_CLOCK_TIME_NONE); gst_libcamera_pad_set_pool(srcpad, nullptr); + } } g_clear_object(&self->allocator); From 6a09deaf7d4a402051ee86ddc83d5af18e92eec9 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 19 Jun 2025 11:05:53 +0100 Subject: [PATCH 067/106] controls: Add FrameWallClock control Add a FrameWallClock control that reports the same moment as the frame's SensorTimestamp, but in wallclock units. Signed-off-by: David Plowman Reviewed-by: Kieran Bingham Reviewed-by: Naushir Patuck Reviewed-by: Laurent Pinchart Signed-off-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_core.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index aa7448645..028919ef3 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -1,6 +1,4 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2019, Google Inc. + # %YAML 1.1 --- @@ -1268,4 +1266,19 @@ controls: description: | Enable or disable the debug metadata. + - FrameWallClock: + type: int64_t + direction: out + description: | + This timestamp corresponds to the same moment in time as the + SensorTimestamp, but is represented as a wall clock time as measured by + the CLOCK_REALTIME clock. + + Being a wall clock measurement, it can be used to synchronise timing + across different devices. + + \sa SensorTimestamp + + The FrameWallClock control can only be returned in metadata. + ... From 2a4e347dfea352901cb2d609489f47157e4c2c2c Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 19 Jun 2025 11:05:54 +0100 Subject: [PATCH 068/106] libcamera: Add ClockRecovery class to generate wallclock timestamps The ClockRecovery class takes pairs of timestamps from two different clocks, and models the second ("output") clock from the first ("input") clock. We can use it, in particular, to get a good wallclock estimate for a frame's SensorTimestamp. Signed-off-by: David Plowman Reviewed-by: Kieran Bingham Reviewed-by: Naushir Patuck Signed-off-by: Naushir Patuck Signed-off-by: Kieran Bingham --- include/libcamera/internal/clock_recovery.h | 68 ++++++ include/libcamera/internal/meson.build | 1 + src/libcamera/clock_recovery.cpp | 230 ++++++++++++++++++++ src/libcamera/meson.build | 1 + 4 files changed, 300 insertions(+) create mode 100644 include/libcamera/internal/clock_recovery.h create mode 100644 src/libcamera/clock_recovery.cpp diff --git a/include/libcamera/internal/clock_recovery.h b/include/libcamera/internal/clock_recovery.h new file mode 100644 index 000000000..43e46b7dc --- /dev/null +++ b/include/libcamera/internal/clock_recovery.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Raspberry Pi Ltd + * + * Camera recovery algorithm + */ +#pragma once + +#include + +namespace libcamera { + +class ClockRecovery +{ +public: + ClockRecovery(); + + void configure(unsigned int numSamples = 100, unsigned int maxJitter = 2000, + unsigned int minSamples = 10, unsigned int errorThreshold = 50000); + void reset(); + + void addSample(); + void addSample(uint64_t input, uint64_t output); + + uint64_t getOutput(uint64_t input); + +private: + /* Approximate number of samples over which the model state persists. */ + unsigned int numSamples_; + /* Remove any output jitter larger than this immediately. */ + unsigned int maxJitter_; + /* Number of samples required before we start to use model estimates. */ + unsigned int minSamples_; + /* Threshold above which we assume the wallclock has been reset. */ + unsigned int errorThreshold_; + + /* How many samples seen (up to numSamples_). */ + unsigned int count_; + /* This gets subtracted from all input values, just to make the numbers easier. */ + uint64_t inputBase_; + /* As above, for the output. */ + uint64_t outputBase_; + /* The previous input sample. */ + uint64_t lastInput_; + /* The previous output sample. */ + uint64_t lastOutput_; + + /* Average x value seen so far. */ + double xAve_; + /* Average y value seen so far */ + double yAve_; + /* Average x^2 value seen so far. */ + double x2Ave_; + /* Average x*y value seen so far. */ + double xyAve_; + + /* + * The latest estimate of linear parameters to derive the output clock + * from the input. + */ + double slope_; + double offset_; + + /* Use this cumulative error to monitor for spontaneous clock updates. */ + double error_; +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index 33f318b2b..5c80a28c4 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -11,6 +11,7 @@ libcamera_internal_headers = files([ 'camera_manager.h', 'camera_sensor.h', 'camera_sensor_properties.h', + 'clock_recovery.h', 'control_serializer.h', 'control_validator.h', 'converter.h', diff --git a/src/libcamera/clock_recovery.cpp b/src/libcamera/clock_recovery.cpp new file mode 100644 index 000000000..abacf444f --- /dev/null +++ b/src/libcamera/clock_recovery.cpp @@ -0,0 +1,230 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Raspberry Pi Ltd + * + * Clock recovery algorithm + */ + +#include "libcamera/internal/clock_recovery.h" + +#include + +#include + +/** + * \file clock_recovery.h + * \brief Clock recovery - deriving one clock from another independent clock + */ + +namespace libcamera { + +LOG_DEFINE_CATEGORY(ClockRec) + +/** + * \class ClockRecovery + * \brief Recover an output clock from an input clock + * + * The ClockRecovery class derives an output clock from an input clock, + * modelling the output clock as being linearly related to the input clock. + * For example, we may use it to derive wall clock timestamps from timestamps + * measured by the internal system clock which counts local time since boot. + * + * When pairs of corresponding input and output timestamps are available, + * they should be submitted to the model with addSample(). The model will + * update, and output clock values for known input clock values can be + * obtained using getOutput(). + * + * As a convenience, if the input clock is indeed the time since boot, and the + * output clock represents a real wallclock time, then addSample() can be + * called with no arguments, and a pair of timestamps will be captured at + * that moment. + * + * The configure() function accepts some configuration parameters to control + * the linear fitting process. + */ + +/** + * \brief Construct a ClockRecovery + */ +ClockRecovery::ClockRecovery() +{ + configure(); + reset(); +} + +/** + * \brief Set configuration parameters + * \param[in] numSamples The approximate duration for which the state of the model + * is persistent + * \param[in] maxJitter New output samples are clamped to no more than this + * amount of jitter, to prevent sudden swings from having a large effect + * \param[in] minSamples The fitted clock model is not used to generate outputs + * until this many samples have been received + * \param[in] errorThreshold If the accumulated differences between input and + * output clocks reaches this amount over a few frames, the model is reset + */ +void ClockRecovery::configure(unsigned int numSamples, unsigned int maxJitter, + unsigned int minSamples, unsigned int errorThreshold) +{ + LOG(ClockRec, Debug) + << "configure " << numSamples << " " << maxJitter << " " << minSamples << " " << errorThreshold; + + numSamples_ = numSamples; + maxJitter_ = maxJitter; + minSamples_ = minSamples; + errorThreshold_ = errorThreshold; +} + +/** + * \brief Reset the clock recovery model and start again from scratch + */ +void ClockRecovery::reset() +{ + LOG(ClockRec, Debug) << "reset"; + + lastInput_ = 0; + lastOutput_ = 0; + xAve_ = 0; + yAve_ = 0; + x2Ave_ = 0; + xyAve_ = 0; + count_ = 0; + error_ = 0.0; + /* + * Setting slope_ and offset_ to zero initially means that the clocks + * advance at exactly the same rate. + */ + slope_ = 0.0; + offset_ = 0.0; +} + +/** + * \brief Add a sample point to the clock recovery model, for recovering a wall + * clock value from the internal system time since boot + * + * This is a convenience function to make it easy to derive a wall clock value + * (using the Linux CLOCK_REALTIME) from the time since the system started + * (measured by CLOCK_BOOTTIME). + */ +void ClockRecovery::addSample() +{ + LOG(ClockRec, Debug) << "addSample"; + + struct timespec bootTime1; + struct timespec bootTime2; + struct timespec wallTime; + + /* Get boot and wall clocks in microseconds. */ + clock_gettime(CLOCK_BOOTTIME, &bootTime1); + clock_gettime(CLOCK_REALTIME, &wallTime); + clock_gettime(CLOCK_BOOTTIME, &bootTime2); + uint64_t boot1 = bootTime1.tv_sec * 1000000ULL + bootTime1.tv_nsec / 1000; + uint64_t boot2 = bootTime2.tv_sec * 1000000ULL + bootTime2.tv_nsec / 1000; + uint64_t boot = (boot1 + boot2) / 2; + uint64_t wall = wallTime.tv_sec * 1000000ULL + wallTime.tv_nsec / 1000; + + addSample(boot, wall); +} + +/** + * \brief Add a sample point to the clock recovery model, specifying the exact + * input and output clock values + * \param[in] input The input clock value + * \param[in] output The value of the output clock at the same moment, as far + * as possible, that the input clock was sampled + * + * This function should be used for corresponding clocks other than the Linux + * BOOTTIME and REALTIME clocks. + */ +void ClockRecovery::addSample(uint64_t input, uint64_t output) +{ + LOG(ClockRec, Debug) << "addSample " << input << " " << output; + + if (count_ == 0) { + inputBase_ = input; + outputBase_ = output; + } + + /* + * We keep an eye on cumulative drift over the last several frames. If this exceeds a + * threshold, then probably the system clock has been updated and we're going to have to + * reset everything and start over. + */ + if (lastOutput_) { + int64_t inputDiff = getOutput(input) - getOutput(lastInput_); + int64_t outputDiff = output - lastOutput_; + error_ = error_ * 0.95 + (outputDiff - inputDiff); + if (std::abs(error_) > errorThreshold_) { + reset(); + inputBase_ = input; + outputBase_ = output; + } + } + lastInput_ = input; + lastOutput_ = output; + + /* + * Never let the new output value be more than maxJitter_ away from what + * we would have expected. This is just to reduce the effect of sudden + * large delays in the measured output. + */ + uint64_t expectedOutput = getOutput(input); + output = std::clamp(output, expectedOutput - maxJitter_, expectedOutput + maxJitter_); + + /* + * We use x, y, x^2 and x*y sums to calculate the best fit line. Here we + * update them by pretending we have count_ samples at the previous fit, + * and now one new one. Gradually the effect of the older values gets + * lost. This is a very simple way of updating the fit (there are much + * more complicated ones!), but it works well enough. Using averages + * instead of sums makes the relative effect of old values and the new + * sample clearer. + */ + double x = static_cast(input - inputBase_); + double y = static_cast(output - outputBase_) - x; + unsigned int count1 = count_ + 1; + xAve_ = (count_ * xAve_ + x) / count1; + yAve_ = (count_ * yAve_ + y) / count1; + x2Ave_ = (count_ * x2Ave_ + x * x) / count1; + xyAve_ = (count_ * xyAve_ + x * y) / count1; + + /* + * Don't update slope and offset until we've seen "enough" sample + * points. Note that the initial settings for slope_ and offset_ + * ensures that the wallclock advances at the same rate as the realtime + * clock (but with their respective initial offsets). + */ + if (count_ > minSamples_) { + /* These are the standard equations for least squares linear regression. */ + slope_ = (count1 * count1 * xyAve_ - count1 * xAve_ * count1 * yAve_) / + (count1 * count1 * x2Ave_ - count1 * xAve_ * count1 * xAve_); + offset_ = yAve_ - slope_ * xAve_; + } + + /* + * Don't increase count_ above numSamples_, as this controls the long-term + * amount of the residual fit. + */ + if (count1 < numSamples_) + count_++; +} + +/** + * \brief Calculate the output clock value according to the model from an input + * clock value + * \param[in] input The input clock value + * + * \return Output clock value + */ +uint64_t ClockRecovery::getOutput(uint64_t input) +{ + double x = static_cast(input - inputBase_); + double y = slope_ * x + offset_; + uint64_t output = y + x + outputBase_; + + LOG(ClockRec, Debug) << "getOutput " << input << " " << output; + + return output; +} + +} /* namespace libcamera */ diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 202db1efe..28a3b0f36 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -21,6 +21,7 @@ libcamera_internal_sources = files([ 'byte_stream_buffer.cpp', 'camera_controls.cpp', 'camera_lens.cpp', + 'clock_recovery.cpp', 'control_serializer.cpp', 'control_validator.cpp', 'converter.cpp', From 1d1ba78b457d636a19c784784cea876f7776f7b4 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 19 Jun 2025 11:05:55 +0100 Subject: [PATCH 069/106] controls: Add camera synchronisation controls for Raspberry Pi New controls are added to control the camera "sync" algorithm, which allows different cameras to synchronise their frames. For the time being, the controls are Raspberry Pi specific, though this is expected to change in future. Signed-off-by: David Plowman Reviewed-by: Naushir Patuck Signed-off-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_rpi.yaml | 112 +++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/src/libcamera/control_ids_rpi.yaml b/src/libcamera/control_ids_rpi.yaml index 8d1e8b47c..a86151123 100644 --- a/src/libcamera/control_ids_rpi.yaml +++ b/src/libcamera/control_ids_rpi.yaml @@ -71,4 +71,116 @@ controls: \sa StatsOutputEnable + - SyncMode: + type: int32_t + direction: in + description: | + Enable or disable camera synchronisation ("sync") mode. + + When sync mode is enabled, a camera will synchronise frames temporally + with other cameras, either attached to the same device or a different + one. There should be one "server" device, which broadcasts timing + information to one or more "clients". Communication is one-way, from + server to clients only, and it is only clients that adjust their frame + timings to match the server. + + Sync mode requires all cameras to be running at (as far as possible) the + same fixed framerate. Clients may continue to make adjustments to keep + their cameras synchronised with the server for the duration of the + session, though any updates after the initial ones should remain small. + + \sa SyncReady + \sa SyncTimer + \sa SyncFrames + + enum: + - name: SyncModeOff + value: 0 + description: Disable sync mode. + - name: SyncModeServer + value: 1 + description: | + Enable sync mode, act as server. The server broadcasts timing + messages to any clients that are listening, so that the clients can + synchronise their camera frames with the server's. + - name: SyncModeClient + value: 2 + description: | + Enable sync mode, act as client. A client listens for any server + messages, and arranges for its camera frames to synchronise as + closely as possible with the server's. Many clients can listen out + for the same server. Clients can also be started ahead of any + servers, causing them merely to wait for the server to start. + + - SyncReady: + type: bool + direction: out + description: | + When using the camera synchronisation algorithm, the server broadcasts + timing information to the clients. This also includes the time (some + number of frames in the future, called the "ready time") at which the + server will signal its controlling application, using this control, to + start using the image frames. + + The client receives the "ready time" from the server, and will signal + its application to start using the frames at this same moment. + + While this control value is false, applications (on both client and + server) should continue to wait, and not use the frames. + + Once this value becomes true, it means that this is the first frame + where the server and its clients have agreed that they will both be + synchronised and that applications should begin consuming frames. + Thereafter, this control will continue to signal the value true for + the rest of the session. + + \sa SyncMode + \sa SyncTimer + \sa SyncFrames + + - SyncTimer: + type: int64_t + direction: out + description: | + This reports the amount of time, in microseconds, until the "ready + time", at which the server and client will signal their controlling + applications that the frames are now synchronised and should be + used. The value may be refined slightly over time, becoming more precise + as the "ready time" approaches. + + Servers always report this value, whereas clients will omit this control + until they have received a message from the server that enables them to + calculate it. + + Normally the value will start positive (the "ready time" is in the + future), and decrease towards zero, before becoming negative (the "ready + time" has elapsed). So there should be just one frame where the timer + value is, or is very close to, zero - the one for which the SyncReady + control becomes true. At this moment, the value indicates how closely + synchronised the client believes it is with the server. + + But note that if frames are being dropped, then the "near zero" valued + frame, or indeed any other, could be skipped. In these cases the timer + value allows an application to deduce that this has happened. + + \sa SyncMode + \sa SyncReady + \sa SyncFrames + + - SyncFrames: + type: int32_t + direction: in + description: | + The number of frames the server should wait, after enabling + SyncModeServer, before signalling (via the SyncReady control) that + frames should be used. This therefore determines the "ready time" for + all synchronised cameras. + + This control value should be set only for the device that is to act as + the server, before or at the same moment at which SyncModeServer is + enabled. + + \sa SyncMode + \sa SyncReady + \sa SyncTimer ... From 1537da74427791bb3b5880e7d002daf8ea42db31 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 19 Jun 2025 11:05:56 +0100 Subject: [PATCH 070/106] pipeline: rpi: Add wallclock timestamp support A ClockRecovery object is added for derived classes to use, and wallclock timestamps are copied into the request metadata for applications. Wallclock timestamps are derived corresponding to the sensor timestamp, and made available to the base pipeline handler class and to IPAs, for both vc4 and pisp platforms. Signed-off-by: David Plowman Reviewed-by: Kieran Bingham Reviewed-by: Naushir Patuck Signed-off-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/libcamera/pipeline/rpi/common/pipeline_base.cpp | 5 +++++ src/libcamera/pipeline/rpi/common/pipeline_base.h | 3 +++ src/libcamera/pipeline/rpi/pisp/pisp.cpp | 10 ++++++++-- src/libcamera/pipeline/rpi/vc4/vc4.cpp | 10 ++++++++-- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index d8c7ca935..e14d3b913 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -685,6 +685,9 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) return ret; } + /* A good moment to add an initial clock sample. */ + data->wallClockRecovery_.addSample(); + /* * Reset the delayed controls with the gain and exposure values set by * the IPA. @@ -1482,6 +1485,8 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request { request->metadata().set(controls::SensorTimestamp, bufferControls.get(controls::SensorTimestamp).value_or(0)); + request->metadata().set(controls::FrameWallClock, + bufferControls.get(controls::FrameWallClock).value_or(0)); if (cropParams_.size()) { std::vector crops; diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 898f31577..4c5743e04 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -20,6 +20,7 @@ #include "libcamera/internal/bayer_format.h" #include "libcamera/internal/camera.h" #include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/clock_recovery.h" #include "libcamera/internal/framebuffer.h" #include "libcamera/internal/media_device.h" #include "libcamera/internal/media_object.h" @@ -172,6 +173,8 @@ public: Config config_; + ClockRecovery wallClockRecovery_; + protected: void fillRequestMetadata(const ControlList &bufferControls, Request *request); diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index ccf135c3d..2df91bacf 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -1755,9 +1755,15 @@ void PiSPCameraData::cfeBufferDequeue(FrameBuffer *buffer) auto [ctrl, delayContext] = delayedCtrls_->get(buffer->metadata().sequence); /* * Add the frame timestamp to the ControlList for the IPA to use - * as it does not receive the FrameBuffer object. + * as it does not receive the FrameBuffer object. Also derive a + * corresponding wallclock value. */ - ctrl.set(controls::SensorTimestamp, buffer->metadata().timestamp); + wallClockRecovery_.addSample(); + uint64_t sensorTimestamp = buffer->metadata().timestamp; + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp / 1000); + + ctrl.set(controls::SensorTimestamp, sensorTimestamp); + ctrl.set(controls::FrameWallClock, wallClockTimestamp); job.sensorControls = std::move(ctrl); job.delayContext = delayContext; } else if (stream == &cfe_[Cfe::Config]) { diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index ac6dab814..e99a7edf8 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -773,9 +773,15 @@ void Vc4CameraData::unicamBufferDequeue(FrameBuffer *buffer) auto [ctrl, delayContext] = delayedCtrls_->get(buffer->metadata().sequence); /* * Add the frame timestamp to the ControlList for the IPA to use - * as it does not receive the FrameBuffer object. + * as it does not receive the FrameBuffer object. Also derive a + * corresponding wallclock value. */ - ctrl.set(controls::SensorTimestamp, buffer->metadata().timestamp); + wallClockRecovery_.addSample(); + uint64_t sensorTimestamp = buffer->metadata().timestamp; + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp / 1000); + + ctrl.set(controls::SensorTimestamp, sensorTimestamp); + ctrl.set(controls::FrameWallClock, wallClockTimestamp); bayerQueue_.push({ buffer, std::move(ctrl), delayContext }); } else { embeddedQueue_.push(buffer); From 02a3b436c4d16498e05c79cd84daacef81906ab4 Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 9 Jun 2025 18:01:51 +0300 Subject: [PATCH 071/106] ipa: rkisp1: Move Sharpness control creation to Filter algorithm The Sharpness control is used solely by the Filter algorithm. Create it there, to avoid exposing it to applications when the algorithm is disabled. Signed-off-by: Laurent Pinchart Reviewed-by: Kieran Bingham Reviewed-by: Paul Elder --- src/ipa/rkisp1/algorithms/filter.cpp | 11 +++++++++++ src/ipa/rkisp1/algorithms/filter.h | 1 + src/ipa/rkisp1/rkisp1.cpp | 1 - 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ipa/rkisp1/algorithms/filter.cpp b/src/ipa/rkisp1/algorithms/filter.cpp index 7598ef8a9..8ad798017 100644 --- a/src/ipa/rkisp1/algorithms/filter.cpp +++ b/src/ipa/rkisp1/algorithms/filter.cpp @@ -39,6 +39,17 @@ LOG_DEFINE_CATEGORY(RkISP1Filter) static constexpr uint32_t kFiltLumWeightDefault = 0x00022040; static constexpr uint32_t kFiltModeDefault = 0x000004f2; +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Filter::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + auto &cmap = context.ctrlMap; + cmap[&controls::Sharpness] = ControlInfo(0.0f, 10.0f, 1.0f); + + return 0; +} /** * \copydoc libcamera::ipa::Algorithm::queueRequest */ diff --git a/src/ipa/rkisp1/algorithms/filter.h b/src/ipa/rkisp1/algorithms/filter.h index 8f858e574..37d8938d3 100644 --- a/src/ipa/rkisp1/algorithms/filter.h +++ b/src/ipa/rkisp1/algorithms/filter.h @@ -21,6 +21,7 @@ public: Filter() = default; ~Filter() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; void queueRequest(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const ControlList &controls) override; diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index 1ed7d7d92..cf66d5553 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -116,7 +116,6 @@ const IPAHwSettings ipaHwSettingsV12{ /* List of controls handled by the RkISP1 IPA */ const ControlInfoMap::Map rkisp1Controls{ { &controls::DebugMetadataEnable, ControlInfo(false, true, false) }, - { &controls::Sharpness, ControlInfo(0.0f, 10.0f, 1.0f) }, { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) }, }; From c19047dfdfb451deac404841b79e1b5a6b89ff5f Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Wed, 4 Jun 2025 16:51:46 +0300 Subject: [PATCH 072/106] gstreamer: Use std::exchange() instead of g_steal_pointer() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit g_steal_pointer) only preserves the type since glib 2.68, requiring casts. Use std::exchange() instead. Signed-off-by: Laurent Pinchart Reviewed-by: Nicolas Dufresne Reviewed-by: Barnabás Pőcze --- src/gstreamer/gstlibcamerasrc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 51ba8b67a..3aca4eeda 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -1087,7 +1087,7 @@ gst_libcamera_src_request_new_pad(GstElement *element, GstPadTemplate *templ, gst_child_proxy_child_added(GST_CHILD_PROXY(self), G_OBJECT(pad), GST_OBJECT_NAME(pad)); - return reinterpret_cast(g_steal_pointer(&pad)); + return std::exchange(pad, nullptr); } static void From 8ea3ef083fd009c1be38a0ff4001ba24da5f5cc1 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Thu, 12 Jun 2025 15:22:36 +0200 Subject: [PATCH 073/106] libcamera: test: Add a failing test for the log level parser Log level parsing doesn't always work as expected. Add a failing test for that. Signed-off-by: Stefan Klug Reviewed-by: Laurent Pinchart Reviewed-by: Kieran Bingham --- test/log/log_api.cpp | 39 ++++++++++++++++++++++++++++++++++++++- test/log/meson.build | 5 +++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/test/log/log_api.cpp b/test/log/log_api.cpp index 0b999738d..8d19cf0ce 100644 --- a/test/log/log_api.cpp +++ b/test/log/log_api.cpp @@ -26,6 +26,11 @@ using namespace std; using namespace libcamera; LOG_DEFINE_CATEGORY(LogAPITest) +LOG_DEFINE_CATEGORY(Cat0) +LOG_DEFINE_CATEGORY(Cat1) +LOG_DEFINE_CATEGORY(Cat2) +LOG_DEFINE_CATEGORY(Cat3) +LOG_DEFINE_CATEGORY(Cat4) class LogAPITest : public Test { @@ -74,6 +79,34 @@ protected: return TestPass; } + int testEnvLevels() + { + setenv("LIBCAMERA_LOG_LEVELS", + "Cat0:0,Cat0:9999,Cat1:INFO,Cat1:INVALID,Cat2:2,Cat2:-1," + "Cat3:ERROR,Cat3:{[]},Cat4:4,Cat4:rubbish", + true); + logSetTarget(libcamera::LoggingTargetNone); + + const std::pair expected[] = { + { _LOG_CATEGORY(Cat0)(), libcamera::LogDebug }, + { _LOG_CATEGORY(Cat1)(), libcamera::LogInfo }, + { _LOG_CATEGORY(Cat2)(), libcamera::LogWarning }, + { _LOG_CATEGORY(Cat3)(), libcamera::LogError }, + { _LOG_CATEGORY(Cat4)(), libcamera::LogFatal }, + }; + bool ok = true; + + for (const auto &[c, s] : expected) { + if (c.severity() != s) { + ok = false; + cerr << "Severity of " << c.name() << " (" << c.severity() << ") " + << "does not equal " << s << endl; + } + } + + return ok ? TestPass : TestFail; + } + int testFile() { int fd = open("/tmp", O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR); @@ -135,7 +168,11 @@ protected: int run() override { - int ret = testFile(); + int ret = testEnvLevels(); + if (ret != TestPass) + return TestFail; + + ret = testFile(); if (ret != TestPass) return TestFail; diff --git a/test/log/meson.build b/test/log/meson.build index 2298ff84e..80b0697a0 100644 --- a/test/log/meson.build +++ b/test/log/meson.build @@ -1,7 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 log_test = [ - {'name': 'log_api', 'sources': ['log_api.cpp']}, + {'name': 'log_api', 'sources': ['log_api.cpp'], 'should_fail': true}, {'name': 'log_process', 'sources': ['log_process.cpp']}, ] @@ -11,5 +11,6 @@ foreach test : log_test link_with : test_libraries, include_directories : test_includes_internal) - test(test['name'], exe, suite : 'log') + test(test['name'], exe, suite : 'log', + should_fail : test.get('should_fail', false)) endforeach From 0dfb052fbdd641f1127d8db00b8656f5584d68d9 Mon Sep 17 00:00:00 2001 From: Stefan Klug Date: Fri, 6 Jun 2025 10:21:35 +0200 Subject: [PATCH 074/106] libcamera: base: Fix log level parsing when multiple categories are listed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For a list of log levels like LIBCAMERA_LOG_LEVELS="CatA:0,CatB:1" only the severity of the last entry is correctly parsed. Due to the change of level to a string_view in 24c2caa1c1b3 ("libcamera: base: log: Use `std::string_view` to avoid some copies") the level is no longer necessarily null terminated as it is a view on the original data. Replace the check for a terminating null by a check for the end position to fix the issue. Fixes: 24c2caa1c1b3 ("libcamera: base: log: Use `std::string_view` to avoid some copies") Signed-off-by: Stefan Klug Reviewed-by: Barnabás Pőcze Reviewed-by: Laurent Pinchart --- src/libcamera/base/log.cpp | 5 +++-- test/log/meson.build | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcamera/base/log.cpp b/src/libcamera/base/log.cpp index 8bf3e1daa..6a8e2a3eb 100644 --- a/src/libcamera/base/log.cpp +++ b/src/libcamera/base/log.cpp @@ -690,8 +690,9 @@ LogSeverity Logger::parseLogLevel(std::string_view level) unsigned int severity = LogInvalid; if (std::isdigit(level[0])) { - auto [end, ec] = std::from_chars(level.data(), level.data() + level.size(), severity); - if (ec != std::errc() || *end != '\0' || severity > LogFatal) + const char *levelEnd = level.data() + level.size(); + auto [end, ec] = std::from_chars(level.data(), levelEnd, severity); + if (ec != std::errc() || end != levelEnd || severity > LogFatal) severity = LogInvalid; } else { for (unsigned int i = 0; i < std::size(names); ++i) { diff --git a/test/log/meson.build b/test/log/meson.build index 80b0697a0..f413c3898 100644 --- a/test/log/meson.build +++ b/test/log/meson.build @@ -1,7 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 log_test = [ - {'name': 'log_api', 'sources': ['log_api.cpp'], 'should_fail': true}, + {'name': 'log_api', 'sources': ['log_api.cpp']}, {'name': 'log_process', 'sources': ['log_process.cpp']}, ] From 5f4d2ac935ee40e901bacc844869c96a20c296c7 Mon Sep 17 00:00:00 2001 From: Kieran Bingham Date: Thu, 19 Jun 2025 12:16:14 +0100 Subject: [PATCH 075/106] libcamera: controls: Revert incorrect SPDX removal In commit 6a09deaf7d4a ("controls: Add FrameWallClock control") the existing SPDX was accidentally removed, likely from a rebase operation at some point. Unfortunately as this patch had already collected Reviewed-by tags, the surruptious removal wasn't noticed until after it was merged. Re-insert the existing SPDX and copyright banner as the header to the control definitions file. Signed-off-by: Kieran Bingham Reviewed-by: Naushir Patuck Reviewed-by: Laurent Pinchart Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_core.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index 028919ef3..46f51436c 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -1,4 +1,6 @@ - +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# Copyright (C) 2019, Google Inc. # %YAML 1.1 --- From a29c53f6a646ed9644a94b2face62d078d8290be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Wed, 22 Jan 2025 13:09:11 +0100 Subject: [PATCH 076/106] meson: Use libyaml wrap file from wrapdb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the libyaml wrap file from the meson wrapdb instead of creating the wrap file manually and using the cmake module. This provides better integration with meson, such as the `force_fallback_for` built-in option. This is also needed because the upstream CMakeLists.txt is out of date, failing with a sufficiently new cmake version: CMake Error at CMakeLists.txt:2 (cmake_minimum_required): Compatibility with CMake < 3.5 has been removed from CMake. The above is nonetheless addressed by https://github.com/yaml/libyaml/pull/314, but the project seems a bit inactive at the moment. The wrap file was added using `meson wrap install libyaml`, and it can be updated using `meson wrap update libyaml`. `default_library=static` is used to match the behaviour of the previously used cmake build. `werror=false` needs to be set because libyaml does not compile without warnings, and that would abort the build process otherwise. Signed-off-by: Barnabás Pőcze Reviewed-by: Laurent Pinchart Reviewed-by: Kieran Bingham --- src/libcamera/meson.build | 16 ++++------------ subprojects/libyaml.wrap | 16 +++++++++++----- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 28a3b0f36..de1eb99b2 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -84,7 +84,10 @@ if not cc.has_function('dlopen') libdl = cc.find_library('dl') endif libudev = dependency('libudev', required : get_option('udev')) -libyaml = dependency('yaml-0.1', required : false) +libyaml = dependency('yaml-0.1', default_options : [ + 'default_library=static', + 'werror=false', +]) # Use one of gnutls or libcrypto (provided by OpenSSL), trying gnutls first. libcrypto = dependency('gnutls', required : false) @@ -120,17 +123,6 @@ if libudev.found() ]) endif -# Fallback to a subproject if libyaml isn't found, as it's not packaged in AOSP. -if not libyaml.found() - cmake = import('cmake') - - libyaml_vars = cmake.subproject_options() - libyaml_vars.add_cmake_defines({'CMAKE_POSITION_INDEPENDENT_CODE': 'ON'}) - libyaml_vars.append_compile_args('c', '-Wno-unused-value') - libyaml_wrap = cmake.subproject('libyaml', options : libyaml_vars) - libyaml = libyaml_wrap.dependency('yaml') -endif - control_sources = [] controls_mode_files = { diff --git a/subprojects/libyaml.wrap b/subprojects/libyaml.wrap index 392416c61..44ac0ff8e 100644 --- a/subprojects/libyaml.wrap +++ b/subprojects/libyaml.wrap @@ -1,7 +1,13 @@ # SPDX-License-Identifier: CC0-1.0 -[wrap-git] -directory = libyaml -url = https://github.com/yaml/libyaml -# tags/0.2.5 -revision = 2c891fc7a770e8ba2fec34fc6b545c672beb37e6 +[wrap-file] +directory = yaml-0.2.5 +source_url = https://pyyaml.org/download/libyaml/yaml-0.2.5.tar.gz +source_filename = yaml-0.2.5.tar.gz +source_hash = c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4 +patch_filename = libyaml_0.2.5-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/libyaml_0.2.5-1/get_patch +patch_hash = bf2e9b922be00b6b00c5fce29d9fb8dc83f0431c77239f3b73e8b254d3f3f5b5 + +[provide] +yaml-0.1 = yaml_dep From e9528306f257416c94bc882b89548e20d0aeebc3 Mon Sep 17 00:00:00 2001 From: Umang Jain Date: Thu, 26 Jun 2025 10:54:41 +0530 Subject: [PATCH 077/106] camera_sensor: Expand on computeTransform() documentation The description for computeTransform() when the desired orientation cannot be achieved, can be expanded a further bit, to clearly report that orientation will be adjusted to native camera sensor mounting rotation. Signed-off-by: Umang Jain Reviewed-by: Jacopo Mondi Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- src/libcamera/sensor/camera_sensor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libcamera/sensor/camera_sensor.cpp b/src/libcamera/sensor/camera_sensor.cpp index d19b5e2e7..4f2fd2690 100644 --- a/src/libcamera/sensor/camera_sensor.cpp +++ b/src/libcamera/sensor/camera_sensor.cpp @@ -302,8 +302,9 @@ int CameraSensor::setEmbeddedDataEnabled(bool enable) * camera sensor, likely at configure() time. * * If the requested \a orientation cannot be obtained, the \a orientation - * parameter is adjusted to report the current image orientation and - * Transform::Identity is returned. + * parameter is adjusted to report the native image orientation (i.e. resulting + * from the physical mounting rotation of the camera sensor, without any + * transformation) and Transform::Identity is returned. * * If the requested \a orientation can be obtained, the function computes a * Transform and does not adjust \a orientation. From 35ee8752b745024d5f00c34c3c17072e57b93d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Thu, 5 Jun 2025 14:18:38 +0200 Subject: [PATCH 078/106] libcamera: pipeline: uvcvideo: Silently ignore `AeEnable` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `AeEnable` control is handled in `Camera::queueRequest()` but it still reaches the pipeline handler because a single element cannot be removed from a `ControlList`. So ignore it silently. Fixes: ffcecda4d5b9b5 ("libcamera: pipeline: uvcvideo: Report new AeEnable control as available") Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Laurent Pinchart Reviewed-by: Paul Elder --- src/libcamera/pipeline/uvcvideo/uvcvideo.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp index 58aa0eb4c..d52b88042 100644 --- a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp +++ b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp @@ -331,6 +331,8 @@ int PipelineHandlerUVC::processControl(const UVCCameraData *data, ControlList *c cid = V4L2_CID_GAIN; else if (id == controls::Gamma) cid = V4L2_CID_GAMMA; + else if (id == controls::AeEnable) + return 0; /* Handled in `Camera::queueRequest()`. */ else return -EINVAL; From 5f94209b1d2149a3ceea3692bae49256038c2bb3 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Fri, 27 Jun 2025 10:46:26 +0100 Subject: [PATCH 079/106] pipeline: rpi: Fix for enumerating the media graphs When there are multiple entities between the sensor and CFE device (e.g. a serialiser and deserialiser or multiple mux devices), the media graph enumeration would work incorrectly and report that the frontend entity was not found. This is because the found flag was stored locally in a boolean and got lost in the recursion. Fix this by explicitly tracking and returning the frontend found flag through the return value of enumerateVideoDevices(). This ensures the flag does not get lost through nested recursion. This flag can also be used to fail a camera registration if the frontend is not found. Signed-off-by: Naushir Patuck Reviewed-by: Umang Jain Reviewed-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- .../pipeline/rpi/common/pipeline_base.cpp | 22 +++++++++++-------- .../pipeline/rpi/common/pipeline_base.h | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index e14d3b913..eafe94427 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -806,7 +806,8 @@ int PipelineHandlerBase::registerCamera(std::unique_ptr &camera * chain. There may be a cascade of devices in this chain! */ MediaLink *link = sensorEntity->getPadByIndex(0)->links()[0]; - data->enumerateVideoDevices(link, frontendName); + if (!data->enumerateVideoDevices(link, frontendName)) + return -EINVAL; ipa::RPi::InitResult result; if (data->loadIPA(&result)) { @@ -1018,16 +1019,20 @@ void CameraData::freeBuffers() * | Sensor2 | | Sensor3 | * +---------+ +---------+ */ -void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &frontend) +bool CameraData::enumerateVideoDevices(MediaLink *link, const std::string &frontend) { const MediaPad *sinkPad = link->sink(); const MediaEntity *entity = sinkPad->entity(); bool frontendFound = false; + /* Once we reach the Frontend entity, we are done. */ + if (link->sink()->entity()->name() == frontend) + return true; + /* We only deal with Video Mux and Bridge devices in cascade. */ if (entity->function() != MEDIA_ENT_F_VID_MUX && entity->function() != MEDIA_ENT_F_VID_IF_BRIDGE) - return; + return false; /* Find the source pad for this Video Mux or Bridge device. */ const MediaPad *sourcePad = nullptr; @@ -1039,7 +1044,7 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front * and this branch in the cascade. */ if (sourcePad) - return; + return false; sourcePad = pad; } @@ -1056,12 +1061,9 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front * other Video Mux and Bridge devices. */ for (MediaLink *l : sourcePad->links()) { - enumerateVideoDevices(l, frontend); - /* Once we reach the Frontend entity, we are done. */ - if (l->sink()->entity()->name() == frontend) { - frontendFound = true; + frontendFound = enumerateVideoDevices(l, frontend); + if (frontendFound) break; - } } /* This identifies the end of our entity enumeration recursion. */ @@ -1076,6 +1078,8 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front bridgeDevices_.clear(); } } + + return frontendFound; } int CameraData::loadPipelineConfiguration() diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 4c5743e04..4bce4ec4f 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -68,7 +68,7 @@ public: void freeBuffers(); virtual void platformFreeBuffers() = 0; - void enumerateVideoDevices(MediaLink *link, const std::string &frontend); + bool enumerateVideoDevices(MediaLink *link, const std::string &frontend); int loadPipelineConfiguration(); int loadIPA(ipa::RPi::InitResult *result); From 6b5cc1c92a33704e23bab2e5da699fa1d0678803 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Barnab=C3=A1s=20P=C5=91cze?= Date: Thu, 22 May 2025 13:06:02 +0200 Subject: [PATCH 080/106] libcamera: pipeline: uvcvideo: Handle controls during startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Process the control list passed to `Camera::start()`, and set the V4L2 controls accordingly. Signed-off-by: Barnabás Pőcze Reviewed-by: Kieran Bingham Reviewed-by: Umang Jain Reviewed-by: Paul Elder --- src/libcamera/pipeline/uvcvideo/uvcvideo.cpp | 27 +++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp index d52b88042..4b5816dfd 100644 --- a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp +++ b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp @@ -100,7 +100,7 @@ public: private: int processControl(const UVCCameraData *data, ControlList *controls, unsigned int id, const ControlValue &value); - int processControls(UVCCameraData *data, Request *request); + int processControls(UVCCameraData *data, const ControlList &reqControls); bool acquireDevice(Camera *camera) override; void releaseDevice(Camera *camera) override; @@ -287,7 +287,7 @@ int PipelineHandlerUVC::exportFrameBuffers(Camera *camera, Stream *stream, return data->video_->exportBuffers(count, buffers); } -int PipelineHandlerUVC::start(Camera *camera, [[maybe_unused]] const ControlList *controls) +int PipelineHandlerUVC::start(Camera *camera, const ControlList *controls) { UVCCameraData *data = cameraData(camera); unsigned int count = data->stream_.configuration().bufferCount; @@ -296,13 +296,22 @@ int PipelineHandlerUVC::start(Camera *camera, [[maybe_unused]] const ControlList if (ret < 0) return ret; - ret = data->video_->streamOn(); - if (ret < 0) { - data->video_->releaseBuffers(); - return ret; + if (controls) { + ret = processControls(data, *controls); + if (ret < 0) + goto err_release_buffers; } + ret = data->video_->streamOn(); + if (ret < 0) + goto err_release_buffers; + return 0; + +err_release_buffers: + data->video_->releaseBuffers(); + + return ret; } void PipelineHandlerUVC::stopDevice(Camera *camera) @@ -412,11 +421,11 @@ int PipelineHandlerUVC::processControl(const UVCCameraData *data, ControlList *c return 0; } -int PipelineHandlerUVC::processControls(UVCCameraData *data, Request *request) +int PipelineHandlerUVC::processControls(UVCCameraData *data, const ControlList &reqControls) { ControlList controls(data->video_->controls()); - for (const auto &[id, value] : request->controls()) + for (const auto &[id, value] : reqControls) processControl(data, &controls, id, value); for (const auto &ctrl : controls) @@ -444,7 +453,7 @@ int PipelineHandlerUVC::queueRequestDevice(Camera *camera, Request *request) return -ENOENT; } - int ret = processControls(data, request); + int ret = processControls(data, request->controls()); if (ret < 0) return ret; From 30114cadd8cb651231efcd5fe32f2c8a0a72b7b4 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:22 +0100 Subject: [PATCH 081/106] ipa: rpi: Defer initialising AF LensPosition ControlInfo and value This fixes two small bugs: We previously populated LensPosition's ControlInfo with hard-coded values, ignoring the tuning file. Now we query the AfAlgorithm to get limits (over all AF ranges) and default (for AfRangeNormal). We previously sent a default position to the lens driver, even when a user-specified starting position would follow. Defer doing this, to reduce unnecessary lens movement at startup (for some drivers). Bug: https://bugs.libcamera.org/show_bug.cgi?id=258 Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/common/ipa_base.cpp | 53 +++++++++++++++++---------- src/ipa/rpi/controller/af_algorithm.h | 8 +++- src/ipa/rpi/controller/rpi/af.cpp | 16 +++++++- src/ipa/rpi/controller/rpi/af.h | 4 +- 4 files changed, 57 insertions(+), 24 deletions(-) diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index 6565f5366..a5bdcbb58 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -233,25 +233,6 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa agcStatus.analogueGain = defaultAnalogueGain; applyAGC(&agcStatus, ctrls); - /* - * Set the lens to the default (typically hyperfocal) position - * on first start. - */ - if (lensPresent_) { - RPiController::AfAlgorithm *af = - dynamic_cast(controller_.getAlgorithm("af")); - - if (af) { - float defaultPos = - ipaAfControls.at(&controls::LensPosition).def().get(); - ControlList lensCtrl(lensCtrls_); - int32_t hwpos; - - af->setLensPosition(defaultPos, &hwpos); - lensCtrl.set(V4L2_CID_FOCUS_ABSOLUTE, hwpos); - result->lensControls = std::move(lensCtrl); - } - } } result->sensorControls = std::move(ctrls); @@ -281,8 +262,20 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa ctrlMap.merge(ControlInfoMap::Map(ipaColourControls)); /* Declare Autofocus controls, only if we have a controllable lens */ - if (lensPresent_) + if (lensPresent_) { ctrlMap.merge(ControlInfoMap::Map(ipaAfControls)); + RPiController::AfAlgorithm *af = + dynamic_cast(controller_.getAlgorithm("af")); + if (af) { + double min, max, dflt; + af->getLensLimits(min, max); + dflt = af->getDefaultLensPosition(); + ctrlMap[&controls::LensPosition] = + ControlInfo(static_cast(min), + static_cast(max), + static_cast(dflt)); + } + } result->controlInfo = ControlInfoMap(std::move(ctrlMap), controls::controls); @@ -320,6 +313,26 @@ void IpaBase::start(const ControlList &controls, StartResult *result) /* Make a note of this as it tells us the HDR status of the first few frames. */ hdrStatus_ = agcStatus.hdr; + /* + * AF: If no lens position was specified, drive lens to a default position. + * This had to be deferred (not initialised by a constructor) until here + * to ensure that exactly ONE starting position is sent to the lens driver. + * It should be the static API default, not dependent on AF range or mode. + */ + if (firstStart_ && lensPresent_) { + RPiController::AfAlgorithm *af = dynamic_cast( + controller_.getAlgorithm("af")); + if (af && !af->getLensPosition()) { + int32_t hwpos; + double pos = af->getDefaultLensPosition(); + if (af->setLensPosition(pos, &hwpos, true)) { + ControlList lensCtrls(lensCtrls_); + lensCtrls.set(V4L2_CID_FOCUS_ABSOLUTE, hwpos); + setLensControls.emit(lensCtrls); + } + } + } + /* * Initialise frame counts, and decide how many frames must be hidden or * "mistrusted", which depends on whether this is a startup from cold, diff --git a/src/ipa/rpi/controller/af_algorithm.h b/src/ipa/rpi/controller/af_algorithm.h index ad9b57545..382609f9b 100644 --- a/src/ipa/rpi/controller/af_algorithm.h +++ b/src/ipa/rpi/controller/af_algorithm.h @@ -33,6 +33,10 @@ public: * * getMode() is provided mainly for validating controls. * getLensPosition() is provided for populating DeviceStatus. + * + * getDefaultlensPosition() and getLensLimits() were added for + * populating ControlInfoMap. They return the static API limits + * which should be independent of the current range or mode. */ enum AfRange { AfRangeNormal = 0, @@ -66,7 +70,9 @@ public: } virtual void setMode(AfMode mode) = 0; virtual AfMode getMode() const = 0; - virtual bool setLensPosition(double dioptres, int32_t *hwpos) = 0; + virtual double getDefaultLensPosition() const = 0; + virtual void getLensLimits(double &min, double &max) const = 0; + virtual bool setLensPosition(double dioptres, int32_t *hwpos, bool force = false) = 0; virtual std::optional getLensPosition() const = 0; virtual void triggerScan() = 0; virtual void cancelScan() = 0; diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 2157eb94f..041cb51db 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -715,11 +715,23 @@ void Af::setWindows(libcamera::Span const &wins) invalidateWeights(); } -bool Af::setLensPosition(double dioptres, int *hwpos) +double Af::getDefaultLensPosition() const +{ + return cfg_.ranges[AfRangeNormal].focusDefault; +} + +void Af::getLensLimits(double &min, double &max) const +{ + /* Limits for manual focus are set by map, not by ranges */ + min = cfg_.map.domain().start; + max = cfg_.map.domain().end; +} + +bool Af::setLensPosition(double dioptres, int *hwpos, bool force) { bool changed = false; - if (mode_ == AfModeManual) { + if (mode_ == AfModeManual || force) { LOG(RPiAf, Debug) << "setLensPosition: " << dioptres; ftarget_ = cfg_.map.domain().clamp(dioptres); changed = !(initted_ && fsmooth_ == ftarget_); diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index 317a51f3e..d8ece1ab1 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -54,7 +54,9 @@ public: void setWindows(libcamera::Span const &wins) override; void setMode(AfMode mode) override; AfMode getMode() const override; - bool setLensPosition(double dioptres, int32_t *hwpos) override; + double getDefaultLensPosition() const override; + void getLensLimits(double &min, double &max) const override; + bool setLensPosition(double dioptres, int32_t *hwpos, bool force) override; std::optional getLensPosition() const override; void triggerScan() override; void cancelScan() override; From a283287fbf0c633591698bef6670729748296f02 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:23 +0100 Subject: [PATCH 082/106] ipa: rpi: controller: Improve findPeak() function in AF algorithm Improve quadratic peak fitting in findPeak(). The old approximation was good but only valid when points were equally spaced and the MAX was not at one end of the series. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 041cb51db..8df614ed7 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -436,15 +436,28 @@ double Af::findPeak(unsigned i) const { double f = scanData_[i].focus; - if (i > 0 && i + 1 < scanData_.size()) { - double dropLo = scanData_[i].contrast - scanData_[i - 1].contrast; - double dropHi = scanData_[i].contrast - scanData_[i + 1].contrast; - if (0.0 <= dropLo && dropLo < dropHi) { - double param = 0.3125 * (1.0 - dropLo / dropHi) * (1.6 - dropLo / dropHi); - f += param * (scanData_[i - 1].focus - f); - } else if (0.0 <= dropHi && dropHi < dropLo) { - double param = 0.3125 * (1.0 - dropHi / dropLo) * (1.6 - dropHi / dropLo); - f += param * (scanData_[i + 1].focus - f); + if (scanData_.size() >= 3) { + /* + * Given the sample with the highest contrast score and its two + * neighbours either side (or same side if at the end of a scan), + * solve for the best lens position by fitting a parabola. + * Adapted from awb.cpp: interpolateQaudaratic() + */ + + if (i == 0) + i++; + else if (i + 1 >= scanData_.size()) + i--; + + double abx = scanData_[i - 1].focus - scanData_[i].focus; + double aby = scanData_[i - 1].contrast - scanData_[i].contrast; + double cbx = scanData_[i + 1].focus - scanData_[i].focus; + double cby = scanData_[i + 1].contrast - scanData_[i].contrast; + double denom = 2.0 * (aby * cbx - cby * abx); + if (std::abs(denom) >= (1.0 / 64.0) && denom * abx > 0.0) { + f = (aby * cbx * cbx - cby * abx * abx) / denom; + f = std::clamp(f, std::min(abx, cbx), std::max(abx, cbx)); + f += scanData_[i].focus; } } From 0fa2b05a868665c27d74433b227d54d8f3af6c89 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:24 +0100 Subject: [PATCH 083/106] ipa: rpi: controller: AutoFocus weighting tweak In getPhase(), stop using different weights for sumWc and sumWcp. This should improve linearity e.g. in earlyTerminationByPhase(). Phases are slightly larger but confidence values slightly reduced. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 8df614ed7..5304f54ca 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -328,9 +328,8 @@ bool Af::getPhase(PdafRegions const ®ions, double &phase, double &conf) if (c >= cfg_.confThresh) { if (c > cfg_.confClip) c = cfg_.confClip; - c -= (cfg_.confThresh >> 2); + c -= (cfg_.confThresh >> 1); sumWc += w * c; - c -= (cfg_.confThresh >> 2); sumWcp += (int64_t)(w * c) * (int64_t)data.phase; } } From 429a5ab48fe8e5ba1179f179a02a9b57cf037a6c Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:25 +0100 Subject: [PATCH 084/106] ipa: rpi: controller: Autofocus CAF/PDAF stability tweak When in Continuous AF mode using PDAF, only move the lens when phase has had the same sign for at least 4 frames. This reduces lens wobble in e.g. noisy conditions. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 12 +++++++++++- src/ipa/rpi/controller/rpi/af.h | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 5304f54ca..3b2d6167d 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -181,9 +181,11 @@ Af::Af(Controller *controller) ftarget_(-1.0), fsmooth_(-1.0), prevContrast_(0.0), + prevPhase_(0.0), skipCount_(0), stepCount_(0), dropCount_(0), + sameSignCount_(0), scanMaxContrast_(0.0), scanMinContrast_(1.0e9), scanData_(), @@ -513,6 +515,13 @@ void Af::doAF(double contrast, double phase, double conf) return; } + /* Count frames for which PDAF phase has had same sign */ + if (phase * prevPhase_ <= 0.0) + sameSignCount_ = 0; + else + sameSignCount_++; + prevPhase_ = phase; + if (scanState_ == ScanState::Pdaf) { /* * Use PDAF closed-loop control whenever available, in both CAF @@ -522,7 +531,8 @@ void Af::doAF(double contrast, double phase, double conf) * scan only after a number of frames with low PDAF confidence. */ if (conf > (dropCount_ ? 1.0 : 0.25) * cfg_.confEpsilon) { - doPDAF(phase, conf); + if (mode_ == AfModeAuto || sameSignCount_ >= 3) + doPDAF(phase, conf); if (stepCount_ > 0) stepCount_--; else if (mode_ != AfModeContinuous) diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index d8ece1ab1..b06a3a16f 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -158,7 +158,9 @@ private: bool initted_; double ftarget_, fsmooth_; double prevContrast_; + double prevPhase_; unsigned skipCount_, stepCount_, dropCount_; + unsigned sameSignCount_; unsigned scanMaxIndex_; double scanMaxContrast_, scanMinContrast_; std::vector scanData_; From 3d44987bc606a84a9191b39900b0fd17444b2a6c Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:26 +0100 Subject: [PATCH 085/106] ipa: rpi: controller: AutoFocus tweak earlyTerminationByPhase() Increase threshold for ETBP, from "confEpsilon" to "confThresh". Correct sign test to take account of pdafGain sign (typically -ve). Reduce allowed extrapolation range, but relax the check in the case of Continuous AF, when we go back into the PDAF closed loop. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 3b2d6167d..ecc0fc417 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -411,7 +411,7 @@ void Af::doPDAF(double phase, double conf) bool Af::earlyTerminationByPhase(double phase) { if (scanData_.size() > 0 && - scanData_[scanData_.size() - 1].conf >= cfg_.confEpsilon) { + scanData_[scanData_.size() - 1].conf >= cfg_.confThresh) { double oldFocus = scanData_[scanData_.size() - 1].focus; double oldPhase = scanData_[scanData_.size() - 1].phase; @@ -420,11 +420,12 @@ bool Af::earlyTerminationByPhase(double phase) * Interpolate/extrapolate the lens position for zero phase. * Check that the extrapolation is well-conditioned. */ - if ((ftarget_ - oldFocus) * (phase - oldPhase) > 0.0) { + if ((ftarget_ - oldFocus) * (phase - oldPhase) * cfg_.speeds[speed_].pdafGain < 0.0) { double param = phase / (phase - oldPhase); - if (-3.0 <= param && param <= 3.5) { - ftarget_ += param * (oldFocus - ftarget_); + if ((-2.5 <= param || mode_ == AfModeContinuous) && param <= 3.0) { LOG(RPiAf, Debug) << "ETBP: param=" << param; + param = std::max(param, -2.5); + ftarget_ += param * (oldFocus - ftarget_); return true; } } @@ -562,7 +563,7 @@ void Af::doAF(double contrast, double phase, double conf) else scanState_ = ScanState::Idle; scanData_.clear(); - } else if (conf >= cfg_.confEpsilon && earlyTerminationByPhase(phase)) { + } else if (conf >= cfg_.confThresh && earlyTerminationByPhase(phase)) { scanState_ = ScanState::Settle; stepCount_ = (mode_ == AfModeContinuous) ? 0 : cfg_.speeds[speed_].stepFrames; From 686f88707caa51895057f8f22a611af9f8af2c16 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:27 +0100 Subject: [PATCH 086/106] ipa: rpi: controller: Autofocus to use AWB statistics; re-trigger Analyse AWB statistics: used both for scene change detection and to detect IR lighting (when a flag is set in the tuning file). Option to suppress PDAF altogether when IR lighting is detected. Rather than being based solely on PDAF "dropout", allow a scan to be (re-)triggered whenever the scene changes and then stabilizes, based on contrast and average RGB statistics within the AF window. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 137 +++++++++++++++++++++++++++--- src/ipa/rpi/controller/rpi/af.h | 37 +++++--- 2 files changed, 149 insertions(+), 25 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index ecc0fc417..4396420a0 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -46,6 +46,8 @@ Af::SpeedDependentParams::SpeedDependentParams() : stepCoarse(1.0), stepFine(0.25), contrastRatio(0.75), + retriggerRatio(0.75), + retriggerDelay(10), pdafGain(-0.02), pdafSquelch(0.125), maxSlew(2.0), @@ -60,6 +62,7 @@ Af::CfgParams::CfgParams() confThresh(16), confClip(512), skipFrames(5), + checkForIR(false), map() { } @@ -87,6 +90,8 @@ void Af::SpeedDependentParams::read(const libcamera::YamlObject ¶ms) readNumber(stepCoarse, params, "step_coarse"); readNumber(stepFine, params, "step_fine"); readNumber(contrastRatio, params, "contrast_ratio"); + readNumber(retriggerRatio, params, "retrigger_ratio"); + readNumber(retriggerDelay, params, "retrigger_delay"); readNumber(pdafGain, params, "pdaf_gain"); readNumber(pdafSquelch, params, "pdaf_squelch"); readNumber(maxSlew, params, "max_slew"); @@ -137,6 +142,7 @@ int Af::CfgParams::read(const libcamera::YamlObject ¶ms) readNumber(confThresh, params, "conf_thresh"); readNumber(confClip, params, "conf_clip"); readNumber(skipFrames, params, "skip_frames"); + readNumber(checkForIR, params, "check_for_ir"); if (params.contains("map")) map = params["map"].get(ipa::Pwl{}); @@ -176,29 +182,37 @@ Af::Af(Controller *controller) useWindows_(false), phaseWeights_(), contrastWeights_(), + awbWeights_(), scanState_(ScanState::Idle), initted_(false), + irFlag_(false), ftarget_(-1.0), fsmooth_(-1.0), prevContrast_(0.0), + oldSceneContrast_(0.0), + prevAverage_{ 0.0, 0.0, 0.0 }, + oldSceneAverage_{ 0.0, 0.0, 0.0 }, prevPhase_(0.0), skipCount_(0), stepCount_(0), dropCount_(0), sameSignCount_(0), + sceneChangeCount_(0), scanMaxContrast_(0.0), scanMinContrast_(1.0e9), scanData_(), reportState_(AfState::Idle) { /* - * Reserve space for data, to reduce memory fragmentation. It's too early - * to query the size of the PDAF (from camera) and Contrast (from ISP) - * statistics, but these are plausible upper bounds. + * Reserve space for data structures, to reduce memory fragmentation. + * It's too early to query the size of the PDAF sensor data, so guess. */ + windows_.reserve(1); phaseWeights_.w.reserve(16 * 12); contrastWeights_.w.reserve(getHardwareConfig().focusRegions.width * getHardwareConfig().focusRegions.height); + contrastWeights_.w.reserve(getHardwareConfig().awbRegions.width * + getHardwareConfig().awbRegions.height); scanData_.reserve(32); } @@ -309,6 +323,7 @@ void Af::invalidateWeights() { phaseWeights_.sum = 0; contrastWeights_.sum = 0; + awbWeights_.sum = 0; } bool Af::getPhase(PdafRegions const ®ions, double &phase, double &conf) @@ -365,6 +380,54 @@ double Af::getContrast(const FocusRegions &focusStats) return (contrastWeights_.sum > 0) ? ((double)sumWc / (double)contrastWeights_.sum) : 0.0; } +/* + * Get the average R, G, B values in AF window[s] (from AWB statistics). + * Optionally, check if all of {R,G,B} are within 4:5 of each other + * across more than 50% of the counted area and within the AF window: + * for an RGB sensor this strongly suggests that IR lighting is in use. + */ + +bool Af::getAverageAndTestIr(const RgbyRegions &awbStats, double rgb[3]) +{ + libcamera::Size size = awbStats.size(); + if (size.height != awbWeights_.rows || + size.width != awbWeights_.cols || awbWeights_.sum == 0) { + LOG(RPiAf, Debug) << "Recompute RGB weights " << size.width << 'x' << size.height; + computeWeights(&awbWeights_, size.height, size.width); + } + + uint64_t sr = 0, sg = 0, sb = 0, sw = 1; + uint64_t greyCount = 0, allCount = 0; + for (unsigned i = 0; i < awbStats.numRegions(); ++i) { + uint64_t r = awbStats.get(i).val.rSum; + uint64_t g = awbStats.get(i).val.gSum; + uint64_t b = awbStats.get(i).val.bSum; + uint64_t w = awbWeights_.w[i]; + if (w) { + sw += w; + sr += w * r; + sg += w * g; + sb += w * b; + } + if (cfg_.checkForIR) { + if (4 * r < 5 * b && 4 * b < 5 * r && + 4 * r < 5 * g && 4 * g < 5 * r && + 4 * b < 5 * g && 4 * g < 5 * b) + greyCount += awbStats.get(i).counted; + allCount += awbStats.get(i).counted; + } + } + + rgb[0] = sr / (double)sw; + rgb[1] = sg / (double)sw; + rgb[2] = sb / (double)sw; + + return (cfg_.checkForIR && 2 * greyCount > allCount && + 4 * sr < 5 * sb && 4 * sb < 5 * sr && + 4 * sr < 5 * sg && 4 * sg < 5 * sr && + 4 * sb < 5 * sg && 4 * sg < 5 * sb); +} + void Af::doPDAF(double phase, double conf) { /* Apply loop gain */ @@ -473,6 +536,8 @@ void Af::doScan(double contrast, double phase, double conf) if (scanData_.empty() || contrast > scanMaxContrast_) { scanMaxContrast_ = contrast; scanMaxIndex_ = scanData_.size(); + if (scanState_ != ScanState::Fine) + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); } if (contrast < scanMinContrast_) scanMinContrast_ = contrast; @@ -523,27 +588,63 @@ void Af::doAF(double contrast, double phase, double conf) sameSignCount_++; prevPhase_ = phase; + if (mode_ == AfModeManual) + return; /* nothing to do */ + if (scanState_ == ScanState::Pdaf) { /* * Use PDAF closed-loop control whenever available, in both CAF * mode and (for a limited number of iterations) when triggered. - * If PDAF fails (due to poor contrast, noise or large defocus), - * fall back to a CDAF-based scan. To avoid "nuisance" scans, - * scan only after a number of frames with low PDAF confidence. + * If PDAF fails (due to poor contrast, noise or large defocus) + * for at least dropoutFrames, fall back to a CDAF-based scan + * immediately (in triggered-auto) or on scene change (in CAF). */ - if (conf > (dropCount_ ? 1.0 : 0.25) * cfg_.confEpsilon) { + if (conf >= cfg_.confEpsilon) { if (mode_ == AfModeAuto || sameSignCount_ >= 3) doPDAF(phase, conf); if (stepCount_ > 0) stepCount_--; else if (mode_ != AfModeContinuous) scanState_ = ScanState::Idle; + oldSceneContrast_ = contrast; + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); + sceneChangeCount_ = 0; dropCount_ = 0; - } else if (++dropCount_ == cfg_.speeds[speed_].dropoutFrames) + return; + } else { + dropCount_++; + if (dropCount_ < cfg_.speeds[speed_].dropoutFrames) + return; + if (mode_ != AfModeContinuous) { + startProgrammedScan(); + return; + } + /* else fall through to waiting for a scene change */ + } + } + if (scanState_ < ScanState::Coarse && mode_ == AfModeContinuous) { + /* + * In CAF mode, not in a scan, and PDAF is unavailable. + * Wait for a scene change, followed by stability. + */ + if (contrast + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneContrast_ || + oldSceneContrast_ + 1.0 < cfg_.speeds[speed_].retriggerRatio * contrast || + prevAverage_[0] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[0] || + oldSceneAverage_[0] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[0] || + prevAverage_[1] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[1] || + oldSceneAverage_[1] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[1] || + prevAverage_[2] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[2] || + oldSceneAverage_[2] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[2]) { + oldSceneContrast_ = contrast; + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); + sceneChangeCount_ = 1; + } else if (sceneChangeCount_) + sceneChangeCount_++; + if (sceneChangeCount_ >= cfg_.speeds[speed_].retriggerDelay) startProgrammedScan(); } else if (scanState_ >= ScanState::Coarse && fsmooth_ == ftarget_) { /* - * Scanning sequence. This means PDAF has become unavailable. + * CDAF-based scanning sequence. * Allow a delay between steps for CDAF FoM statistics to be * updated, and a "settling time" at the end of the sequence. * [A coarse or fine scan can be abandoned if two PDAF samples @@ -562,11 +663,14 @@ void Af::doAF(double contrast, double phase, double conf) scanState_ = ScanState::Pdaf; else scanState_ = ScanState::Idle; + dropCount_ = 0; + sceneChangeCount_ = 0; + oldSceneContrast_ = std::max(scanMaxContrast_, prevContrast_); scanData_.clear(); } else if (conf >= cfg_.confThresh && earlyTerminationByPhase(phase)) { + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); scanState_ = ScanState::Settle; - stepCount_ = (mode_ == AfModeContinuous) ? 0 - : cfg_.speeds[speed_].stepFrames; + stepCount_ = (mode_ == AfModeContinuous) ? 0 : cfg_.speeds[speed_].stepFrames; } else doScan(contrast, phase, conf); } @@ -596,7 +700,8 @@ void Af::updateLensPosition() void Af::startAF() { /* Use PDAF if the tuning file allows it; else CDAF. */ - if (cfg_.speeds[speed_].dropoutFrames > 0 && + if (cfg_.speeds[speed_].pdafGain != 0.0 && + cfg_.speeds[speed_].dropoutFrames > 0 && (mode_ == AfModeContinuous || cfg_.speeds[speed_].pdafFrames > 0)) { if (!initted_) { ftarget_ = cfg_.ranges[range_].focusDefault; @@ -606,6 +711,8 @@ void Af::startAF() scanState_ = ScanState::Pdaf; scanData_.clear(); dropCount_ = 0; + oldSceneContrast_ = 0.0; + sceneChangeCount_ = 0; reportState_ = AfState::Scanning; } else startProgrammedScan(); @@ -656,7 +763,7 @@ void Af::prepare(Metadata *imageMetadata) uint32_t oldSt = stepCount_; if (imageMetadata->get("pdaf.regions", regions) == 0) getPhase(regions, phase, conf); - doAF(prevContrast_, phase, conf); + doAF(prevContrast_, phase, irFlag_ ? 0 : conf); updateLensPosition(); LOG(RPiAf, Debug) << std::fixed << std::setprecision(2) << static_cast(reportState_) @@ -666,7 +773,8 @@ void Af::prepare(Metadata *imageMetadata) << " ft" << oldFt << "->" << ftarget_ << " fs" << oldFs << "->" << fsmooth_ << " cont=" << (int)prevContrast_ - << " phase=" << (int)phase << " conf=" << (int)conf; + << " phase=" << (int)phase << " conf=" << (int)conf + << (irFlag_ ? " IR" : ""); } /* Report status and produce new lens setting */ @@ -690,6 +798,7 @@ void Af::process(StatisticsPtr &stats, [[maybe_unused]] Metadata *imageMetadata) { (void)imageMetadata; prevContrast_ = getContrast(stats->focusRegions); + irFlag_ = getAverageAndTestIr(stats->awbRegions, prevAverage_); } /* Controls */ diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index b06a3a16f..e1700f998 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -15,20 +15,28 @@ /* * This algorithm implements a hybrid of CDAF and PDAF, favouring PDAF. * - * Whenever PDAF is available, it is used in a continuous feedback loop. - * When triggered in auto mode, we simply enable AF for a limited number - * of frames (it may terminate early if the delta becomes small enough). + * Whenever PDAF is available (and reports sufficiently high confidence), + * it is used for continuous feedback control of the lens position. When + * triggered in Auto mode, we enable the loop for a limited number of frames + * (it may terminate sooner if the phase becomes small). In CAF mode, the + * PDAF loop runs continuously. Very small lens movements are suppressed. * * When PDAF confidence is low (due e.g. to low contrast or extreme defocus) * or PDAF data are absent, fall back to CDAF with a programmed scan pattern. - * A coarse and fine scan are performed, using ISP's CDAF focus FoM to - * estimate the lens position with peak contrast. This is slower due to - * extra latency in the ISP, and requires a settling time between steps. + * A coarse and fine scan are performed, using the ISP's CDAF contrast FoM + * to estimate the lens position with peak contrast. (This is slower due to + * extra latency in the ISP, and requires a settling time between steps.) + * The scan may terminate early if PDAF recovers and allows the zero-phase + * lens position to be interpolated. * - * Some hysteresis is applied to the switch between PDAF and CDAF, to avoid - * "nuisance" scans. During each interval where PDAF is not working, only - * ONE scan will be performed; CAF cannot track objects using CDAF alone. + * In CAF mode, the fallback to a CDAF scan is triggered when PDAF fails to + * report high confidence and a configurable number of frames have elapsed + * since the last image change since either PDAF was working or a previous + * scan found peak contrast. Image changes are detected using both contrast + * and AWB statistics (within the AF window[s]). * + * IR lighting can interfere with the correct operation of PDAF, so we + * optionally try to detect it (from AWB statistics). */ namespace RPiController { @@ -85,6 +93,8 @@ private: double stepCoarse; /* used for scans */ double stepFine; /* used for scans */ double contrastRatio; /* used for scan termination and reporting */ + double retriggerRatio; /* contrast and RGB ratio for re-triggering */ + uint32_t retriggerDelay; /* frames of stability before re-triggering */ double pdafGain; /* coefficient for PDAF feedback loop */ double pdafSquelch; /* PDAF stability parameter (device-specific) */ double maxSlew; /* limit for lens movement per frame */ @@ -103,6 +113,7 @@ private: uint32_t confThresh; /* PDAF confidence cell min (sensor-specific) */ uint32_t confClip; /* PDAF confidence cell max (sensor-specific) */ uint32_t skipFrames; /* frames to skip at start or modeswitch */ + bool checkForIR; /* Set this if PDAF is unreliable in IR light */ libcamera::ipa::Pwl map; /* converts dioptres -> lens driver position */ CfgParams(); @@ -131,6 +142,7 @@ private: void invalidateWeights(); bool getPhase(PdafRegions const ®ions, double &phase, double &conf); double getContrast(const FocusRegions &focusStats); + bool getAverageAndTestIr(const RgbyRegions &awbStats, double rgb[3]); void doPDAF(double phase, double conf); bool earlyTerminationByPhase(double phase); double findPeak(unsigned index) const; @@ -152,15 +164,18 @@ private: bool useWindows_; RegionWeights phaseWeights_; RegionWeights contrastWeights_; + RegionWeights awbWeights_; /* Working state. */ ScanState scanState_; - bool initted_; + bool initted_, irFlag_; double ftarget_, fsmooth_; - double prevContrast_; + double prevContrast_, oldSceneContrast_; + double prevAverage_[3], oldSceneAverage_[3]; double prevPhase_; unsigned skipCount_, stepCount_, dropCount_; unsigned sameSignCount_; + unsigned sceneChangeCount_; unsigned scanMaxIndex_; double scanMaxContrast_, scanMinContrast_; std::vector scanData_; From ea5f451c5660621cf1787e25c73983cec244d782 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:28 +0100 Subject: [PATCH 087/106] ipa: rpi: controller: AutoFocus bidirectional scanning To reduce unnecessary lens movements, allow the CDAF-based search procedure to start from either end of the range; or if not near an end, from the current lens position. This sometimes requires a second coarse scan, if the first one started in the middle and did not find peak contrast. Shorten the fine scan from 5 steps to 3 steps; allow fine scan to be omitted altogether when "step_fine": 0 in the tuning file. Move updateLensPosition() out of startProgrammedScan() to avoid calling it more than once per iteration. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 97 ++++++++++++++++++++----------- src/ipa/rpi/controller/rpi/af.h | 9 +-- 2 files changed, 67 insertions(+), 39 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 4396420a0..eaaca4bc9 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -200,6 +200,7 @@ Af::Af(Controller *controller) sceneChangeCount_(0), scanMaxContrast_(0.0), scanMinContrast_(1.0e9), + scanStep_(0.0), scanData_(), reportState_(AfState::Idle) { @@ -251,13 +252,14 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met << statsRegion_.height; invalidateWeights(); - if (scanState_ >= ScanState::Coarse && scanState_ < ScanState::Settle) { + if (scanState_ >= ScanState::Coarse1 && scanState_ < ScanState::Settle) { /* * If a scan was in progress, re-start it, as CDAF statistics * may have changed. Though if the application is just about * to take a still picture, this will not help... */ startProgrammedScan(); + updateLensPosition(); } skipCount_ = cfg_.skipFrames; } @@ -543,31 +545,42 @@ void Af::doScan(double contrast, double phase, double conf) scanMinContrast_ = contrast; scanData_.emplace_back(ScanRecord{ ftarget_, contrast, phase, conf }); - if (scanState_ == ScanState::Coarse) { - if (ftarget_ >= cfg_.ranges[range_].focusMax || - contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { - /* - * Finished course scan, or termination based on contrast. - * Jump to just after max contrast and start fine scan. - */ - ftarget_ = std::min(ftarget_, findPeak(scanMaxIndex_) + - 2.0 * cfg_.speeds[speed_].stepFine); - scanState_ = ScanState::Fine; - scanData_.clear(); - } else - ftarget_ += cfg_.speeds[speed_].stepCoarse; - } else { /* ScanState::Fine */ - if (ftarget_ <= cfg_.ranges[range_].focusMin || scanData_.size() >= 5 || - contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { - /* - * Finished fine scan, or termination based on contrast. - * Use quadratic peak-finding to find best contrast position. - */ - ftarget_ = findPeak(scanMaxIndex_); + if ((scanStep_ >= 0.0 && ftarget_ >= cfg_.ranges[range_].focusMax) || + (scanStep_ <= 0.0 && ftarget_ <= cfg_.ranges[range_].focusMin) || + (scanState_ == ScanState::Fine && scanData_.size() >= 3) || + contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + double pk = findPeak(scanMaxIndex_); + /* + * Finished a scan, by hitting a limit or due to constrast dropping off. + * If this is a first coarse scan and we didn't bracket the peak, reverse! + * If this is a fine scan, or no fine step was defined, we've finished. + * Otherwise, start fine scan in opposite direction. + */ + if (scanState_ == ScanState::Coarse1 && + scanData_[0].contrast >= cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + scanStep_ = -scanStep_; + scanState_ = ScanState::Coarse2; + } else if (scanState_ == ScanState::Fine || cfg_.speeds[speed_].stepFine <= 0.0) { + ftarget_ = pk; scanState_ = ScanState::Settle; - } else - ftarget_ -= cfg_.speeds[speed_].stepFine; - } + } else if (scanState_ == ScanState::Coarse1 && + scanData_[0].contrast >= cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + scanStep_ = -scanStep_; + scanState_ = ScanState::Coarse2; + } else if (scanStep_ >= 0.0) { + ftarget_ = std::min(pk + cfg_.speeds[speed_].stepFine, + cfg_.ranges[range_].focusMax); + scanStep_ = -cfg_.speeds[speed_].stepFine; + scanState_ = ScanState::Fine; + } else { + ftarget_ = std::max(pk - cfg_.speeds[speed_].stepFine, + cfg_.ranges[range_].focusMin); + scanStep_ = cfg_.speeds[speed_].stepFine; + scanState_ = ScanState::Fine; + } + scanData_.clear(); + } else + ftarget_ += scanStep_; stepCount_ = (ftarget_ == fsmooth_) ? 0 : cfg_.speeds[speed_].stepFrames; } @@ -622,7 +635,7 @@ void Af::doAF(double contrast, double phase, double conf) /* else fall through to waiting for a scene change */ } } - if (scanState_ < ScanState::Coarse && mode_ == AfModeContinuous) { + if (scanState_ < ScanState::Coarse1 && mode_ == AfModeContinuous) { /* * In CAF mode, not in a scan, and PDAF is unavailable. * Wait for a scene change, followed by stability. @@ -642,7 +655,7 @@ void Af::doAF(double contrast, double phase, double conf) sceneChangeCount_++; if (sceneChangeCount_ >= cfg_.speeds[speed_].retriggerDelay) startProgrammedScan(); - } else if (scanState_ >= ScanState::Coarse && fsmooth_ == ftarget_) { + } else if (scanState_ >= ScanState::Coarse1 && fsmooth_ == ftarget_) { /* * CDAF-based scanning sequence. * Allow a delay between steps for CDAF FoM statistics to be @@ -714,15 +727,27 @@ void Af::startAF() oldSceneContrast_ = 0.0; sceneChangeCount_ = 0; reportState_ = AfState::Scanning; - } else + } else { startProgrammedScan(); + updateLensPosition(); + } } void Af::startProgrammedScan() { - ftarget_ = cfg_.ranges[range_].focusMin; - updateLensPosition(); - scanState_ = ScanState::Coarse; + if (!initted_ || mode_ != AfModeContinuous || + fsmooth_ <= cfg_.ranges[range_].focusMin + 2.0 * cfg_.speeds[speed_].stepCoarse) { + ftarget_ = cfg_.ranges[range_].focusMin; + scanStep_ = cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse2; + } else if (fsmooth_ >= cfg_.ranges[range_].focusMax - 2.0 * cfg_.speeds[speed_].stepCoarse) { + ftarget_ = cfg_.ranges[range_].focusMax; + scanStep_ = -cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse2; + } else { + scanStep_ = -cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse1; + } scanMaxContrast_ = 0.0; scanMinContrast_ = 1.0e9; scanMaxIndex_ = 0; @@ -785,7 +810,9 @@ void Af::prepare(Metadata *imageMetadata) else status.pauseState = AfPauseState::Running; - if (mode_ == AfModeAuto && scanState_ != ScanState::Idle) + if (scanState_ == ScanState::Idle) + status.state = AfState::Idle; + else if (mode_ == AfModeAuto) status.state = AfState::Scanning; else status.state = reportState_; @@ -907,7 +934,7 @@ void Af::setMode(AfAlgorithm::AfMode mode) pauseFlag_ = false; if (mode == AfModeContinuous) scanState_ = ScanState::Trigger; - else if (mode != AfModeAuto || scanState_ < ScanState::Coarse) + else if (mode != AfModeAuto || scanState_ < ScanState::Coarse1) goIdle(); } } @@ -923,11 +950,11 @@ void Af::pause(AfAlgorithm::AfPause pause) if (mode_ == AfModeContinuous) { if (pause == AfPauseResume && pauseFlag_) { pauseFlag_ = false; - if (scanState_ < ScanState::Coarse) + if (scanState_ < ScanState::Coarse1) scanState_ = ScanState::Trigger; } else if (pause != AfPauseResume && !pauseFlag_) { pauseFlag_ = true; - if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse) + if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse1) goIdle(); } } diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index e1700f998..d35a39d12 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -75,7 +75,8 @@ private: Idle = 0, Trigger, Pdaf, - Coarse, + Coarse1, + Coarse2, Fine, Settle }; @@ -90,8 +91,8 @@ private: }; struct SpeedDependentParams { - double stepCoarse; /* used for scans */ - double stepFine; /* used for scans */ + double stepCoarse; /* in dioptres; used for scans */ + double stepFine; /* in dioptres; used for scans */ double contrastRatio; /* used for scan termination and reporting */ double retriggerRatio; /* contrast and RGB ratio for re-triggering */ uint32_t retriggerDelay; /* frames of stability before re-triggering */ @@ -177,7 +178,7 @@ private: unsigned sameSignCount_; unsigned sceneChangeCount_; unsigned scanMaxIndex_; - double scanMaxContrast_, scanMinContrast_; + double scanMaxContrast_, scanMinContrast_, scanStep_; std::vector scanData_; AfState reportState_; }; From 619da07f733c767b3e3ce1d8c400533275fc94bb Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Fri, 20 Jun 2025 13:42:29 +0100 Subject: [PATCH 088/106] ipa: rpi: Update IMX708 camera tuning files for AutoFocus changes Explicitly add new parameters: "retrigger_ratio", "retrigger_delay", "check_for_ir". Tweak other parameters to suit algorithm changes. (Though existing tuning files should still work acceptably.) Add AfSpeedFast parameters for the Raspberry Pi V3 standard lens. Signed-off-by: Nick Hollinghurst Signed-off-by: Naushir Patuck Reviewed-by: Naushir Patuck Signed-off-by: Kieran Bingham --- src/ipa/rpi/pisp/data/imx708.json | 23 ++++++++++++++++++--- src/ipa/rpi/pisp/data/imx708_noir.json | 23 ++++++++++++++++++--- src/ipa/rpi/pisp/data/imx708_wide.json | 19 ++++++++++------- src/ipa/rpi/pisp/data/imx708_wide_noir.json | 19 ++++++++++------- src/ipa/rpi/vc4/data/imx708.json | 23 ++++++++++++++++++--- src/ipa/rpi/vc4/data/imx708_noir.json | 23 ++++++++++++++++++--- src/ipa/rpi/vc4/data/imx708_wide.json | 19 ++++++++++------- src/ipa/rpi/vc4/data/imx708_wide_noir.json | 19 ++++++++++------- 8 files changed, 128 insertions(+), 40 deletions(-) diff --git a/src/ipa/rpi/pisp/data/imx708.json b/src/ipa/rpi/pisp/data/imx708.json index e8d25c216..7f2e78655 100644 --- a/src/ipa/rpi/pisp/data/imx708.json +++ b/src/ipa/rpi/pisp/data/imx708.json @@ -1139,11 +1139,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1151,6 +1167,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -1267,4 +1284,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_noir.json b/src/ipa/rpi/pisp/data/imx708_noir.json index e69afb0c6..c5e6a2652 100644 --- a/src/ipa/rpi/pisp/data/imx708_noir.json +++ b/src/ipa/rpi/pisp/data/imx708_noir.json @@ -1156,11 +1156,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1168,6 +1184,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -1230,4 +1247,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_wide.json b/src/ipa/rpi/pisp/data/imx708_wide.json index 9fff05d93..8550cdfc1 100644 --- a/src/ipa/rpi/pisp/data/imx708_wide.json +++ b/src/ipa/rpi/pisp/data/imx708_wide.json @@ -1148,23 +1148,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1172,6 +1176,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -1290,4 +1295,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_wide_noir.json b/src/ipa/rpi/pisp/data/imx708_wide_noir.json index 75d1149b6..069a06180 100644 --- a/src/ipa/rpi/pisp/data/imx708_wide_noir.json +++ b/src/ipa/rpi/pisp/data/imx708_wide_noir.json @@ -1057,23 +1057,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1081,6 +1085,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -1145,4 +1150,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708.json b/src/ipa/rpi/vc4/data/imx708.json index 4de6f0796..e54ceff49 100644 --- a/src/ipa/rpi/vc4/data/imx708.json +++ b/src/ipa/rpi/vc4/data/imx708.json @@ -638,11 +638,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -650,6 +666,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -668,4 +685,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_noir.json b/src/ipa/rpi/vc4/data/imx708_noir.json index 7b7ee874f..f351a1800 100644 --- a/src/ipa/rpi/vc4/data/imx708_noir.json +++ b/src/ipa/rpi/vc4/data/imx708_noir.json @@ -737,11 +737,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -749,6 +765,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -767,4 +784,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_wide.json b/src/ipa/rpi/vc4/data/imx708_wide.json index 6f45aafc0..bf1b122cd 100644 --- a/src/ipa/rpi/vc4/data/imx708_wide.json +++ b/src/ipa/rpi/vc4/data/imx708_wide.json @@ -637,23 +637,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -661,6 +665,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -679,4 +684,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_wide_noir.json b/src/ipa/rpi/vc4/data/imx708_wide_noir.json index b9a5227e1..ea1c8c690 100644 --- a/src/ipa/rpi/vc4/data/imx708_wide_noir.json +++ b/src/ipa/rpi/vc4/data/imx708_wide_noir.json @@ -628,23 +628,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -652,6 +656,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -670,4 +675,4 @@ } } ] -} \ No newline at end of file +} From 17eed522e8ff835fd96158aed40818346636032b Mon Sep 17 00:00:00 2001 From: Christian Rauch Date: Mon, 7 Jul 2025 20:37:46 +0200 Subject: [PATCH 089/106] subprojects: libpisp: Update to 1.2.1 Update the libpisp wrap to use the latest 1.2.1 release which silences an 'unused-parameter' warning. Bug: https://github.com/raspberrypi/libpisp/pull/43 Reviewed-by: Naushir Patuck Signed-off-by: Christian Rauch Acked-by: Kieran Bingham Acked-by: Laurent Pinchart Signed-off-by: Laurent Pinchart --- subprojects/libpisp.wrap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subprojects/libpisp.wrap b/subprojects/libpisp.wrap index 8b62c036f..0e0c7baaf 100644 --- a/subprojects/libpisp.wrap +++ b/subprojects/libpisp.wrap @@ -2,5 +2,5 @@ [wrap-git] url = https://github.com/raspberrypi/libpisp.git -revision = v1.2.0 +revision = v1.2.1 depth = 1 From 525325440b5047ac94ce96a33e03f233b8b3c59f Mon Sep 17 00:00:00 2001 From: Harvey Yang Date: Mon, 9 Dec 2024 16:34:07 +0000 Subject: [PATCH 090/106] V4L2VideoDevice: Call FrameBuffer::Private::cancel() in streamOff() At the moment `V4L2VideoDevice::streamOff()` sets `FrameBuffer::Private`'s metadata directly, while that's equivalent to calling `FrameBuffer::Private::cancel()`. To ease code tracing, this patch replace the manual modification with the function call. Signed-off-by: Harvey Yang Reviewed-by: Kieran Bingham Reviewed-by: Umang Jain Signed-off-by: Kieran Bingham --- src/libcamera/v4l2_videodevice.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp index d53aa2d3c..7822bf1ef 100644 --- a/src/libcamera/v4l2_videodevice.cpp +++ b/src/libcamera/v4l2_videodevice.cpp @@ -2031,10 +2031,9 @@ int V4L2VideoDevice::streamOff() /* Send back all queued buffers. */ for (auto it : queuedBuffers_) { FrameBuffer *buffer = it.second; - FrameMetadata &metadata = buffer->_d()->metadata(); cache_->put(it.first); - metadata.status = FrameMetadata::FrameCancelled; + buffer->_d()->cancel(); bufferReady.emit(buffer); } From e6fb24ffdb3efb66829a8b0a8ce5628e4552ee30 Mon Sep 17 00:00:00 2001 From: Nick Hollinghurst Date: Tue, 8 Jul 2025 10:22:39 +0100 Subject: [PATCH 091/106] ipa: rpi: Fix bug in AfState reporting A previous change introduced a bug in which it reported AfStateIdle when idle in Auto mode, when it should continue to report the most recent AF cycle's outcome (AfStateFocused or AfStateFailed). Also fix the Pause method so it won't reset state to AfStateIdle when paused in Continuous AF mode (to match documented behaviour). Fixes: ea5f451c5660 ("ipa: rpi: controller: AutoFocus bidirectional scanning") Signed-off-by: Nick Hollinghurst Reviewed-by: David Plowman Tested-by: David Plowman Signed-off-by: Kieran Bingham --- src/ipa/rpi/controller/rpi/af.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index eaaca4bc9..26e599303 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -810,10 +810,10 @@ void Af::prepare(Metadata *imageMetadata) else status.pauseState = AfPauseState::Running; - if (scanState_ == ScanState::Idle) - status.state = AfState::Idle; - else if (mode_ == AfModeAuto) + if (mode_ == AfModeAuto && scanState_ != ScanState::Idle) status.state = AfState::Scanning; + else if (mode_ == AfModeManual) + status.state = AfState::Idle; else status.state = reportState_; status.lensSetting = initted_ ? std::optional(cfg_.map.eval(fsmooth_)) @@ -954,8 +954,10 @@ void Af::pause(AfAlgorithm::AfPause pause) scanState_ = ScanState::Trigger; } else if (pause != AfPauseResume && !pauseFlag_) { pauseFlag_ = true; - if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse1) - goIdle(); + if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse1) { + scanState_ = ScanState::Idle; + scanData_.clear(); + } } } } From a4372127534dd9093a0ec4c3e12465ae800257e6 Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Tue, 8 Jul 2025 09:49:14 +0100 Subject: [PATCH 092/106] libcamera: controls: Remove hyphenation in control description text Remove the hyphenation in "micro-seconds" in the description for the ExposureTime control to match the rest of the document. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Kieran Bingham Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_core.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index 46f51436c..566e15337 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -212,7 +212,7 @@ controls: description: | Exposure time for the frame applied in the sensor device. - This value is specified in micro-seconds. + This value is specified in microseconds. This control will only take effect if ExposureTimeMode is Manual. If this control is set when ExposureTimeMode is Auto, the value will be From 29a88d85b730baed52a2f2e5fde2a927474ce41c Mon Sep 17 00:00:00 2001 From: Naushir Patuck Date: Tue, 8 Jul 2025 09:49:15 +0100 Subject: [PATCH 093/106] libcamera: controls: Use nanoseconds units for FrameWallClock Use nanoseconds for the FrameWallClock control to match the units for other timestamp controls, including SensorTimestamp. Update the RPi pipeline handlers to match the new nanoseconds units when converting from SensorTimestamp to FrameWallClock. Signed-off-by: Naushir Patuck Reviewed-by: David Plowman Reviewed-by: Kieran Bingham Signed-off-by: Kieran Bingham --- src/libcamera/control_ids_core.yaml | 3 ++- src/libcamera/pipeline/rpi/pisp/pisp.cpp | 2 +- src/libcamera/pipeline/rpi/vc4/vc4.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index 566e15337..eec4b4f93 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -1274,7 +1274,8 @@ controls: description: | This timestamp corresponds to the same moment in time as the SensorTimestamp, but is represented as a wall clock time as measured by - the CLOCK_REALTIME clock. + the CLOCK_REALTIME clock. Like SensorTimestamp, the timestamp value is + expressed in nanoseconds. Being a wall clock measurement, it can be used to synchronise timing across different devices. diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index 2df91bacf..92b9070c1 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -1760,7 +1760,7 @@ void PiSPCameraData::cfeBufferDequeue(FrameBuffer *buffer) */ wallClockRecovery_.addSample(); uint64_t sensorTimestamp = buffer->metadata().timestamp; - uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp / 1000); + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp); ctrl.set(controls::SensorTimestamp, sensorTimestamp); ctrl.set(controls::FrameWallClock, wallClockTimestamp); diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index e99a7edf8..5cadef527 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -778,7 +778,7 @@ void Vc4CameraData::unicamBufferDequeue(FrameBuffer *buffer) */ wallClockRecovery_.addSample(); uint64_t sensorTimestamp = buffer->metadata().timestamp; - uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp / 1000); + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp); ctrl.set(controls::SensorTimestamp, sensorTimestamp); ctrl.set(controls::FrameWallClock, wallClockTimestamp); From fb7208397533b73b2b8e3bada936a83161235a4a Mon Sep 17 00:00:00 2001 From: Umang Jain Date: Tue, 1 Jul 2025 15:38:02 +0530 Subject: [PATCH 094/106] camera: Fix spell error Correct 'CameraConfigutation' spell error to 'CameraConfiguration'. Signed-off-by: Umang Jain Reviewed-by: Jacopo Mondi Reviewed-by: Kieran Bingham Signed-off-by: Kieran Bingham --- src/libcamera/camera.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/camera.cpp b/src/libcamera/camera.cpp index c180a5fdd..b4f9d2433 100644 --- a/src/libcamera/camera.cpp +++ b/src/libcamera/camera.cpp @@ -488,7 +488,7 @@ std::size_t CameraConfiguration::size() const * * \return A CameraConfiguration::Status value that describes the validation * status. - * \retval CameraConfigutation::Adjusted The configuration has been adjusted + * \retval CameraConfiguration::Adjusted The configuration has been adjusted * and is now valid. The color space of some or all of the streams may have * been changed. The caller shall check the color spaces carefully. * \retval CameraConfiguration::Valid The configuration was already valid and From afd9890b7b9a65d95ef3295f52e01e0ec9ce7b3f Mon Sep 17 00:00:00 2001 From: Laurent Pinchart Date: Mon, 24 Feb 2025 03:19:34 +0200 Subject: [PATCH 095/106] libcamera: delayed_controls: Inherit from Object class A second use-after-free bug related to signals staying connected after the receiver DelayedControls instance gets deleted has been found, this time in the simple pipeline handler. Fix the issue once and for all by making the DelayedControls class inherit from Object. This will disconnect signals automatically upon deletion of the receiver. Signed-off-by: Laurent Pinchart Tested-by: Stanislaw Gruszka Tested-by: Isaac Scott Reviewed-by: Isaac Scott Reviewed-by: Kieran Bingham Signed-off-by: Kieran Bingham --- include/libcamera/internal/delayed_controls.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/libcamera/internal/delayed_controls.h b/include/libcamera/internal/delayed_controls.h index e8d3014d9..b64d8bba7 100644 --- a/include/libcamera/internal/delayed_controls.h +++ b/include/libcamera/internal/delayed_controls.h @@ -10,13 +10,15 @@ #include #include +#include + #include namespace libcamera { class V4L2Device; -class DelayedControls +class DelayedControls : public Object { public: struct ControlParams { From 98921d93d06c7c6e805f6cafe8efee2dd2721793 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 5 Apr 2025 14:59:17 +0300 Subject: [PATCH 096/106] gitignore: ignore my setup Signed-off-by: Vasiliy Doylov --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 51d314408..656f622f0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ *.pyc __pycache__/ venv/ +.vscode/ +.cache/ +compile_commands.json From f7bf4c8d4f2bade02df875c311caaa36d21b9e3d Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 7 Jun 2025 02:22:30 +0300 Subject: [PATCH 097/106] CI: Add local forgejo CI Signed-off-by: Vasiliy Doylov --- .forgejo/workflows/build-alpine.yaml | 58 +++++++++ .forgejo/workflows/sync-with-upstream.yaml | 18 +++ package/alpine/APKBUILD | 132 +++++++++++++++++++++ 3 files changed, 208 insertions(+) create mode 100644 .forgejo/workflows/build-alpine.yaml create mode 100644 .forgejo/workflows/sync-with-upstream.yaml create mode 100644 package/alpine/APKBUILD diff --git a/.forgejo/workflows/build-alpine.yaml b/.forgejo/workflows/build-alpine.yaml new file mode 100644 index 000000000..9fc5ff5ea --- /dev/null +++ b/.forgejo/workflows/build-alpine.yaml @@ -0,0 +1,58 @@ +name: PostmarketOS Build +run-name: PostmarketOS Build +on: + push: + workflow_dispatch: + +jobs: + prepare: + name: Prepare + runs-on: Pmbootstrap + outputs: + time: ${{ steps.time.outputs.time }} + steps: + - name: Set start Time + id: time + shell: sh + run: echo time=$(date +"%Y%m%d%H%M%S") >> $GITHUB_OUTPUT + - name: Update pmbootstrap + uses: actions/pmbootstrap-update@master + - name: Remove libcamera aport + run: rm -rf ${{env.PMB_PMAPORTS}}/temp/libcamera + + build: + name: Build for ${{ matrix.info.arch }} + runs-on: Pmbootstrap + strategy: + matrix: + info: + - arch: x86_64 + - arch: aarch64 + needs: prepare + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Build packages + id: build + uses: actions/pmbootstrap-build@main + with: + name: libcamera + aports: ${{github.workspace}}/package/alpine + arch: ${{ matrix.info.arch }} + src: ${{github.workspace}} + time: ${{ needs.prepare.outputs.time }} + - name: "Upload packages" + uses: actions/upload-alpine-package@main + with: + files: ${{steps.build.outputs.packages}} + secret: ${{secrets.PACKAGE_TOKEN}} + + clean: + name: "Clean" + runs-on: Pmbootstrap + needs: build + if: always() + continue-on-error: true + steps: + - name: Update pmbootstrap + uses: actions/pmbootstrap-update@master diff --git a/.forgejo/workflows/sync-with-upstream.yaml b/.forgejo/workflows/sync-with-upstream.yaml new file mode 100644 index 000000000..808cbaee8 --- /dev/null +++ b/.forgejo/workflows/sync-with-upstream.yaml @@ -0,0 +1,18 @@ +name: Sync fork with upstream +run-name: Sync fork with upstream +on: + schedule: + - cron: "@daily" + workflow_dispatch: + +jobs: + sync: + name: Sync + runs-on: Misc + steps: + - name: Sync repository with upstream + uses: actions/sync-with-mirror@main + with: + secret: ${{ secrets.PUSH_TOKEN }} + name: libcamera + branch: master diff --git a/package/alpine/APKBUILD b/package/alpine/APKBUILD new file mode 100644 index 000000000..ddfc72950 --- /dev/null +++ b/package/alpine/APKBUILD @@ -0,0 +1,132 @@ +pkgname=libcamera +pkgver=9999999 +pkgrel=0 +pkgdesc="Linux camera framework" +url="https://libcamera.org/" +arch="all" +license="LGPL-2.1-or-later AND GPL-2.0-or-later" +depends_dev=" + eudev-dev + glib-dev + gnutls-dev + gst-plugins-bad-dev + qt6-qtbase-dev + " + +makedepends="$depends_dev + coreutils + doxygen + graphviz + gtest-dev + libevent-dev + libpisp-dev + libunwind-dev + libyuv-dev + linux-headers + meson + py3-jinja2 + py3-ply + py3-sphinx + py3-yaml + qt6-qttools-dev + yaml-dev + " +subpackages=" + $pkgname-dbg + $pkgname-dev + $pkgname-doc + qcam + $pkgname-gstreamer + $pkgname-v4l2 + $pkgname-tools + " +source="" + +builddir="$srcdir/$pkgname-v$_pkgver" +# gstreamer tests fail +# manual strip because ipa .sign files depend on the file contents- have to re-sign after strip +options="!strip !check" + +case "$CARCH" in +arm*|aarch64) + subpackages="$subpackages $pkgname-raspberrypi" + ;; +esac + +case "$CARCH" in +ppc64le|s390x|riscv64|loongarch64) + # doesn't install any ipa + ;; +*) + # WIP: HACK? Don't depend on this this shit + # depends="$pkgname-ipa=$pkgver-r$pkgrel" + subpackages="$subpackages $pkgname-ipa" + ;; +esac + +build() { + abuild-meson \ + -Dtest=false \ + -Dv4l2=true \ + -Dwerror=false \ + . output + meson compile -C output +} + +package() { + DESTDIR="$pkgdir" meson install --no-rebuild -C output + + # manual strip first.. + scanelf --recursive \ + --nobanner \ + --etype "ET_DYN,ET_EXEC" \ + --format "%F" \ + "$pkgdir" \ + | while read -r file; do + strip "$file" + done +} + +ipa() { + depends="" + amove usr/lib/libcamera + # then sign ipa's + local ipa + for ipa in "$subpkgdir"/usr/lib/libcamera/ipa/ipa*.so; do + msg "signing $ipa" + "$builddir"/src/ipa/ipa-sign.sh \ + "$(find "$builddir"/output -type f -iname "*ipa-priv-key.pem")" \ + "$ipa" \ + "$ipa".sign + done +} + +qcam() { + depends="" + amove usr/bin/qcam +} + +gstreamer() { + depends="" + amove usr/lib/gstreamer-1.0 +} + +v4l2() { + depends="" + amove usr/libexec/libcamera/v4l2-compat.so +} + +raspberrypi() { + depends="" + amove usr/share/libcamera/ipa/rpi + amove usr/libexec/libcamera/raspberrypi_ipa_proxy + amove usr/share/libcamera/pipeline/rpi/vc4 +} + +tools() { + depends="" + amove usr/bin/cam + amove usr/bin/lc-compliance +} + +sha512sums="" From e3b71632540cf30b8d542c7e754327703c7e5a92 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sun, 23 Feb 2025 23:03:48 +0300 Subject: [PATCH 098/106] libcamera: software_isp: Add brightness control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 29 +++++++++++++++++++++++++++++ src/ipa/simple/algorithms/agc.h | 7 +++++++ src/ipa/simple/ipa_context.h | 2 ++ 3 files changed, 38 insertions(+) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index c46bb0ebe..249caa264 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -41,6 +41,33 @@ Agc::Agc() { } +int Agc::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + return 0; +} + +int Agc::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.brightness = std::optional(); + + return 0; +} + +void Agc::queueRequest(typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &brightness = controls.get(controls::Brightness); + if (brightness.has_value()) { + context.activeState.knobs.brightness = brightness; + LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); + } +} + void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) { /* @@ -54,6 +81,8 @@ void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, dou double next; int32_t &exposure = frameContext.sensor.exposure; double &again = frameContext.sensor.gain; + const auto brightness = context.activeState.knobs.brightness.value_or(1.0); + exposureMSV /= brightness; if (exposureMSV < kExposureOptimal - kExposureSatisfactory) { next = exposure * kExpNumeratorUp / kExpDenominator; diff --git a/src/ipa/simple/algorithms/agc.h b/src/ipa/simple/algorithms/agc.h index 112d9f5a1..00bc101e7 100644 --- a/src/ipa/simple/algorithms/agc.h +++ b/src/ipa/simple/algorithms/agc.h @@ -19,6 +19,13 @@ public: Agc(); ~Agc() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; void process(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const SwIspStats *stats, diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index a471b80ae..afc28ba22 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -64,6 +64,8 @@ struct IPAActiveState { /* 0..2 range, 1.0 = normal */ std::optional contrast; std::optional saturation; + /* 0..2 range, 1.0 = normal */ + std::optional brightness; } knobs; }; From 30e17d48c71230ca41eeafacbd99df5cd6831b09 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sun, 16 Mar 2025 22:12:10 +0300 Subject: [PATCH 099/106] libcamera: software_isp: Add AGC disable control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 11 +++++++++++ src/ipa/simple/ipa_context.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index 249caa264..de20863d8 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -44,6 +44,7 @@ Agc::Agc() int Agc::init(IPAContext &context, [[maybe_unused]] const YamlObject &tuningData) { + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); return 0; } @@ -52,6 +53,7 @@ int Agc::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { context.activeState.knobs.brightness = std::optional(); + context.activeState.knobs.ae_enabled = std::optional(); return 0; } @@ -62,10 +64,15 @@ void Agc::queueRequest(typename Module::Context &context, const ControlList &controls) { const auto &brightness = controls.get(controls::Brightness); + const auto &ae_enabled = controls.get(controls::AeEnable); if (brightness.has_value()) { context.activeState.knobs.brightness = brightness; LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); } + if (ae_enabled.has_value()) { + context.activeState.knobs.ae_enabled = ae_enabled; + LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); + } } void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) @@ -132,6 +139,10 @@ void Agc::process(IPAContext &context, const SwIspStats *stats, ControlList &metadata) { + const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); + if (!ae_enable) + return; + utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index afc28ba22..8e8add4c3 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -66,6 +66,8 @@ struct IPAActiveState { std::optional saturation; /* 0..2 range, 1.0 = normal */ std::optional brightness; + /* 0..1 range, 1 = normal */ + std::optional ae_enabled; } knobs; }; From 7d195335897e991a750c9a09229363f30b2a5100 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Mon, 17 Mar 2025 04:24:56 +0300 Subject: [PATCH 100/106] libcamera: software_isp: Add focus control Signed-off-by: Vasiliy Doylov --- .../internal/software_isp/software_isp.h | 4 +- include/libcamera/ipa/soft.mojom | 3 +- src/ipa/simple/algorithms/af.cpp | 71 +++++++++++++++++++ src/ipa/simple/algorithms/af.h | 40 +++++++++++ src/ipa/simple/algorithms/meson.build | 1 + src/ipa/simple/data/uncalibrated.yaml | 1 + src/ipa/simple/ipa_context.h | 9 +++ src/ipa/simple/soft_simple.cpp | 18 ++++- src/libcamera/pipeline/simple/simple.cpp | 30 ++++++-- src/libcamera/software_isp/software_isp.cpp | 4 +- 10 files changed, 169 insertions(+), 12 deletions(-) create mode 100644 src/ipa/simple/algorithms/af.cpp create mode 100644 src/ipa/simple/algorithms/af.h diff --git a/include/libcamera/internal/software_isp/software_isp.h b/include/libcamera/internal/software_isp/software_isp.h index 786246592..8f3a6b1b1 100644 --- a/include/libcamera/internal/software_isp/software_isp.h +++ b/include/libcamera/internal/software_isp/software_isp.h @@ -86,11 +86,11 @@ public: Signal outputBufferReady; Signal ispStatsReady; Signal metadataReady; - Signal setSensorControls; + Signal setSensorControls; private: void saveIspParams(); - void setSensorCtrls(const ControlList &sensorControls); + void setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls); void statsReady(uint32_t frame, uint32_t bufferId); void inputReady(FrameBuffer *input); void outputReady(FrameBuffer *output); diff --git a/include/libcamera/ipa/soft.mojom b/include/libcamera/ipa/soft.mojom index 77328c5fd..e5767532c 100644 --- a/include/libcamera/ipa/soft.mojom +++ b/include/libcamera/ipa/soft.mojom @@ -10,6 +10,7 @@ import "include/libcamera/ipa/core.mojom"; struct IPAConfigInfo { libcamera.ControlInfoMap sensorControls; + libcamera.ControlInfoMap lensControls; }; interface IPASoftInterface { @@ -32,7 +33,7 @@ interface IPASoftInterface { }; interface IPASoftEventInterface { - setSensorControls(libcamera.ControlList sensorControls); + setSensorControls(libcamera.ControlList sensorControls, libcamera.ControlList lensControls); setIspParams(); metadataReady(uint32 frame, libcamera.ControlList metadata); }; diff --git a/src/ipa/simple/algorithms/af.cpp b/src/ipa/simple/algorithms/af.cpp new file mode 100644 index 000000000..b51ed95e4 --- /dev/null +++ b/src/ipa/simple/algorithms/af.cpp @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#include "af.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftAutoFocus) + +namespace ipa::soft::algorithms { + +Af::Af() +{ +} + +int Af::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::LensPosition] = ControlInfo(0.0f, 100.0f, 50.0f); + return 0; +} + +int Af::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.focus_pos = std::optional(); + + return 0; +} + +void Af::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &focus_pos = controls.get(controls::LensPosition); + if (focus_pos.has_value()) { + context.activeState.knobs.focus_pos = focus_pos; + LOG(IPASoftAutoFocus, Debug) << "Setting focus position to " << focus_pos.value(); + } +} + +void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAFrameContext &frameContext, [[maybe_unused]] double exposureMSV) +{ + frameContext.lens.focus_pos = context.activeState.knobs.focus_pos.value_or(50.0) / 100.0 * (context.configuration.focus.focus_max - context.configuration.focus.focus_min); +} + +void Af::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const SwIspStats *stats, + [[maybe_unused]] ControlList &metadata) +{ + updateFocus(context, frameContext, 0); +} + +REGISTER_IPA_ALGORITHM(Af, "Af") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/af.h b/src/ipa/simple/algorithms/af.h new file mode 100644 index 000000000..a575ef102 --- /dev/null +++ b/src/ipa/simple/algorithms/af.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Af : public Algorithm +{ +public: + Af(); + ~Af() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateFocus(IPAContext &context, IPAFrameContext &frameContext, double focus); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build index 2d0adb059..dec59ee8c 100644 --- a/src/ipa/simple/algorithms/meson.build +++ b/src/ipa/simple/algorithms/meson.build @@ -6,4 +6,5 @@ soft_simple_ipa_algorithms = files([ 'blc.cpp', 'ccm.cpp', 'lut.cpp', + 'af.cpp', ]) diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index 5508e6686..d14b34294 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -16,4 +16,5 @@ algorithms: # 0, 0, 1] - Lut: - Agc: + - Af: ... diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index 8e8add4c3..71b9bb637 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -34,6 +34,9 @@ struct IPASessionConfiguration { struct { std::optional level; } black; + struct { + int32_t focus_min, focus_max; + } focus; }; struct IPAActiveState { @@ -68,6 +71,8 @@ struct IPAActiveState { std::optional brightness; /* 0..1 range, 1 = normal */ std::optional ae_enabled; + /* 0..100 range, 50.0 = normal */ + std::optional focus_pos; } knobs; }; @@ -81,6 +86,10 @@ struct IPAFrameContext : public FrameContext { double gain; } sensor; + struct { + int32_t focus_pos; + } lens; + struct { double red; double blue; diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp index c94c4cd55..53fa74c03 100644 --- a/src/ipa/simple/soft_simple.cpp +++ b/src/ipa/simple/soft_simple.cpp @@ -77,6 +77,7 @@ private: SwIspStats *stats_; std::unique_ptr camHelper_; ControlInfoMap sensorInfoMap_; + ControlInfoMap lensInfoMap_; /* Local parameter storage */ struct IPAContext context_; @@ -196,6 +197,7 @@ int IPASoftSimple::init(const IPASettings &settings, int IPASoftSimple::configure(const IPAConfigInfo &configInfo) { sensorInfoMap_ = configInfo.sensorControls; + lensInfoMap_ = configInfo.lensControls; const ControlInfo &exposureInfo = sensorInfoMap_.find(V4L2_CID_EXPOSURE)->second; const ControlInfo &gainInfo = sensorInfoMap_.find(V4L2_CID_ANALOGUE_GAIN)->second; @@ -205,6 +207,17 @@ int IPASoftSimple::configure(const IPAConfigInfo &configInfo) context_.activeState = {}; context_.frameContexts.clear(); + if (lensInfoMap_.empty()) { + LOG(IPASoft, Warning) << "No camera leans found! Focus control disabled."; + context_.configuration.focus.focus_min = 0; + context_.configuration.focus.focus_max = 0; + } else { + const ControlInfo &lensInfo = lensInfoMap_.find(V4L2_CID_FOCUS_ABSOLUTE)->second; + context_.configuration.focus.focus_min = lensInfo.min().get(); + context_.configuration.focus.focus_max = lensInfo.max().get(); + LOG(IPASoft, Warning) << "Camera leans found! Focus: " << context_.configuration.focus.focus_min << "-" << context_.configuration.focus.focus_max; + } + context_.configuration.agc.lineDuration = context_.sensorInfo.minLineLength * 1.0s / context_.sensorInfo.pixelRate; context_.configuration.agc.exposureMin = exposureInfo.min().get(); @@ -327,7 +340,10 @@ void IPASoftSimple::processStats(const uint32_t frame, ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(camHelper_ ? camHelper_->gainCode(againNew) : againNew)); - setSensorControls.emit(ctrls); + ControlList lens_ctrls(lensInfoMap_); + lens_ctrls.set(V4L2_CID_FOCUS_ABSOLUTE, frameContext.lens.focus_pos); + + setSensorControls.emit(ctrls, lens_ctrls); } std::string IPASoftSimple::logPrefix() const diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp index efb07051b..4c002ca68 100644 --- a/src/libcamera/pipeline/simple/simple.cpp +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -30,6 +30,7 @@ #include #include "libcamera/internal/camera.h" +#include "libcamera/internal/camera_lens.h" #include "libcamera/internal/camera_sensor.h" #include "libcamera/internal/camera_sensor_properties.h" #include "libcamera/internal/converter.h" @@ -41,6 +42,8 @@ #include "libcamera/internal/v4l2_subdevice.h" #include "libcamera/internal/v4l2_videodevice.h" +#include "libcamera/controls.h" + namespace libcamera { LOG_DEFINE_CATEGORY(SimplePipeline) @@ -356,7 +359,7 @@ private: void ispStatsReady(uint32_t frame, uint32_t bufferId); void metadataReady(uint32_t frame, const ControlList &metadata); - void setSensorControls(const ControlList &sensorControls); + void setSensorControls(const ControlList &sensorControls, const ControlList &lensControls); }; class SimpleCameraConfiguration : public CameraConfiguration @@ -1002,7 +1005,7 @@ void SimpleCameraData::metadataReady(uint32_t frame, const ControlList &metadata tryCompleteRequest(info->request); } -void SimpleCameraData::setSensorControls(const ControlList &sensorControls) +void SimpleCameraData::setSensorControls(const ControlList &sensorControls, const ControlList &lensControls) { delayedCtrls_->push(sensorControls); /* @@ -1013,10 +1016,21 @@ void SimpleCameraData::setSensorControls(const ControlList &sensorControls) * but it also bypasses delayedCtrls_, creating AGC regulation issues. * Both problems should be fixed. */ - if (!frameStartEmitter_) { - ControlList ctrls(sensorControls); - sensor_->setControls(&ctrls); - } + if (frameStartEmitter_) + return; + + ControlList ctrls(sensorControls); + sensor_->setControls(&ctrls); + + CameraLens *focusLens = sensor_->focusLens(); + if (!focusLens) + return; + + if (!lensControls.contains(V4L2_CID_FOCUS_ABSOLUTE)) + return; + + const ControlValue &focusValue = lensControls.get(V4L2_CID_FOCUS_ABSOLUTE); + focusLens->setFocusPosition(focusValue.get()); } /* Retrieve all source pads connected to a sink pad through active routes. */ @@ -1406,6 +1420,10 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) } else { ipa::soft::IPAConfigInfo configInfo; configInfo.sensorControls = data->sensor_->controls(); + if (data->sensor_->focusLens() != nullptr) + configInfo.lensControls = data->sensor_->focusLens()->controls(); + else + configInfo.lensControls = ControlInfoMap(); return data->swIsp_->configure(inputCfg, outputCfgs, configInfo); } } diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index 28e2a360e..8f41591c8 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -395,9 +395,9 @@ void SoftwareIsp::saveIspParams() debayerParams_ = *sharedParams_; } -void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls) +void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls) { - setSensorControls.emit(sensorControls); + setSensorControls.emit(sensorControls, lensControls); } void SoftwareIsp::statsReady(uint32_t frame, uint32_t bufferId) From d0bf6e7f8862e3e8e17bb08d998f0ad706e809e6 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 21 May 2025 21:24:09 +0300 Subject: [PATCH 101/106] libcamera: software_isp: Add manual exposure control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 14 ++++++++++++-- src/ipa/simple/ipa_context.h | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index de20863d8..35f4798cf 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -46,6 +46,8 @@ int Agc::init(IPAContext &context, { context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + context.ctrlMap[&controls::ExposureValue] = ControlInfo(0.0f, 0.5f, 1.0f); + return 0; } @@ -65,6 +67,7 @@ void Agc::queueRequest(typename Module::Context &context, { const auto &brightness = controls.get(controls::Brightness); const auto &ae_enabled = controls.get(controls::AeEnable); + const auto &exposure_value = controls.get(controls::ExposureValue); if (brightness.has_value()) { context.activeState.knobs.brightness = brightness; LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); @@ -73,6 +76,10 @@ void Agc::queueRequest(typename Module::Context &context, context.activeState.knobs.ae_enabled = ae_enabled; LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); } + if (exposure_value.has_value()) { + context.activeState.knobs.exposure_value = exposure_value.value(); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << exposure_value.value(); + } } void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) @@ -140,13 +147,16 @@ void Agc::process(IPAContext &context, ControlList &metadata) { const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); - if (!ae_enable) - return; + if (!ae_enable) + frameContext.sensor.exposure = (int32_t)( context.activeState.knobs.exposure_value.value_or(0.5) * (context.configuration.agc.exposureMax - context.configuration.agc.exposureMin)); utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << frameContext.sensor.exposure; + if (!ae_enable) + return; /* * Calculate Mean Sample Value (MSV) according to formula from: diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index 71b9bb637..c5b5527b5 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -71,6 +71,8 @@ struct IPAActiveState { std::optional brightness; /* 0..1 range, 1 = normal */ std::optional ae_enabled; + /* 0..1 range, 0.5 = normal */ + std::optional exposure_value; /* 0..100 range, 50.0 = normal */ std::optional focus_pos; } knobs; From 6e8d4d86d75624ec9a84bc5084238ef14428460e Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 21 May 2025 22:24:01 +0300 Subject: [PATCH 102/106] libcamera: software_isp: Add control to disable statistic collection Signed-off-by: Vasiliy Doylov --- .../internal/software_isp/debayer_params.h | 9 +++ src/ipa/simple/algorithms/meson.build | 1 + src/ipa/simple/algorithms/stat.cpp | 65 +++++++++++++++++++ src/ipa/simple/algorithms/stat.h | 38 +++++++++++ src/ipa/simple/data/uncalibrated.yaml | 1 + src/ipa/simple/ipa_context.h | 2 + src/libcamera/software_isp/debayer_cpu.cpp | 12 ++-- src/libcamera/software_isp/debayer_cpu.h | 1 + 8 files changed, 123 insertions(+), 6 deletions(-) create mode 100644 src/ipa/simple/algorithms/stat.cpp create mode 100644 src/ipa/simple/algorithms/stat.h diff --git a/include/libcamera/internal/software_isp/debayer_params.h b/include/libcamera/internal/software_isp/debayer_params.h index 217cd5d92..6c36defc9 100644 --- a/include/libcamera/internal/software_isp/debayer_params.h +++ b/include/libcamera/internal/software_isp/debayer_params.h @@ -49,6 +49,15 @@ struct DebayerParams { CcmLookupTable greenCcm; CcmLookupTable blueCcm; LookupTable gammaLut; + + /* + * Statistic controls + * + * Statistic collecting are very slow. We can disable it for some actions like + * video capture or streaming. + * TODO: Add statistic window control + */ + bool collect_stats; }; } /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build index dec59ee8c..e5666b262 100644 --- a/src/ipa/simple/algorithms/meson.build +++ b/src/ipa/simple/algorithms/meson.build @@ -7,4 +7,5 @@ soft_simple_ipa_algorithms = files([ 'ccm.cpp', 'lut.cpp', 'af.cpp', + 'stat.cpp', ]) diff --git a/src/ipa/simple/algorithms/stat.cpp b/src/ipa/simple/algorithms/stat.cpp new file mode 100644 index 000000000..181a5d818 --- /dev/null +++ b/src/ipa/simple/algorithms/stat.cpp @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Debayer statistic controls + */ + +#include "stat.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftStatistic) + +namespace ipa::soft::algorithms { + +Stat::Stat() +{ +} + +int Stat::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::DebugMetadataEnable] = ControlInfo(false, true, true); + return 0; +} + +int Stat::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.stats_enabled = std::optional(); + + return 0; +} + +void Stat::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &stats_enabled = controls.get(controls::DebugMetadataEnable); + if (stats_enabled.has_value()) { + context.activeState.knobs.stats_enabled = stats_enabled; + LOG(IPASoftStatistic, Debug) << "Setting debayer enabled to " << stats_enabled.value(); + } +} + +void Stat::prepare([[maybe_unused]]IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]]IPAFrameContext &frameContext, + [[maybe_unused]] DebayerParams *params) +{ + params->collect_stats = context.activeState.knobs.stats_enabled.value_or(true); +} + +REGISTER_IPA_ALGORITHM(Stat, "Stat") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/stat.h b/src/ipa/simple/algorithms/stat.h new file mode 100644 index 000000000..dc0051cb0 --- /dev/null +++ b/src/ipa/simple/algorithms/stat.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Debayer statistic controls + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Stat : public Algorithm +{ +public: + Stat(); + ~Stat() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void prepare(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + DebayerParams *params) override; + +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index d14b34294..47eaf9c61 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -17,4 +17,5 @@ algorithms: - Lut: - Agc: - Af: + - Stat: ... diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index c5b5527b5..cfc524a13 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -75,6 +75,8 @@ struct IPAActiveState { std::optional exposure_value; /* 0..100 range, 50.0 = normal */ std::optional focus_pos; + /* 0..1 range, 1 = normal */ + std::optional stats_enabled; } knobs; }; diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 66f6038c1..c6f070225 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -668,7 +668,7 @@ void DebayerCpu::process2(const uint8_t *src, uint8_t *dst) for (unsigned int y = window_.y; y < yEnd; y += 2) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(y, linePointers); + if (this->enable_statistic) stats_->processLine0(y, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -683,7 +683,7 @@ void DebayerCpu::process2(const uint8_t *src, uint8_t *dst) if (window_.y == 0) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(yEnd, linePointers); + if (this->enable_statistic) stats_->processLine0(yEnd, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -720,7 +720,7 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) for (unsigned int y = window_.y; y < yEnd; y += 4) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(y, linePointers); + if (this->enable_statistic) stats_->processLine0(y, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -733,7 +733,7 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine2(y, linePointers); + if (this->enable_statistic) stats_->processLine2(y, linePointers); (this->*debayer2_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -771,7 +771,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output for (const FrameBuffer::Plane &plane : output->planes()) dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); - + enable_statistic = params.collect_stats; green_ = params.green; greenCcm_ = params.greenCcm; if (swapRedBlueGains_) { @@ -805,7 +805,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output return; } - stats_->startFrame(); + if(this->enable_statistic) stats_->startFrame(); if (inputConfig_.patternSize.height == 2) process2(in.planes()[0].data(), out.planes()[0].data()); diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 926195e98..54304053f 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -165,6 +165,7 @@ private: /* Skip 30 frames for things to stabilize then measure 30 frames */ static constexpr unsigned int kFramesToSkip = 30; static constexpr unsigned int kLastFrameToMeasure = 60; + bool enable_statistic = true; }; } /* namespace libcamera */ From 7fe3e610cdac6adb0de1740e1ddd1c64fd471caf Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 21 Jun 2025 01:05:32 +0300 Subject: [PATCH 103/106] HACK: WIP: Clean queued request before assertion :D Signed-off-by: Vasiliy Doylov --- src/libcamera/pipeline_handler.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index d84dff3c9..31d501a13 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -372,6 +372,8 @@ void PipelineHandler::stop(Camera *camera) /* Make sure no requests are pending. */ Camera::Private *data = camera->_d(); + // WIP: Just clean for now, idk maybe something wrong with thread sync? + data->queuedRequests_.clear(); ASSERT(data->queuedRequests_.empty()); data->requestSequence_ = 0; From a106a436326d3c9e17388ddb822c8f61e93c5dd1 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 9 Jul 2025 16:07:14 +0300 Subject: [PATCH 104/106] libcamera: software_isp: Add autofocus Signed-off-by: Vasiliy Doylov --- .../internal/software_isp/swisp_stats.h | 4 ++ src/ipa/simple/algorithms/af.cpp | 62 ++++++++++++++++++- src/ipa/simple/algorithms/af.h | 1 + src/ipa/simple/ipa_context.h | 6 ++ src/libcamera/software_isp/swstats_cpu.cpp | 16 ++++- 5 files changed, 85 insertions(+), 4 deletions(-) diff --git a/include/libcamera/internal/software_isp/swisp_stats.h b/include/libcamera/internal/software_isp/swisp_stats.h index ae11f112e..3377dd825 100644 --- a/include/libcamera/internal/software_isp/swisp_stats.h +++ b/include/libcamera/internal/software_isp/swisp_stats.h @@ -44,6 +44,10 @@ struct SwIspStats { * \brief A histogram of luminance values */ Histogram yHistogram; + /** + * \brief Holds the sharpness of an image + */ + uint64_t sharpness; }; } /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/af.cpp b/src/ipa/simple/algorithms/af.cpp index b51ed95e4..52ddf7f1a 100644 --- a/src/ipa/simple/algorithms/af.cpp +++ b/src/ipa/simple/algorithms/af.cpp @@ -27,14 +27,18 @@ int Af::init(IPAContext &context, [[maybe_unused]] const YamlObject &tuningData) { context.ctrlMap[&controls::LensPosition] = ControlInfo(0.0f, 100.0f, 50.0f); + context.ctrlMap[&controls::AfTrigger] = ControlInfo(0, 1, 0); return 0; } int Af::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { + context.activeState.knobs.focus_sweep = std::optional(); context.activeState.knobs.focus_pos = std::optional(); - + context.activeState.knobs.focus_sweep = false; + context.activeState.knobs.focus_pos = 0; + context.configuration.focus.skip = 10; return 0; } @@ -44,10 +48,24 @@ void Af::queueRequest([[maybe_unused]] typename Module::Context &context, const ControlList &controls) { const auto &focus_pos = controls.get(controls::LensPosition); + const auto &af_trigger = controls.get(controls::AfTrigger); if (focus_pos.has_value()) { context.activeState.knobs.focus_pos = focus_pos; LOG(IPASoftAutoFocus, Debug) << "Setting focus position to " << focus_pos.value(); } + if (af_trigger.has_value()) { + context.activeState.knobs.focus_sweep = af_trigger.value() == 1; + if(context.activeState.knobs.focus_sweep){ + context.activeState.knobs.focus_pos = 0; + context.configuration.focus.focus_max_pos = 0; + context.configuration.focus.sharpness_max = 0; + context.configuration.focus.start = 0; + context.configuration.focus.stop = 100; + context.configuration.focus.step = 25; + LOG(IPASoftAutoFocus, Info) << "Starting focus sweep"; + } + } + } void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAFrameContext &frameContext, [[maybe_unused]] double exposureMSV) @@ -55,12 +73,54 @@ void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAF frameContext.lens.focus_pos = context.activeState.knobs.focus_pos.value_or(50.0) / 100.0 * (context.configuration.focus.focus_max - context.configuration.focus.focus_min); } +void Af::step(uint32_t& skip, double& start, double& stop, double& step, double& focus_pos, double& max_pos, uint64_t& max_sharp, uint64_t sharp, bool& sweep){ + if(!sweep) + return; + if(skip != 0){ + skip --; + return; + } + skip = 2; + if(focus_pos < start) { + focus_pos = start; + return; + } + if(sharp > max_sharp) { + max_sharp = sharp; + max_pos = focus_pos; + } + if(focus_pos >= stop) { + LOG(IPASoftAutoFocus, Info) << "Best focus on step " <sharpness, + context.activeState.knobs.focus_sweep.value()); updateFocus(context, frameContext, 0); } diff --git a/src/ipa/simple/algorithms/af.h b/src/ipa/simple/algorithms/af.h index a575ef102..901393717 100644 --- a/src/ipa/simple/algorithms/af.h +++ b/src/ipa/simple/algorithms/af.h @@ -33,6 +33,7 @@ public: private: void updateFocus(IPAContext &context, IPAFrameContext &frameContext, double focus); + void step(uint32_t& skip, double& start, double& stop, double& step, double& focus_pos, double& max_pos, uint64_t& max_sharp, uint64_t sharp, bool& sweep); }; } /* namespace ipa::soft::algorithms */ diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index cfc524a13..f59c4006f 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -36,6 +36,10 @@ struct IPASessionConfiguration { } black; struct { int32_t focus_min, focus_max; + double focus_max_pos; + uint64_t sharpness_max; + double start, stop, step; + uint32_t skip; } focus; }; @@ -77,6 +81,8 @@ struct IPAActiveState { std::optional focus_pos; /* 0..1 range, 1 = normal */ std::optional stats_enabled; + /* 0..1 range, 0 = normal */ + std::optional focus_sweep; } knobs; }; diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index c520c806e..a6a73f483 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -147,7 +147,10 @@ static constexpr unsigned int kBlueYMul = 29; /* 0.114 * 256 */ \ uint64_t sumR = 0; \ uint64_t sumG = 0; \ - uint64_t sumB = 0; + uint64_t sumB = 0; \ + pixel_t r0 = 0, r1 = 0, b0 = 0, \ + b1 = 0, g0 = 0, g1 = 0; \ + uint64_t sharpness = 0; #define SWSTATS_ACCUMULATE_LINE_STATS(div) \ sumR += r; \ @@ -157,12 +160,18 @@ static constexpr unsigned int kBlueYMul = 29; /* 0.114 * 256 */ yVal = r * kRedYMul; \ yVal += g * kGreenYMul; \ yVal += b * kBlueYMul; \ - stats_.yHistogram[yVal * SwIspStats::kYHistogramSize / (256 * 256 * (div))]++; + stats_.yHistogram[yVal * SwIspStats::kYHistogramSize / (256 * 256 * (div))]++; \ + if (r0 != 0) \ + sharpness += abs(r - 2*r1 + r0) * kRedYMul + abs(g - 2*g1 + g0) * kGreenYMul + abs(b - 2*b1 + b0) * kBlueYMul; \ + r0 = r1; g0 = g1; b0 = b1; \ + r1 = r; g1 = g; b1 = b; \ + #define SWSTATS_FINISH_LINE_STATS() \ stats_.sumR_ += sumR; \ stats_.sumG_ += sumG; \ - stats_.sumB_ += sumB; + stats_.sumB_ += sumB; \ + stats_.sharpness += sharpness; void SwStatsCpu::statsBGGR8Line0(const uint8_t *src[]) { @@ -306,6 +315,7 @@ void SwStatsCpu::startFrame(void) stats_.sumR_ = 0; stats_.sumB_ = 0; stats_.sumG_ = 0; + stats_.sharpness = 0; stats_.yHistogram.fill(0); } From f6d95130f75473c71ccc0fbd9a6d463d53afb152 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 9 Jul 2025 17:48:32 +0300 Subject: [PATCH 105/106] gstreamer: fix crash on stream stop Signed-off-by: Vasiliy Doylov --- src/gstreamer/gstlibcamerasrc.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index 3aca4eeda..da8eb4e5c 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -236,6 +236,8 @@ GstLibcameraSrcState::requestCompleted(Request *request) GLibLocker locker(&lock_); controls_.readMetadata(request); + if(queuedRequests_.empty()) + return; wrap = std::move(queuedRequests_.front()); queuedRequests_.pop(); From 2c5bda6f089cf9a8fa8a919695c33c1cf924332f Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 9 Jul 2025 18:13:29 +0300 Subject: [PATCH 106/106] gstreamer: remove dublicated property Signed-off-by: Vasiliy Doylov --- src/gstreamer/gstlibcameraprovider.cpp | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/gstreamer/gstlibcameraprovider.cpp b/src/gstreamer/gstlibcameraprovider.cpp index 5da96ea3f..08862363c 100644 --- a/src/gstreamer/gstlibcameraprovider.cpp +++ b/src/gstreamer/gstlibcameraprovider.cpp @@ -32,7 +32,7 @@ GST_DEBUG_CATEGORY_STATIC(provider_debug); */ enum { - PROP_DEVICE_NAME = 1, + PROP_DEVICE_ = 1, }; #define GST_TYPE_LIBCAMERA_DEVICE gst_libcamera_device_get_type() @@ -76,14 +76,11 @@ gst_libcamera_device_reconfigure_element(GstDevice *device, static void gst_libcamera_device_set_property(GObject *object, guint prop_id, - const GValue *value, GParamSpec *pspec) + [[maybe_unused]]const GValue *value, GParamSpec *pspec) { - GstLibcameraDevice *device = GST_LIBCAMERA_DEVICE(object); + // GstLibcameraDevice *device = GST_LIBCAMERA_DEVICE(object); switch (prop_id) { - case PROP_DEVICE_NAME: - device->name = g_value_dup_string(value); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -117,12 +114,6 @@ gst_libcamera_device_class_init(GstLibcameraDeviceClass *klass) object_class->set_property = gst_libcamera_device_set_property; object_class->finalize = gst_libcamera_device_finalize; - - GParamSpec *pspec = g_param_spec_string("name", "Name", - "The name of the camera device", "", - (GParamFlags)(G_PARAM_STATIC_STRINGS | G_PARAM_WRITABLE | - G_PARAM_CONSTRUCT_ONLY)); - g_object_class_install_property(object_class, PROP_DEVICE_NAME, pspec); } static GstDevice *