From c189fc550473e81d51bdcfc01fc99c8b06fdc647 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:13 +0200 Subject: [PATCH 01/42] libcamera: swstats_cpu: Update statsProcessFn() / processLine0() documentation Update the documentation of the statsProcessFn() / processLine0() src[] pointer argument to take into account that swstats_cpu may also be used with planar input data or with non Bayer single plane input data. The statsProcessFn typedef is private, so no documentation is generated for it. Move the new updated src[] pointer argument documentation to processLine0() so that it gets included in the generated docs. Reviewed-by: Kieran Bingham Reviewed-by: Milan Zamazal Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/swstats_cpu.cpp | 27 +++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index c520c806e..a9a3e77a8 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -71,6 +71,19 @@ namespace libcamera { * patternSize height == 1. * It'll process line 0 and 1 for input formats with patternSize height >= 2. * This function may only be called after a successful setWindow() call. + * + * This function takes an array of src pointers each pointing to a line in + * the source image. + * + * Bayer input data requires (patternSize_.height + 1) src pointers, with + * the middle element of the array pointing to the actual line being processed. + * Earlier element(s) will point to the previous line(s) and later element(s) + * to the next line(s). See the DebayerCpu::debayerFn documentation for details. + * + * Planar input data requires a src pointer for each plane, with src[0] pointing + * to the line in plane 0, etc. + * + * For non Bayer single plane input data only a single src pointer is required. */ /** @@ -89,20 +102,6 @@ namespace libcamera { * \brief Signals that the statistics are ready */ -/** - * \typedef SwStatsCpu::statsProcessFn - * \brief Called when there is data to get statistics from - * \param[in] src The input data - * - * These functions take an array of (patternSize_.height + 1) src - * pointers each pointing to a line in the source image. The middle - * element of the array will point to the actual line being processed. - * Earlier element(s) will point to the previous line(s) and later - * element(s) to the next line(s). - * - * See the documentation of DebayerCpu::debayerFn for more details. - */ - /** * \var unsigned int SwStatsCpu::ySkipMask_ * \brief Skip lines where this bitmask is set in y From 26dba2e048695eab1d11186e7921f9d79e553683 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:14 +0200 Subject: [PATCH 02/42] libcamera: swstats_cpu: Drop patternSize_ documentation patternSize_ is a private variable and its meaning is already documented in the patternSize() getter documentation. Move the list of valid sizes to the patternSize() getter documentation and drop the patternSize_ documentation. While at it also add 1x1 as valid size for use with future support of single plane non Bayer input data. Reviewed-by: Kieran Bingham Reviewed-by: Milan Zamazal Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/swstats_cpu.cpp | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index a9a3e77a8..5e4246a9c 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -58,6 +58,8 @@ namespace libcamera { * also indicates if processLine2() should be called or not. * This may only be called after a successful configure() call. * + * Valid sizes are: 1x1, 2x2, 4x2 or 4x4. + * * \return The pattern size */ @@ -112,13 +114,6 @@ namespace libcamera { * \brief Statistics window, set by setWindow(), used every line */ -/** - * \var Size SwStatsCpu::patternSize_ - * \brief The size of the bayer pattern - * - * Valid sizes are: 2x2, 4x2 or 4x4. - */ - /** * \var unsigned int SwStatsCpu::xShift_ * \brief The offset of x, applied to window_.x for bayer variants From d9ffeb0bf11436adcb609e3549926eb5921bd38c Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:15 +0200 Subject: [PATCH 03/42] libcamera: swstats_cpu: Move header to libcamera/internal/software_isp Move the swstats_cpu.h file to include/libcamera/internal/software_isp/ so that it can be used outside the src/libcamera/software_isp/ directory. Reviewed-by: Milan Zamazal Reviewed-by: Kieran Bingham Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/software_isp/meson.build | 1 + .../libcamera/internal}/software_isp/swstats_cpu.h | 0 src/libcamera/software_isp/debayer_cpu.h | 2 +- src/libcamera/software_isp/swstats_cpu.cpp | 2 +- 4 files changed, 3 insertions(+), 2 deletions(-) rename {src/libcamera => include/libcamera/internal}/software_isp/swstats_cpu.h (100%) diff --git a/include/libcamera/internal/software_isp/meson.build b/include/libcamera/internal/software_isp/meson.build index 508ddddca..ea3f3f1c1 100644 --- a/include/libcamera/internal/software_isp/meson.build +++ b/include/libcamera/internal/software_isp/meson.build @@ -4,4 +4,5 @@ libcamera_internal_headers += files([ 'debayer_params.h', 'software_isp.h', 'swisp_stats.h', + 'swstats_cpu.h', ]) diff --git a/src/libcamera/software_isp/swstats_cpu.h b/include/libcamera/internal/software_isp/swstats_cpu.h similarity index 100% rename from src/libcamera/software_isp/swstats_cpu.h rename to include/libcamera/internal/software_isp/swstats_cpu.h diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 926195e98..89a89893d 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -18,9 +18,9 @@ #include #include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" #include "debayer.h" -#include "swstats_cpu.h" namespace libcamera { diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index 5e4246a9c..aa5654dcc 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -9,7 +9,7 @@ * CPU based software statistics implementation */ -#include "swstats_cpu.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" #include From ba4218669b46fe8af63b96eaff09e9ae57793721 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:16 +0200 Subject: [PATCH 04/42] libcamera: software_isp: Move benchmark code to its own class Move the code for the builtin benchmark to its own small Benchmark class. Reviewed-by: Kieran Bingham Reviewed-by: Milan Zamazal Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- .../internal/software_isp/benchmark.h | 36 +++++++ .../internal/software_isp/meson.build | 1 + src/libcamera/software_isp/benchmark.cpp | 93 +++++++++++++++++++ src/libcamera/software_isp/debayer_cpu.cpp | 36 +------ src/libcamera/software_isp/debayer_cpu.h | 7 +- src/libcamera/software_isp/meson.build | 1 + 6 files changed, 135 insertions(+), 39 deletions(-) create mode 100644 include/libcamera/internal/software_isp/benchmark.h create mode 100644 src/libcamera/software_isp/benchmark.cpp diff --git a/include/libcamera/internal/software_isp/benchmark.h b/include/libcamera/internal/software_isp/benchmark.h new file mode 100644 index 000000000..8af250154 --- /dev/null +++ b/include/libcamera/internal/software_isp/benchmark.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Authors: + * Hans de Goede + * + * Simple builtin benchmark to measure software ISP processing times + */ + +#pragma once + +#include +#include + +namespace libcamera { + +class Benchmark +{ +public: + Benchmark(); + ~Benchmark(); + + void startFrame(void); + void finishFrame(void); + +private: + unsigned int measuredFrames_; + int64_t frameProcessTime_; + timespec frameStartTime_; + /* Skip 30 frames for things to stabilize then measure 30 frames */ + static constexpr unsigned int kFramesToSkip = 30; + static constexpr unsigned int kLastFrameToMeasure = 60; +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/software_isp/meson.build b/include/libcamera/internal/software_isp/meson.build index ea3f3f1c1..df7c3b97d 100644 --- a/include/libcamera/internal/software_isp/meson.build +++ b/include/libcamera/internal/software_isp/meson.build @@ -1,6 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 libcamera_internal_headers += files([ + 'benchmark.h', 'debayer_params.h', 'software_isp.h', 'swisp_stats.h', diff --git a/src/libcamera/software_isp/benchmark.cpp b/src/libcamera/software_isp/benchmark.cpp new file mode 100644 index 000000000..b3da3c416 --- /dev/null +++ b/src/libcamera/software_isp/benchmark.cpp @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Authors: + * Hans de Goede + * + * Simple builtin benchmark to measure software ISP processing times + */ + +#include "libcamera/internal/software_isp/benchmark.h" + +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(Benchmark) + +/** + * \class Benchmark + * \brief Simple builtin benchmark + * + * Simple builtin benchmark to measure software ISP processing times. + */ + +/** + * \brief Constructs a Benchmark object + */ +Benchmark::Benchmark() + : measuredFrames_(0), frameProcessTime_(0) +{ +} + +Benchmark::~Benchmark() +{ +} + +static inline int64_t timeDiff(timespec &after, timespec &before) +{ + return (after.tv_sec - before.tv_sec) * 1000000000LL + + (int64_t)after.tv_nsec - (int64_t)before.tv_nsec; +} + +/** + * \brief Start measuring process time for a single frame + * + * Call this function before processing frame data to start measuring + * the process time for a frame. + */ +void Benchmark::startFrame(void) +{ + if (measuredFrames_ >= Benchmark::kLastFrameToMeasure) + return; + + frameStartTime_ = {}; + clock_gettime(CLOCK_MONOTONIC_RAW, &frameStartTime_); +} + +/** + * \brief Finish measuring process time for a single frame + * + * Call this function after processing frame data to finish measuring + * the process time for a frame. + * + * This function will log frame processing time information after + * Benchmark::kLastFrameToMeasure frames have been processed. + */ +void Benchmark::finishFrame(void) +{ + if (measuredFrames_ >= Benchmark::kLastFrameToMeasure) + return; + + measuredFrames_++; + + if (measuredFrames_ <= Benchmark::kFramesToSkip) + return; + + timespec frameEndTime = {}; + clock_gettime(CLOCK_MONOTONIC_RAW, &frameEndTime); + frameProcessTime_ += timeDiff(frameEndTime, frameStartTime_); + + if (measuredFrames_ == Benchmark::kLastFrameToMeasure) { + const unsigned int measuredFrames = Benchmark::kLastFrameToMeasure - + Benchmark::kFramesToSkip; + LOG(Benchmark, Info) + << "Processed " << measuredFrames + << " frames in " << frameProcessTime_ / 1000 << "us, " + << frameProcessTime_ / (1000 * measuredFrames) + << " us/frame"; + } +} + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 66f6038c1..8d30bf4a2 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -554,9 +554,6 @@ int DebayerCpu::configure(const StreamConfiguration &inputCfg, lineBuffers_[i].resize(lineBufferLength_); } - measuredFrames_ = 0; - frameProcessTime_ = 0; - return 0; } @@ -746,24 +743,9 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) } } -namespace { - -inline int64_t timeDiff(timespec &after, timespec &before) -{ - return (after.tv_sec - before.tv_sec) * 1000000000LL + - (int64_t)after.tv_nsec - (int64_t)before.tv_nsec; -} - -} /* namespace */ - void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params) { - timespec frameStartTime; - - if (measuredFrames_ < DebayerCpu::kLastFrameToMeasure) { - frameStartTime = {}; - clock_gettime(CLOCK_MONOTONIC_RAW, &frameStartTime); - } + bench_.startFrame(); std::vector dmaSyncers; for (const FrameBuffer::Plane &plane : input->planes()) @@ -817,21 +799,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output dmaSyncers.clear(); /* Measure before emitting signals */ - if (measuredFrames_ < DebayerCpu::kLastFrameToMeasure && - ++measuredFrames_ > DebayerCpu::kFramesToSkip) { - timespec frameEndTime = {}; - clock_gettime(CLOCK_MONOTONIC_RAW, &frameEndTime); - frameProcessTime_ += timeDiff(frameEndTime, frameStartTime); - if (measuredFrames_ == DebayerCpu::kLastFrameToMeasure) { - const unsigned int measuredFrames = DebayerCpu::kLastFrameToMeasure - - DebayerCpu::kFramesToSkip; - LOG(Debayer, Info) - << "Processed " << measuredFrames - << " frames in " << frameProcessTime_ / 1000 << "us, " - << frameProcessTime_ / (1000 * measuredFrames) - << " us/frame"; - } - } + bench_.finishFrame(); /* * Buffer ids are currently not used, so pass zeros as its parameter. diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 89a89893d..182607cda 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -17,6 +17,7 @@ #include +#include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/bayer_format.h" #include "libcamera/internal/software_isp/swstats_cpu.h" @@ -160,11 +161,7 @@ private: unsigned int xShift_; /* Offset of 0/1 applied to window_.x */ bool enableInputMemcpy_; bool swapRedBlueGains_; - unsigned int measuredFrames_; - int64_t frameProcessTime_; - /* Skip 30 frames for things to stabilize then measure 30 frames */ - static constexpr unsigned int kFramesToSkip = 30; - static constexpr unsigned int kLastFrameToMeasure = 60; + Benchmark bench_; }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/meson.build b/src/libcamera/software_isp/meson.build index aac7eda7b..59fa5f02a 100644 --- a/src/libcamera/software_isp/meson.build +++ b/src/libcamera/software_isp/meson.build @@ -8,6 +8,7 @@ if not softisp_enabled endif libcamera_internal_sources += files([ + 'benchmark.cpp', 'debayer.cpp', 'debayer_cpu.cpp', 'software_isp.cpp', From d4bc61f716328d59094f730a69354835137078b0 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 10 May 2025 16:12:17 +0200 Subject: [PATCH 05/42] libcamera: swstats_cpu: Add processFrame() method Add a method to the SwstatsCpu class to process a whole Framebuffer in one go, rather then line by line. This is useful for gathering stats when debayering is not necessary or is not done on the CPU. Reviewed-by: Milan Zamazal Signed-off-by: Hans de Goede Signed-off-by: Bryan O'Donoghue --- .../internal/software_isp/swstats_cpu.h | 12 +++++ src/libcamera/software_isp/swstats_cpu.cpp | 51 +++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/include/libcamera/internal/software_isp/swstats_cpu.h b/include/libcamera/internal/software_isp/swstats_cpu.h index 26a2f462e..fa47cec91 100644 --- a/include/libcamera/internal/software_isp/swstats_cpu.h +++ b/include/libcamera/internal/software_isp/swstats_cpu.h @@ -18,12 +18,16 @@ #include #include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/framebuffer.h" #include "libcamera/internal/shared_mem_object.h" #include "libcamera/internal/software_isp/swisp_stats.h" +#include "benchmark.h" + namespace libcamera { class PixelFormat; +class MappedFrameBuffer; struct StreamConfiguration; class SwStatsCpu @@ -42,6 +46,7 @@ public: void setWindow(const Rectangle &window); void startFrame(); void finishFrame(uint32_t frame, uint32_t bufferId); + void processFrame(uint32_t frame, uint32_t bufferId, FrameBuffer *input); void processLine0(unsigned int y, const uint8_t *src[]) { @@ -65,6 +70,7 @@ public: private: using statsProcessFn = void (SwStatsCpu::*)(const uint8_t *src[]); + using processFrameFn = void (SwStatsCpu::*)(MappedFrameBuffer &in); int setupStandardBayerOrder(BayerFormat::Order order); /* Bayer 8 bpp unpacked */ @@ -77,6 +83,10 @@ private: void statsBGGR10PLine0(const uint8_t *src[]); void statsGBRG10PLine0(const uint8_t *src[]); + void processBayerFrame2(MappedFrameBuffer &in); + + processFrameFn processFrame_; + /* Variables set by configure(), used every line */ statsProcessFn stats0_; statsProcessFn stats2_; @@ -89,9 +99,11 @@ private: Size patternSize_; unsigned int xShift_; + unsigned int stride_; SharedMemObject sharedStats_; SwIspStats stats_; + Benchmark bench_; }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index aa5654dcc..1ff15f5b7 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -16,6 +16,7 @@ #include #include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/mapped_framebuffer.h" namespace libcamera { @@ -360,11 +361,14 @@ int SwStatsCpu::setupStandardBayerOrder(BayerFormat::Order order) */ int SwStatsCpu::configure(const StreamConfiguration &inputCfg) { + stride_ = inputCfg.stride; + BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputCfg.pixelFormat); if (bayerFormat.packing == BayerFormat::Packing::None && setupStandardBayerOrder(bayerFormat.order) == 0) { + processFrame_ = &SwStatsCpu::processBayerFrame2; switch (bayerFormat.bitDepth) { case 8: stats0_ = &SwStatsCpu::statsBGGR8Line0; @@ -385,6 +389,7 @@ int SwStatsCpu::configure(const StreamConfiguration &inputCfg) /* Skip every 3th and 4th line, sample every other 2x2 block */ ySkipMask_ = 0x02; xShift_ = 0; + processFrame_ = &SwStatsCpu::processBayerFrame2; switch (bayerFormat.order) { case BayerFormat::BGGR: @@ -425,4 +430,50 @@ void SwStatsCpu::setWindow(const Rectangle &window) window_.height &= ~(patternSize_.height - 1); } +void SwStatsCpu::processBayerFrame2(MappedFrameBuffer &in) +{ + const uint8_t *src = in.planes()[0].data(); + const uint8_t *linePointers[3]; + + /* Adjust src for starting at window_.y */ + src += window_.y * stride_; + + for (unsigned int y = 0; y < window_.height; y += 2) { + if (y & ySkipMask_) { + src += stride_ * 2; + continue; + } + + /* linePointers[0] is not used by any stats0_ functions */ + linePointers[1] = src; + linePointers[2] = src + stride_; + (this->*stats0_)(linePointers); + src += stride_ * 2; + } +} + +/** + * \brief Calculate statistics for a frame in one go + * \param[in] frame The frame number + * \param[in] bufferId ID of the statistics buffer + * \param[in] input The frame to process + * + * This may only be called after a successful setWindow() call. + */ +void SwStatsCpu::processFrame(uint32_t frame, uint32_t bufferId, FrameBuffer *input) +{ + bench_.startFrame(); + startFrame(); + + MappedFrameBuffer in(input, MappedFrameBuffer::MapFlag::Read); + if (!in.isValid()) { + LOG(SwStatsCpu, Error) << "mmap-ing buffer(s) failed"; + return; + } + + (this->*processFrame_)(in); + finishFrame(frame, bufferId); + bench_.finishFrame(); +} + } /* namespace libcamera */ From 1142f89aae2d89f6190675e5129198b187eb3f95 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 21 Sep 2024 12:41:42 +0100 Subject: [PATCH 06/42] libcamera: MappedFrameBuffer: Latch a pointer to the framebuffer Take a pointer to the given framebuffer as a private member for further use in later patches. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/mapped_framebuffer.h | 3 +++ src/libcamera/mapped_framebuffer.cpp | 2 ++ 2 files changed, 5 insertions(+) diff --git a/include/libcamera/internal/mapped_framebuffer.h b/include/libcamera/internal/mapped_framebuffer.h index 6aaabf508..75ac2c8f6 100644 --- a/include/libcamera/internal/mapped_framebuffer.h +++ b/include/libcamera/internal/mapped_framebuffer.h @@ -55,6 +55,9 @@ public: using MapFlags = Flags; MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags); + +private: + const FrameBuffer *buffer_; }; LIBCAMERA_FLAGS_ENABLE_OPERATORS(MappedFrameBuffer::MapFlag) diff --git a/src/libcamera/mapped_framebuffer.cpp b/src/libcamera/mapped_framebuffer.cpp index f54bbf21f..f5ee770c6 100644 --- a/src/libcamera/mapped_framebuffer.cpp +++ b/src/libcamera/mapped_framebuffer.cpp @@ -238,6 +238,8 @@ MappedFrameBuffer::MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags) planes_.emplace_back(info.address + plane.offset, plane.length); } + + buffer_ = buffer; } } /* namespace libcamera */ From 2929cdcc58a9730eff287a86fae0bddb3a312676 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 11 May 2025 15:16:44 +0100 Subject: [PATCH 07/42] libcamera: MappedFrameBuffer: Add MappedFrameBuffer::getPlaneFD() Add MappedFrameBuffer::getPlaneFD() which takes a plane index and returns the file descriptor associated with it. This fd will be used to feed into eglCreateImageKHR for both texture creation on upload and directly render-to-texture where the texture buffer comes from the fd given to eglCreateImageKHR. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/mapped_framebuffer.h | 1 + src/libcamera/mapped_framebuffer.cpp | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/include/libcamera/internal/mapped_framebuffer.h b/include/libcamera/internal/mapped_framebuffer.h index 75ac2c8f6..9a5355c76 100644 --- a/include/libcamera/internal/mapped_framebuffer.h +++ b/include/libcamera/internal/mapped_framebuffer.h @@ -55,6 +55,7 @@ public: using MapFlags = Flags; MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags); + int getPlaneFD(int plane); private: const FrameBuffer *buffer_; diff --git a/src/libcamera/mapped_framebuffer.cpp b/src/libcamera/mapped_framebuffer.cpp index f5ee770c6..d5f347d4d 100644 --- a/src/libcamera/mapped_framebuffer.cpp +++ b/src/libcamera/mapped_framebuffer.cpp @@ -242,4 +242,9 @@ MappedFrameBuffer::MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags) buffer_ = buffer; } +int MappedFrameBuffer::getPlaneFD(int plane) +{ + return buffer_->planes()[plane].fd.get(); +} + } /* namespace libcamera */ From 831a7fefa92d323e2f75e1d62baeecb537696280 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 19 May 2024 00:10:16 +0100 Subject: [PATCH 08/42] libcamera: software_isp: Move useful items from DebayerCpu to Debayer base class The DebayerCpu class has a number of variables, embedded structures and methods which are useful to DebayerGpu implementation. Move relevant variables and methods to base class. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.h | 33 ++++++++++++++++++++++- src/libcamera/software_isp/debayer_cpu.h | 34 +----------------------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index ba033d440..45da822a2 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -14,11 +14,13 @@ #include #include +#include #include #include #include +#include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/software_isp/debayer_params.h" namespace libcamera { @@ -27,7 +29,7 @@ class FrameBuffer; LOG_DECLARE_CATEGORY(Debayer) -class Debayer +class Debayer : public Object { public: virtual ~Debayer() = 0; @@ -45,9 +47,38 @@ public: virtual SizeRange sizes(PixelFormat inputFormat, const Size &inputSize) = 0; + virtual const SharedFD &getStatsFD() = 0; + + unsigned int frameSize() { return outputConfig_.frameSize; } + Signal inputBufferReady; Signal outputBufferReady; + struct DebayerInputConfig { + Size patternSize; + unsigned int bpp; /* Memory used per pixel, not precision */ + unsigned int stride; + std::vector outputFormats; + }; + + struct DebayerOutputConfig { + unsigned int bpp; /* Memory used per pixel, not precision */ + unsigned int stride; + unsigned int frameSize; + }; + + DebayerInputConfig inputConfig_; + DebayerOutputConfig outputConfig_; + DebayerParams::LookupTable red_; + DebayerParams::LookupTable green_; + DebayerParams::LookupTable blue_; + DebayerParams::CcmLookupTable redCcm_; + DebayerParams::CcmLookupTable greenCcm_; + DebayerParams::CcmLookupTable blueCcm_; + DebayerParams::LookupTable gammaLut_; + bool swapRedBlueGains_; + Benchmark bench_; + private: virtual Size patternSize(PixelFormat inputFormat) = 0; }; diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 182607cda..0b4b16e1a 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -17,7 +17,6 @@ #include -#include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/bayer_format.h" #include "libcamera/internal/software_isp/swstats_cpu.h" @@ -25,7 +24,7 @@ namespace libcamera { -class DebayerCpu : public Debayer, public Object +class DebayerCpu : public Debayer { public: DebayerCpu(std::unique_ptr stats); @@ -48,13 +47,6 @@ public: */ const SharedFD &getStatsFD() { return stats_->getStatsFD(); } - /** - * \brief Get the output frame size - * - * \return The output frame size - */ - unsigned int frameSize() { return outputConfig_.frameSize; } - private: /** * \brief Called to debayer 1 line of Bayer input data to output format @@ -111,19 +103,6 @@ private: template void debayer10P_RGRG_BGR888(uint8_t *dst, const uint8_t *src[]); - struct DebayerInputConfig { - Size patternSize; - unsigned int bpp; /* Memory used per pixel, not precision */ - unsigned int stride; - std::vector outputFormats; - }; - - struct DebayerOutputConfig { - unsigned int bpp; /* Memory used per pixel, not precision */ - unsigned int stride; - unsigned int frameSize; - }; - int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); int setupStandardBayerOrder(BayerFormat::Order order); @@ -139,20 +118,11 @@ private: /* Max. supported Bayer pattern height is 4, debayering this requires 5 lines */ static constexpr unsigned int kMaxLineBuffers = 5; - DebayerParams::LookupTable red_; - DebayerParams::LookupTable green_; - DebayerParams::LookupTable blue_; - DebayerParams::CcmLookupTable redCcm_; - DebayerParams::CcmLookupTable greenCcm_; - DebayerParams::CcmLookupTable blueCcm_; - DebayerParams::LookupTable gammaLut_; debayerFn debayer0_; debayerFn debayer1_; debayerFn debayer2_; debayerFn debayer3_; Rectangle window_; - DebayerInputConfig inputConfig_; - DebayerOutputConfig outputConfig_; std::unique_ptr stats_; std::vector lineBuffers_[kMaxLineBuffers]; unsigned int lineBufferLength_; @@ -160,8 +130,6 @@ private: unsigned int lineBufferIndex_; unsigned int xShift_; /* Offset of 0/1 applied to window_.x */ bool enableInputMemcpy_; - bool swapRedBlueGains_; - Benchmark bench_; }; } /* namespace libcamera */ From 3c0a21fd30a05556e05ff8166cf1f45bf2c12e6c Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 29 Apr 2025 12:49:49 +0100 Subject: [PATCH 09/42] libcamera: software_isp: Move Bayer parans init from DebayerCpu to Debayer Move the initialisation of Bayer params and CCM to a new constructor in the Debayer class. Ensure we call the base class constructor from DebayerCpu's constructor in the expected constructor order Debayer then DebayerCpu. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.cpp | 11 +++++++++++ src/libcamera/software_isp/debayer.h | 1 + src/libcamera/software_isp/debayer_cpu.cpp | 10 +--------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/libcamera/software_isp/debayer.cpp b/src/libcamera/software_isp/debayer.cpp index e9e18c488..29fdcbbfd 100644 --- a/src/libcamera/software_isp/debayer.cpp +++ b/src/libcamera/software_isp/debayer.cpp @@ -103,6 +103,17 @@ namespace libcamera { LOG_DEFINE_CATEGORY(Debayer) +Debayer::Debayer() +{ + /* Initialize color lookup tables */ + for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { + red_[i] = green_[i] = blue_[i] = i; + redCcm_[i] = { static_cast(i), 0, 0 }; + greenCcm_[i] = { 0, static_cast(i), 0 }; + blueCcm_[i] = { 0, 0, static_cast(i) }; + } +} + Debayer::~Debayer() { } diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index 45da822a2..01b6e916f 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -32,6 +32,7 @@ LOG_DECLARE_CATEGORY(Debayer) class Debayer : public Object { public: + Debayer(); virtual ~Debayer() = 0; virtual int configure(const StreamConfiguration &inputCfg, diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 8d30bf4a2..13db8a8c3 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -40,7 +40,7 @@ namespace libcamera { * \param[in] stats Pointer to the stats object to use */ DebayerCpu::DebayerCpu(std::unique_ptr stats) - : stats_(std::move(stats)) + : Debayer(), stats_(std::move(stats)) { /* * Reading from uncached buffers may be very slow. @@ -51,14 +51,6 @@ DebayerCpu::DebayerCpu(std::unique_ptr stats) * future. */ enableInputMemcpy_ = true; - - /* Initialize color lookup tables */ - for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { - red_[i] = green_[i] = blue_[i] = i; - redCcm_[i] = { static_cast(i), 0, 0 }; - greenCcm_[i] = { 0, static_cast(i), 0 }; - blueCcm_[i] = { 0, 0, static_cast(i) }; - } } DebayerCpu::~DebayerCpu() = default; From c3a8493a12f26c59d59057ee4a0bb38337ea556f Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 20 Apr 2025 01:02:25 +0100 Subject: [PATCH 10/42] libcamera: software_isp: Move param select code to Debayer base class Move the parameter selection code into the Debayer base class in-order to facilitate reuse of the lookup tables in the eGL shaders. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.cpp | 40 ++++++++++++++++++++++ src/libcamera/software_isp/debayer.h | 5 +++ src/libcamera/software_isp/debayer_cpu.cpp | 25 ++------------ 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/libcamera/software_isp/debayer.cpp b/src/libcamera/software_isp/debayer.cpp index 29fdcbbfd..75e4bffab 100644 --- a/src/libcamera/software_isp/debayer.cpp +++ b/src/libcamera/software_isp/debayer.cpp @@ -187,4 +187,44 @@ Debayer::~Debayer() * \brief Signals when the output buffer is ready */ +/** + * \fn void Debayer::setParams(DebayerParams ¶ms) + * \brief Select the bayer params to use for the next frame debayer + * \param[in] params The parameters to be used in debayering + */ +void Debayer::setParams(DebayerParams ¶ms) +{ + green_ = params.green; + greenCcm_ = params.greenCcm; + if (swapRedBlueGains_) { + red_ = params.blue; + blue_ = params.red; + redCcm_ = params.blueCcm; + blueCcm_ = params.redCcm; + for (unsigned int i = 0; i < 256; i++) { + std::swap(redCcm_[i].r, redCcm_[i].b); + std::swap(blueCcm_[i].r, blueCcm_[i].b); + } + } else { + red_ = params.red; + blue_ = params.blue; + redCcm_ = params.redCcm; + blueCcm_ = params.blueCcm; + } + gammaLut_ = params.gammaLut; +} + +/** + * \fn void Debayer::dmaSyncBegin(DebayerParams ¶ms) + * \brief Common CPU/GPU Dma Sync Buffer begin + */ +void Debayer::dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output) +{ + for (const FrameBuffer::Plane &plane : input->planes()) + dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Read); + + for (const FrameBuffer::Plane &plane : output->planes()) + dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); +} + } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index 01b6e916f..0af66b556 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -20,6 +20,7 @@ #include #include +#include "libcamera/internal/dma_buf_allocator.h" #include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/software_isp/debayer_params.h" @@ -82,6 +83,10 @@ public: private: virtual Size patternSize(PixelFormat inputFormat) = 0; + +protected: + void setParams(DebayerParams ¶ms); + void dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output); }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 13db8a8c3..4ef573a37 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -22,7 +22,6 @@ #include #include "libcamera/internal/bayer_format.h" -#include "libcamera/internal/dma_buf_allocator.h" #include "libcamera/internal/framebuffer.h" #include "libcamera/internal/mapped_framebuffer.h" @@ -740,30 +739,10 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output bench_.startFrame(); std::vector dmaSyncers; - for (const FrameBuffer::Plane &plane : input->planes()) - dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Read); - for (const FrameBuffer::Plane &plane : output->planes()) - dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); + dmaSyncBegin(dmaSyncers, input, output); - green_ = params.green; - greenCcm_ = params.greenCcm; - if (swapRedBlueGains_) { - red_ = params.blue; - blue_ = params.red; - redCcm_ = params.blueCcm; - blueCcm_ = params.redCcm; - for (unsigned int i = 0; i < 256; i++) { - std::swap(redCcm_[i].r, redCcm_[i].b); - std::swap(blueCcm_[i].r, blueCcm_[i].b); - } - } else { - red_ = params.red; - blue_ = params.blue; - redCcm_ = params.redCcm; - blueCcm_ = params.blueCcm; - } - gammaLut_ = params.gammaLut; + setParams(params); /* Copy metadata from the input buffer */ FrameMetadata &metadata = output->_d()->metadata(); From 8ca941b7b590fd44ad4e66541eb2f3e155705485 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 27 May 2025 16:02:37 +0100 Subject: [PATCH 11/42] libcamera: software_isp: Move isStandardBayerOrder to base class isStandardBayerOrder is useful to both CPU and GPU debayer logic and reusable as-is for both. Move to shared location in base class. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.cpp | 10 ++++++++++ src/libcamera/software_isp/debayer.h | 2 ++ src/libcamera/software_isp/debayer_cpu.cpp | 6 ------ src/libcamera/software_isp/debayer_cpu.h | 1 - 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/libcamera/software_isp/debayer.cpp b/src/libcamera/software_isp/debayer.cpp index 75e4bffab..d0e17d20b 100644 --- a/src/libcamera/software_isp/debayer.cpp +++ b/src/libcamera/software_isp/debayer.cpp @@ -227,4 +227,14 @@ void Debayer::dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *inpu dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); } +/** + * \fn void Debayer::isStandardBayerOrder(BayerFormat::Order order) + * \brief Common method to validate standard Bayer order + */ +bool Debayer::isStandardBayerOrder(BayerFormat::Order order) +{ + return order == BayerFormat::BGGR || order == BayerFormat::GBRG || + order == BayerFormat::GRBG || order == BayerFormat::RGGB; +} + } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index 0af66b556..214bcdd3c 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -20,6 +20,7 @@ #include #include +#include "libcamera/internal/bayer_format.h" #include "libcamera/internal/dma_buf_allocator.h" #include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/software_isp/debayer_params.h" @@ -87,6 +88,7 @@ private: protected: void setParams(DebayerParams ¶ms); void dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output); + bool isStandardBayerOrder(BayerFormat::Order order); }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 4ef573a37..e56492848 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -282,12 +282,6 @@ void DebayerCpu::debayer10P_RGRG_BGR888(uint8_t *dst, const uint8_t *src[]) } } -static bool isStandardBayerOrder(BayerFormat::Order order) -{ - return order == BayerFormat::BGGR || order == BayerFormat::GBRG || - order == BayerFormat::GRBG || order == BayerFormat::RGGB; -} - /* * Setup the Debayer object according to the passed in parameters. * Return 0 on success, a negative errno value on failure diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 0b4b16e1a..a043a9541 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -17,7 +17,6 @@ #include -#include "libcamera/internal/bayer_format.h" #include "libcamera/internal/software_isp/swstats_cpu.h" #include "debayer.h" From 87ccbc69caf22dde18668b6e442c90c50b609f5e Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Mon, 16 Dec 2024 17:03:36 +0000 Subject: [PATCH 12/42] libcamera: software_isp: Start the ISP thread in configure OpenGL is not thread-safe and in fact associates invisible handles with the threadid of the calling context. As a result we need to make configure() and process() in SoftISP execute on the same thread. Move start thread into configure() as a first step towards this. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/software_isp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index 28e2a360e..7bee8f068 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -159,8 +159,6 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor, metadataReady.emit(frame, metadata); }); ipa_->setSensorControls.connect(this, &SoftwareIsp::setSensorCtrls); - - debayer_->moveToThread(&ispWorkerThread_); } SoftwareIsp::~SoftwareIsp() @@ -262,6 +260,9 @@ int SoftwareIsp::configure(const StreamConfiguration &inputCfg, if (ret < 0) return ret; + debayer_->moveToThread(&ispWorkerThread_); + ispWorkerThread_.start(); + return debayer_->configure(inputCfg, outputCfgs, ccmEnabled_); } @@ -343,7 +344,6 @@ int SoftwareIsp::start() if (ret) return ret; - ispWorkerThread_.start(); return 0; } From 355fd41dfdb5101388598535266292de35b95b33 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 18 Dec 2024 00:59:39 +0000 Subject: [PATCH 13/42] libcamera: software_isp: Move configure to worker thread OpenGL requires both configure() and process() to operate on the same thread. As preparation for that, move current CPU configure into the WorkerThread with a ConnectionTypeBlocking invocation of &DebayerCpu::configure. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/software_isp.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index 7bee8f068..e8fa8a17a 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -263,7 +263,15 @@ int SoftwareIsp::configure(const StreamConfiguration &inputCfg, debayer_->moveToThread(&ispWorkerThread_); ispWorkerThread_.start(); - return debayer_->configure(inputCfg, outputCfgs, ccmEnabled_); + ret = debayer_->invokeMethod(&Debayer::configure, + ConnectionTypeBlocking, inputCfg, + outputCfgs, ccmEnabled_); + if (ret) { + ispWorkerThread_.exit(); + ispWorkerThread_.wait(); + } + + return ret; } /** @@ -386,7 +394,7 @@ void SoftwareIsp::stop() void SoftwareIsp::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output) { ipa_->computeParams(frame); - debayer_->invokeMethod(&DebayerCpu::process, + debayer_->invokeMethod(&Debayer::process, ConnectionTypeQueued, frame, input, output, debayerParams_); } From 3142e3761932bc340e81ff8b335af27bd0e6590f Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 22 Apr 2025 22:25:37 +0100 Subject: [PATCH 14/42] libcamera: software_isp: debayer: Make the debayer_ object of type class Debayer not DebayerCpu Make the type of object Debayer not DebayerCpu thus allowing us to assign the object to either DebayerCpu or DebayerEGL. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/software_isp/software_isp.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/libcamera/internal/software_isp/software_isp.h b/include/libcamera/internal/software_isp/software_isp.h index 786246592..ad89c9b3c 100644 --- a/include/libcamera/internal/software_isp/software_isp.h +++ b/include/libcamera/internal/software_isp/software_isp.h @@ -37,7 +37,7 @@ namespace libcamera { -class DebayerCpu; +class Debayer; class FrameBuffer; class PixelFormat; class Stream; @@ -94,8 +94,7 @@ private: void statsReady(uint32_t frame, uint32_t bufferId); void inputReady(FrameBuffer *input); void outputReady(FrameBuffer *output); - - std::unique_ptr debayer_; + std::unique_ptr debayer_; Thread ispWorkerThread_; SharedMemObject sharedParams_; DebayerParams debayerParams_; From c2a68a2e447f1d7d411ebcbc984b62305d635b03 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 9 May 2025 14:34:27 +0100 Subject: [PATCH 15/42] libcamera: software_isp: debayer: Extend DebayerParams struct to hold a copy of per-frame CCM values Add an element to the DebayerParams structure to capture the calculated CCM. This allows us to pass the CCM into the eGL Debayer shader later on. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/software_isp/debayer_params.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/libcamera/internal/software_isp/debayer_params.h b/include/libcamera/internal/software_isp/debayer_params.h index 217cd5d92..56803195e 100644 --- a/include/libcamera/internal/software_isp/debayer_params.h +++ b/include/libcamera/internal/software_isp/debayer_params.h @@ -13,6 +13,8 @@ #include #include +#include "libcamera/internal/matrix.h" + namespace libcamera { struct DebayerParams { @@ -49,6 +51,11 @@ struct DebayerParams { CcmLookupTable greenCcm; CcmLookupTable blueCcm; LookupTable gammaLut; + + /* + * Per frame CCM values as calcualted by the IPA + */ + Matrix ccm; }; } /* namespace libcamera */ From 367b29199bef1d158dfa0ce77cf8d6ef20003a86 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 24 Jan 2024 15:06:00 +0000 Subject: [PATCH 16/42] libcamera: shaders: Move GL shader programs to src/libcamera/assets/shader Moving the GL shaders to src/libcamera/assets/shader to allow for reuse of these inside of the SoftISP. Signed-off-by: Bryan O'Donoghue --- .../libcamera/internal/shaders}/RGB.frag | 0 .../internal/shaders}/YUV_2_planes.frag | 0 .../internal/shaders}/YUV_3_planes.frag | 0 .../internal/shaders}/YUV_packed.frag | 0 .../internal/shaders}/bayer_1x_packed.frag | 0 .../libcamera/internal/shaders}/bayer_8.frag | 0 .../libcamera/internal/shaders}/bayer_8.vert | 0 .../libcamera/internal/shaders}/identity.vert | 0 src/apps/qcam/assets/shader/shaders.qrc | 16 ++--- src/apps/qcam/viewfinder_gl.cpp | 70 +++++++++---------- 10 files changed, 43 insertions(+), 43 deletions(-) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/RGB.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/YUV_2_planes.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/YUV_3_planes.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/YUV_packed.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/bayer_1x_packed.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/bayer_8.frag (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/bayer_8.vert (100%) rename {src/apps/qcam/assets/shader => include/libcamera/internal/shaders}/identity.vert (100%) diff --git a/src/apps/qcam/assets/shader/RGB.frag b/include/libcamera/internal/shaders/RGB.frag similarity index 100% rename from src/apps/qcam/assets/shader/RGB.frag rename to include/libcamera/internal/shaders/RGB.frag diff --git a/src/apps/qcam/assets/shader/YUV_2_planes.frag b/include/libcamera/internal/shaders/YUV_2_planes.frag similarity index 100% rename from src/apps/qcam/assets/shader/YUV_2_planes.frag rename to include/libcamera/internal/shaders/YUV_2_planes.frag diff --git a/src/apps/qcam/assets/shader/YUV_3_planes.frag b/include/libcamera/internal/shaders/YUV_3_planes.frag similarity index 100% rename from src/apps/qcam/assets/shader/YUV_3_planes.frag rename to include/libcamera/internal/shaders/YUV_3_planes.frag diff --git a/src/apps/qcam/assets/shader/YUV_packed.frag b/include/libcamera/internal/shaders/YUV_packed.frag similarity index 100% rename from src/apps/qcam/assets/shader/YUV_packed.frag rename to include/libcamera/internal/shaders/YUV_packed.frag diff --git a/src/apps/qcam/assets/shader/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag similarity index 100% rename from src/apps/qcam/assets/shader/bayer_1x_packed.frag rename to include/libcamera/internal/shaders/bayer_1x_packed.frag diff --git a/src/apps/qcam/assets/shader/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag similarity index 100% rename from src/apps/qcam/assets/shader/bayer_8.frag rename to include/libcamera/internal/shaders/bayer_8.frag diff --git a/src/apps/qcam/assets/shader/bayer_8.vert b/include/libcamera/internal/shaders/bayer_8.vert similarity index 100% rename from src/apps/qcam/assets/shader/bayer_8.vert rename to include/libcamera/internal/shaders/bayer_8.vert diff --git a/src/apps/qcam/assets/shader/identity.vert b/include/libcamera/internal/shaders/identity.vert similarity index 100% rename from src/apps/qcam/assets/shader/identity.vert rename to include/libcamera/internal/shaders/identity.vert diff --git a/src/apps/qcam/assets/shader/shaders.qrc b/src/apps/qcam/assets/shader/shaders.qrc index 96c709f92..04f9d7061 100644 --- a/src/apps/qcam/assets/shader/shaders.qrc +++ b/src/apps/qcam/assets/shader/shaders.qrc @@ -1,13 +1,13 @@ - RGB.frag - YUV_2_planes.frag - YUV_3_planes.frag - YUV_packed.frag - bayer_1x_packed.frag - bayer_8.frag - bayer_8.vert - identity.vert + ../../../../../include/libcamera/internal/shaders/RGB.frag + ../../../../../include/libcamera/internal/shaders/YUV_2_planes.frag + ../../../../../include/libcamera/internal/shaders/YUV_3_planes.frag + ../../../../../include/libcamera/internal/shaders/YUV_packed.frag + ../../../../../include/libcamera/internal/shaders/bayer_1x_packed.frag + ../../../../../include/libcamera/internal/shaders/bayer_8.frag + ../../../../../include/libcamera/internal/shaders/bayer_8.vert + ../../../../../include/libcamera/internal/shaders/identity.vert diff --git a/src/apps/qcam/viewfinder_gl.cpp b/src/apps/qcam/viewfinder_gl.cpp index f31956ff0..70f600650 100644 --- a/src/apps/qcam/viewfinder_gl.cpp +++ b/src/apps/qcam/viewfinder_gl.cpp @@ -141,7 +141,7 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) textureMinMagFilters_ = GL_LINEAR; /* Use identity.vert as the default vertex shader. */ - vertexShaderFile_ = ":identity.vert"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/identity.vert"; fragmentShaderDefines_.clear(); @@ -150,170 +150,170 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) horzSubSample_ = 2; vertSubSample_ = 2; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV21: horzSubSample_ = 2; vertSubSample_ = 2; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV16: horzSubSample_ = 2; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV61: horzSubSample_ = 2; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV24: horzSubSample_ = 1; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV42: horzSubSample_ = 1; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::YUV420: horzSubSample_ = 2; vertSubSample_ = 2; - fragmentShaderFile_ = ":YUV_3_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_3_planes.frag"; break; case libcamera::formats::YVU420: horzSubSample_ = 2; vertSubSample_ = 2; - fragmentShaderFile_ = ":YUV_3_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_3_planes.frag"; break; case libcamera::formats::UYVY: fragmentShaderDefines_.append("#define YUV_PATTERN_UYVY"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::VYUY: fragmentShaderDefines_.append("#define YUV_PATTERN_VYUY"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::YUYV: fragmentShaderDefines_.append("#define YUV_PATTERN_YUYV"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::YVYU: fragmentShaderDefines_.append("#define YUV_PATTERN_YVYU"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::ABGR8888: fragmentShaderDefines_.append("#define RGB_PATTERN rgb"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::ARGB8888: fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::BGRA8888: fragmentShaderDefines_.append("#define RGB_PATTERN gba"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::RGBA8888: fragmentShaderDefines_.append("#define RGB_PATTERN abg"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::BGR888: fragmentShaderDefines_.append("#define RGB_PATTERN rgb"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::RGB888: fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::SBGGR8: firstRed_.setX(1.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG8: firstRed_.setX(0.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG8: firstRed_.setX(1.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB8: firstRed_.setX(0.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SBGGR12_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG12_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG12_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB12_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; default: From a6a1fbd82b83ef3258a61aa284d200559264ef9d Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 26 Jan 2024 22:10:09 +0000 Subject: [PATCH 17/42] utils: gen-shader-headers: Add a utility to generate headers from shaders Two simple script to generate a header that contains GLSL shaders translated to C arrays. Signed-off-by: Bryan O'Donoghue --- utils/gen-shader-header.py | 38 ++++++++++++++++++++++++++++++++ utils/gen-shader-headers.sh | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100755 utils/gen-shader-header.py create mode 100755 utils/gen-shader-headers.sh diff --git a/utils/gen-shader-header.py b/utils/gen-shader-header.py new file mode 100755 index 000000000..6668e648f --- /dev/null +++ b/utils/gen-shader-header.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2025, Bryan O'Donoghue. +# +# Author: Bryan O'Donoghue +# +# A python script which takes a list of shader files and converts into a C +# header. +# +import sys + +try: + with open(sys.argv[2]) as file: + data = file.read() + data_len = len(data) + + name = sys.argv[1].replace(".", "_") + name_len = name + "_len" + + j = 0 + print("unsigned char", name, "[] = {") + for ch in data: + print(f"0x{ord(ch):02x}, ", end="") + j = j + 1 + if j == 16: + print() + j = 0 + if j != 0: + print() + print("};") + + print() + print(f"const unsigned int {name_len}={data_len};") + +except FileNotFoundError: + print(f"File {sys.argv[2]} not found", file=sys.stderr) +except IOError: + print(f"Unable to read {sys.argv[2]}", file=sys.stderr) diff --git a/utils/gen-shader-headers.sh b/utils/gen-shader-headers.sh new file mode 100755 index 000000000..ca4f19f07 --- /dev/null +++ b/utils/gen-shader-headers.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -x + +if [ $# -lt 4 ]; then + echo "Invalid arg count must be >= 5" + exit 1 +fi +src_dir="$1"; shift +build_dir="$1"; shift +build_path=$build_dir/"$1"; shift + +cat < "$build_path" +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* This file is auto-generated, do not edit! */ +/* + * Copyright (C) 2025, Linaro Ltd. + * + */ + +#pragma once + +EOF + +cat <> "$build_path" +/* + * List the names of the shaders at the top of + * header for readability's sake + * +EOF + +for file in "$@"; do + echo "file is $file" + name=$(basename "$build_dir/$file" | tr '.' '_') + echo " * unsigned char $name;" >> "$build_path" +done + +echo "*/" >> "$build_path" + +echo "/* Hex encoded shader data */" >> "$build_path" +for file in "$@"; do + name=$(basename "$build_dir/$file") + "$src_dir/utils/gen-shader-header.py" "$name" "$build_dir/$file" >> "$build_path" + echo >> "$build_path" +done From 8422ee944199cba8e015ce80a52e47c4592ff0e7 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 26 Jan 2024 22:18:13 +0000 Subject: [PATCH 18/42] meson: Automatically generate glsl_shaders.h from specified shader programs Encode the bayer shader files into a header as part of the build process. Qcam already compiles the shader files down into a QT resource file which it references internally. In order to share the debayering shader programs outside of qcam create a generic header which both qcam and libcamera can operate from. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/meson.build | 10 ++++++++++ include/libcamera/internal/shaders/meson.build | 10 ++++++++++ utils/meson.build | 2 ++ 3 files changed, 22 insertions(+) create mode 100644 include/libcamera/internal/shaders/meson.build diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index 33f318b2b..c0b593bfc 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -1,6 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 subdir('tracepoints') +subdir('shaders') libcamera_internal_headers = files([ 'bayer_format.h', @@ -57,5 +58,14 @@ tracepoints_h = custom_target( libcamera_internal_headers += tracepoints_h +libcamera_shader_headers = custom_target( + 'gen-shader-headers', + input : [shader_files], + output : 'glsl_shaders.h', + command : [gen_shader_headers, meson.project_source_root(), meson.project_build_root(), '@OUTPUT@', '@INPUT@'], +) + +libcamera_internal_headers += libcamera_shader_headers + subdir('converter') subdir('software_isp') diff --git a/include/libcamera/internal/shaders/meson.build b/include/libcamera/internal/shaders/meson.build new file mode 100644 index 000000000..386b342d0 --- /dev/null +++ b/include/libcamera/internal/shaders/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 + +# List of shader files to convert to header hex +# for the purposes of inclusion in OpenGL debayering +shader_files = files([ + 'bayer_1x_packed.frag', + 'bayer_8.frag', + 'bayer_8.vert', + 'identity.vert', +]) diff --git a/utils/meson.build b/utils/meson.build index 95d657ac9..3deed8ad4 100644 --- a/utils/meson.build +++ b/utils/meson.build @@ -3,5 +3,7 @@ subdir('codegen') subdir('ipu3') +gen_shader_headers = files('gen-shader-headers.sh') + ## Module signing gen_ipa_priv_key = files('gen-ipa-priv-key.sh') From 89cf67a45fa255a8d0275dfeff15c09597633ec6 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 10 May 2025 04:58:51 +0100 Subject: [PATCH 19/42] libcamera: software_isp: ccm: Populate CCM table to Debayer params structure Populate the DebayerParams CCM table during ccm::prepare(). A copy is made of the CCM into the DebayerParams structure. Signed-off-by: Bryan O'Donoghue --- src/ipa/simple/algorithms/ccm.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ipa/simple/algorithms/ccm.cpp b/src/ipa/simple/algorithms/ccm.cpp index 0a98406c1..cb023878d 100644 --- a/src/ipa/simple/algorithms/ccm.cpp +++ b/src/ipa/simple/algorithms/ccm.cpp @@ -14,6 +14,7 @@ #include #include "libcamera/internal/matrix.h" +#include "libcamera/internal/software_isp/debayer_params.h" namespace { @@ -84,7 +85,7 @@ void Ccm::applySaturation(Matrix &ccm, float saturation) } void Ccm::prepare(IPAContext &context, const uint32_t frame, - IPAFrameContext &frameContext, [[maybe_unused]] DebayerParams *params) + IPAFrameContext &frameContext, DebayerParams *params) { auto &saturation = context.activeState.knobs.saturation; @@ -108,6 +109,7 @@ void Ccm::prepare(IPAContext &context, const uint32_t frame, context.activeState.ccm.ccm = ccm; frameContext.ccm.ccm = ccm; frameContext.saturation = saturation; + params->ccm = ccm; context.activeState.ccm.changed = true; } From 59284f4f27f29c1b1e0fd5af948a826a92bed2d3 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 10 May 2025 06:14:20 +0100 Subject: [PATCH 20/42] libcamera: software_isp: lut: Make gain corrected CCM in lut.cpp available in debayer params There is another CCM calculated in this algorithm callback, this time based on the LUT. Signed-off-by: Bryan O'Donoghue --- src/ipa/simple/algorithms/lut.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ipa/simple/algorithms/lut.cpp b/src/ipa/simple/algorithms/lut.cpp index d1d5f7271..a161adb1a 100644 --- a/src/ipa/simple/algorithms/lut.cpp +++ b/src/ipa/simple/algorithms/lut.cpp @@ -126,6 +126,7 @@ void Lut::prepare(IPAContext &context, auto &red = params->redCcm; auto &green = params->greenCcm; auto &blue = params->blueCcm; + params->ccm = ccm; for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { red[i].r = ccmValue(i, ccm[0][0]); red[i].g = ccmValue(i, ccm[1][0]); From 234849b2b9d41185124a01ed208cfa6e7ff5bbf5 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Mon, 23 Sep 2024 16:11:28 +0100 Subject: [PATCH 21/42] libcamera: software_isp: gbm: Add in a GBM helper class for GPU surface access A helper class to interact with GBM. This will allow us to specify the internal storage format of the CPU when making a texture for the Debayer vertext/fragment shaders and thus ensure we receive an uncompressed and untiled output buffer. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/gbm.h | 55 ++++++++++ include/libcamera/internal/meson.build | 1 + src/libcamera/gbm.cpp | 137 +++++++++++++++++++++++++ src/libcamera/meson.build | 11 ++ 4 files changed, 204 insertions(+) create mode 100644 include/libcamera/internal/gbm.h create mode 100644 src/libcamera/gbm.cpp diff --git a/include/libcamera/internal/gbm.h b/include/libcamera/internal/gbm.h new file mode 100644 index 000000000..a5486cc94 --- /dev/null +++ b/include/libcamera/internal/gbm.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * gbm.h - Helper class for managing GBM interactions. + */ + +#pragma once + +#include + +#include + +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(GBM) + +class GBM +{ +public: + GBM(); + ~GBM(); + + int initSurface(uint32_t width, uint32_t height); + int mapSurface(); + int getFrameBufferData(uint8_t *data_out, size_t data_len); + struct gbm_device *getDevice() { return gbm_device_; } + struct gbm_surface *getSurface() { return gbm_surface_; } + uint32_t getFrameSize() { return framesize_; } + uint32_t getStride() { return stride_; } + PixelFormat getPixelFormat() { return format_; } + +private: + int fd_; + struct gbm_device *gbm_device_; + struct gbm_surface *gbm_surface_; + + struct gbm_bo *gbm_bo_; + uint32_t width_; + uint32_t height_; + uint32_t stride_; + uint32_t offset_; + uint32_t framesize_; + void *map_; + int bo_fd_; + + PixelFormat format_; +}; + +} // namespace libcamera diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index c0b593bfc..4a2919f61 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -23,6 +23,7 @@ libcamera_internal_headers = files([ 'dma_buf_allocator.h', 'formats.h', 'framebuffer.h', + 'gbm.h', 'ipa_data_serializer.h', 'ipa_manager.h', 'ipa_module.h', diff --git a/src/libcamera/gbm.cpp b/src/libcamera/gbm.cpp new file mode 100644 index 000000000..43032093e --- /dev/null +++ b/src/libcamera/gbm.cpp @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl.cpp - Helper class for managing GBM interactions. + */ + +#include "libcamera/internal/gbm.h" + +#include +#include +#include +#include + +#include +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(GBM) + +GBM::GBM() +{ + fd_ = 0; +} + +GBM::~GBM() +{ + if (gbm_surface_) + gbm_surface_destroy(gbm_surface_); + + if (gbm_device_) + gbm_device_destroy(gbm_device_); + + if (fd_ >= 0) + close(fd_); +} + +// this should probably go into its own class to deal with the +// allocation and deletion of frambuffers attached to GBM devices/objects +int GBM::initSurface(uint32_t width, uint32_t height) +{ + const char *dri_node = "/dev/dri/renderD128"; //TODO: get from an env or config setting + + fd_ = open(dri_node, O_RDWR | O_CLOEXEC); //TODO: CLOEXEC ? + if (fd_ < 0) { + LOG(GBM, Error) << "Open " << dri_node << " fail " << fd_; + return fd_; + } + + gbm_device_ = gbm_create_device(fd_); + if (!gbm_device_) { + LOG(GBM, Error) << "gbm_crate_device fail"; + goto fail; + } + + // GBM_FORMAT_RGBA8888 is not supported mesa::src/gbm/dri/gbm_dri.c::gbm_dri_visuals_table[] + // This means we need to choose XRGB8888 or ARGB8888 as the raw buffer format + gbm_surface_ = gbm_surface_create(gbm_device_, width, height, GBM_FORMAT_ARGB8888, + GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR); + if (!gbm_surface_) { + LOG(GBM, Error) << "Unable to create linear gbm surface"; + goto fail; + } + + format_ = libcamera::formats::ARGB8888; + + return 0; +fail: + return -ENODEV; +} + +int GBM::mapSurface() +{ + gbm_bo_ = gbm_surface_lock_front_buffer(gbm_surface_); + if (!gbm_bo_) { + LOG(GBM, Error) << "GBM input buffer object create fail"; + return -ENODEV; + } + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + + bo_fd_ = gbm_bo_get_fd(gbm_bo_); + + if (!bo_fd_) { + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + LOG(GBM, Error) << "Unable to get fd for bo: " << bo_fd_; + return -ENODEV; + } + + stride_ = gbm_bo_get_stride(gbm_bo_); + width_ = gbm_bo_get_width(gbm_bo_); + height_ = gbm_bo_get_height(gbm_bo_); + offset_ = gbm_bo_get_offset(gbm_bo_, 0); + framesize_ = height_ * stride_; + + map_ = mmap(NULL, height_ * stride_, PROT_READ, MAP_SHARED, bo_fd_, 0); + if (map_ == MAP_FAILED) { + LOG(GBM, Error) << "mmap gbm_bo_ fail"; + return -ENODEV; + } + + LOG(GBM, Debug) << " stride " << stride_ + << " width " << width_ + << " height " << height_ + << " offset " << offset_ + << " framesize " << framesize_; + + return 0; +} + +int GBM::getFrameBufferData(uint8_t *data, size_t data_len) +{ + struct dma_buf_sync sync; + + gbm_surface_lock_front_buffer(gbm_surface_); + + sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ; + ioctl(bo_fd_, DMA_BUF_IOCTL_SYNC, &sync); + + if (data_len > framesize_) { + LOG(GBM, Error) << "Invalid read size " << data_len << " max is " << framesize_; + return -EINVAL; + } + + memcpy(data, map_, data_len); + + sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ; + ioctl(bo_fd_, DMA_BUF_IOCTL_SYNC, &sync); + + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + + return 0; +} +} //namespace libcamera diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 202db1efe..0d004694e 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -67,6 +67,16 @@ libcamera_deps = [] libatomic = cc.find_library('atomic', required : false) libthreads = dependency('threads') +libgbm = cc.find_library('gbm', required: false) +gbm_works = cc.check_header('gbm.h', required: false) + +if libgbm.found() and gbm_works + config_h.set('HAVE_GBM', 1) + libcamera_internal_sources += files([ + 'gbm.cpp', + ]) +endif + subdir('base') subdir('converter') subdir('ipa') @@ -188,6 +198,7 @@ libcamera_deps += [ libcamera_base_private, libcrypto, libdl, + libgbm, liblttng, libudev, libyaml, From 94f30456aa4765aa3cf19030f68a7b6696b4cf76 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 25 Feb 2024 16:23:26 +0000 Subject: [PATCH 22/42] libcamera: software_isp: egl: Introduce an eGL base helper class Introduce an eGL base helper class which provides an eGL context based on a passed width and height. The initGLContext function could be overloaded to provide an interface to a real display. A set of helper functions is provided to compile and link GLSL shaders. linkShaderProgram currently compiles vertex/fragment pairs but could be overloaded or passed a parameter to link a compute shader instead. Breaking the eGL interface away from debayering - allows to use the eGL context inside of a dma-buf heap cleanly, reuse that context inside of a debayer layer and conceivably reuse the context in a multi-stage shader pass. Small note the image_attrs[] array doesn't pass checkstyle.py however the elements of the array are in pairs. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/egl.h | 110 +++++++++ src/libcamera/egl.cpp | 369 +++++++++++++++++++++++++++++++ src/libcamera/meson.build | 23 ++ 3 files changed, 502 insertions(+) create mode 100644 include/libcamera/internal/egl.h create mode 100644 src/libcamera/egl.cpp diff --git a/include/libcamera/internal/egl.h b/include/libcamera/internal/egl.h new file mode 100644 index 000000000..04d637d88 --- /dev/null +++ b/include/libcamera/internal/egl.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl_context.cpp - Helper class for managing eGL interactions. + */ + +#pragma once + +#include + +#include + +#include "libcamera/internal/gbm.h" + +#define EGL_EGLEXT_PROTOTYPES +#include +#include +#define GL_GLEXT_PROTOTYPES +#include +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(eGL) + +class eGLImage +{ +public: + eGLImage(uint32_t width, uint32_t height, uint32_t bpp, GLenum texture_unit, uint32_t texture_unit_uniform_id) + { + image_ = EGL_NO_IMAGE_KHR; + width_ = width; + height_ = height; + bpp_ = bpp; + stride_ = width_ * bpp_ / 4; + framesize_ = stride_ * height_; + texture_unit_ = texture_unit; + texture_unit_uniform_id_ = texture_unit_uniform_id; + + glGenTextures(1, &texture_); + } + + ~eGLImage() + { + glDeleteTextures(1, &texture_); + } + + uint32_t width_; + uint32_t height_; + uint32_t stride_; + uint32_t offset_; + uint32_t framesize_; + uint32_t bpp_; + uint32_t texture_unit_uniform_id_; + GLenum texture_unit_; + GLuint texture_; + EGLImageKHR image_; +}; + +class eGL +{ +public: + eGL(); + ~eGL(); + + int initEGLContext(GBM *gbmContext); + int createDMABufTexture2D(eGLImage *eglImage, int fd); + void destroyDMABufTexture(eGLImage *eglImage); + void createTexture2D(eGLImage *eglImage, GLint format, uint32_t width, uint32_t height, void *data); + void createTexture1D(eGLImage *eglImage, GLint format, uint32_t width, void *data); + + void pushEnv(std::vector &shaderEnv, const char *str); + void makeCurrent(); + void swapBuffers(); + + int compileVertexShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + int compileFragmentShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + int linkProgram(GLuint &programIdd, GLuint fragmentshaderId, GLuint vertexshaderId); + void dumpShaderSource(GLuint shaderId); + void useProgram(GLuint programId); + +private: + int fd_; + + EGLDisplay display_; + EGLContext context_; + EGLSurface surface_; + + int compileShader(int shaderType, GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + + PFNEGLEXPORTDMABUFIMAGEMESAPROC eglExportDMABUFImageMESA; + PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOES; + + PFNEGLCREATEIMAGEKHRPROC eglCreateImageKHR; + PFNEGLDESTROYIMAGEKHRPROC eglDestroyImageKHR; + + PFNEGLCLIENTWAITSYNCKHRPROC eglClientWaitSyncKHR; + PFNEGLCREATESYNCKHRPROC eglCreateSyncKHR; +}; +} //namespace libcamera diff --git a/src/libcamera/egl.cpp b/src/libcamera/egl.cpp new file mode 100644 index 000000000..89ece148b --- /dev/null +++ b/src/libcamera/egl.cpp @@ -0,0 +1,369 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl.cpp - Helper class for managing eGL interactions. + */ + +#include "libcamera/internal/egl.h" + +#include +#include +#include +#include + +#include +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(eGL) + +eGL::eGL() +{ +} + +eGL::~eGL() +{ +} + +// Create linear image attached to previous BO object +int eGL::createDMABufTexture2D(eGLImage *eglImage, int fd) +{ + int ret = 0; + + eglImage->stride_ = eglImage->width_ * eglImage->height_; + eglImage->offset_ = 0; + eglImage->framesize_ = eglImage->height_ * eglImage->stride_; + + LOG(eGL, Info) + << " stride " << eglImage->stride_ << " width " << eglImage->width_ << " height " << eglImage->height_ << " offset " << eglImage->offset_ << " framesize " << eglImage->framesize_; + + // TODO: use the dma buf handle from udma heap here directly + // should work for both input and output with fencing + EGLint image_attrs[] = { + EGL_WIDTH, (EGLint)eglImage->width_, + EGL_HEIGHT, (EGLint)eglImage->height_, + EGL_LINUX_DRM_FOURCC_EXT, (int)GBM_FORMAT_ARGB8888, + EGL_DMA_BUF_PLANE0_FD_EXT, fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0, + EGL_DMA_BUF_PLANE0_PITCH_EXT, (EGLint)eglImage->framesize_, + EGL_NONE, EGL_NONE, /* modifier lo */ + EGL_NONE, EGL_NONE, /* modifier hi */ + EGL_NONE, + }; + + eglImage->image_ = eglCreateImageKHR(display_, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + NULL, image_attrs); + + if (eglImage->image_ == EGL_NO_IMAGE_KHR) { + LOG(eGL, Error) << "eglCreateImageKHR fail"; + ret = -ENODEV; + goto done; + } + + // Generate texture, bind, associate image to texture, configure, unbind + glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, eglImage->image_); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + +done: + return ret; +} + +void eGL::destroyDMABufTexture(eGLImage *eglImage) +{ + eglDestroyImage(display_, eglImage->image_); +} + +// +// Generate a 2D texture from an input buffer directly +void eGL::createTexture2D(eGLImage *eglImage, uint32_t width, uint32_t height, void *data) +{ + glActiveTexture(eglImage->texture_unit_); + glBindTexture(GL_TEXTURE_2D, eglImage->texture_); + + // Generate texture, bind, associate image to texture, configure, unbind + glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, data); + + // Nearest filtering + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + + // Wrap to edge to avoid edge artifacts + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +} + +int eGL::initEGLContext(GBM *gbmContext) +{ + EGLint configAttribs[] = { + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_NONE + }; + + EGLint contextAttribs[] = { + EGL_CONTEXT_MAJOR_VERSION, 2, + EGL_NONE + }; + + EGLint numConfigs; + EGLConfig config; + EGLint major; + EGLint minor; + + if (!eglBindAPI(EGL_OPENGL_ES_API)) { + LOG(eGL, Error) << "API bind fail"; + goto fail; + } + + //TODO: use optional eglGetPlatformDisplayEXT ? + display_ = eglGetDisplay(gbmContext->getDevice()); + if (display_ == EGL_NO_DISPLAY) { + LOG(eGL, Error) << "Unable to get EGL display"; + goto fail; + } + + if (eglInitialize(display_, &major, &minor) != EGL_TRUE) { + LOG(eGL, Error) << "eglInitialize fail"; + goto fail; + } + + LOG(eGL, Info) << "EGL: version " << major << "." << minor; + LOG(eGL, Info) << "EGL: EGL_VERSION: " << eglQueryString(display_, EGL_VERSION); + LOG(eGL, Info) << "EGL: EGL_VENDOR: " << eglQueryString(display_, EGL_VENDOR); + LOG(eGL, Info) << "EGL: EGL_CLIENT_APIS: " << eglQueryString(display_, EGL_CLIENT_APIS); + LOG(eGL, Info) << "EGL: EGL_EXTENSIONS: " << eglQueryString(display_, EGL_EXTENSIONS); + + //TODO: interrogate strings to make sure we aren't hooking unsupported functions + // and remember to error out if a function we depend on isn't found. + // we don't use these functions right now but expect to for DMA backed + // texture generation and render-to-texture. One thing we can do is differentiate + // between DMA and non-DMA texture generation based on the presence of these functions + // In reality most - all ? - mesa implementations have these extensions so + // probably no fallback will be required + eglCreateImageKHR = (PFNEGLCREATEIMAGEKHRPROC)eglGetProcAddress("eglCreateImageKHR"); + if (!eglCreateImageKHR) + LOG(eGL, Warning) << "eglCreateImageKHR not found"; + + eglDestroyImageKHR = (PFNEGLDESTROYIMAGEKHRPROC)eglGetProcAddress("eglDestroyImageKHR"); + if (!eglDestroyImageKHR) + LOG(eGL, Warning) << "eglDestroyImageKHR not found"; + + eglExportDMABUFImageMESA = (PFNEGLEXPORTDMABUFIMAGEMESAPROC)eglGetProcAddress("eglExportDMABUFImageMESA"); + if (!eglExportDMABUFImageMESA) + LOG(eGL, Warning) << "eglExportDMABUFImageMESA not found"; + + glEGLImageTargetTexture2DOES = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress("glEGLImageTargetTexture2DOES"); + if (!glEGLImageTargetTexture2DOES) + LOG(eGL, Warning) << "glEGLImageTargetTexture2DOES not found"; + + eglClientWaitSyncKHR = (PFNEGLCLIENTWAITSYNCKHRPROC)eglGetProcAddress("eglClientWaitSyncKHR"); + if (!eglClientWaitSyncKHR) + LOG(eGL, Warning) << "eglClientWaitSyncKHR not found"; + + eglCreateSyncKHR = (PFNEGLCREATESYNCKHRPROC)eglGetProcAddress("eglCreateSyncKHR"); + if (!eglCreateSyncKHR) + LOG(eGL, Warning) << "eglCreateSyncKHR not found"; + + if (eglChooseConfig(display_, configAttribs, &config, 1, &numConfigs) != EGL_TRUE) { + LOG(eGL, Error) << "eglChooseConfig fail"; + goto fail; + } + + context_ = eglCreateContext(display_, config, EGL_NO_CONTEXT, contextAttribs); + if (context_ == EGL_NO_CONTEXT) { + LOG(eGL, Error) << "eglContext returned EGL_NO_CONTEXT"; + goto fail; + } + + surface_ = eglCreateWindowSurface(display_, config, + (EGLNativeWindowType)gbmContext->getSurface(), + NULL); + if (surface_ == EGL_NO_SURFACE) { + LOG(eGL, Error) << "eglCreateWindowSurface fail"; + goto fail; + } + + makeCurrent(); + swapBuffers(); + + return 0; +fail: + + return -ENODEV; +} + +void eGL::makeCurrent(void) +{ + if (eglMakeCurrent(display_, surface_, surface_, context_) != EGL_TRUE) { + LOG(eGL, Error) << "eglMakeCurrent fail"; + } +} + +void eGL::swapBuffers(void) +{ + if (eglSwapBuffers(display_, surface_) != EGL_TRUE) { + LOG(eGL, Error) << "eglSwapBuffers fail"; + } +} + +void eGL::useProgram(GLuint programId) +{ + glUseProgram(programId); +} + +void eGL::pushEnv(std::vector &shaderEnv, const char *str) +{ + std::string addStr = str; + + addStr.push_back('\n'); + shaderEnv.push_back(addStr); +} + +int eGL::compileVertexShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + return compileShader(GL_VERTEX_SHADER, shaderId, shaderData, shaderDataLen, shaderEnv); +} + +int eGL::compileFragmentShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + return compileShader(GL_FRAGMENT_SHADER, shaderId, shaderData, shaderDataLen, shaderEnv); +} + +int eGL::compileShader(int shaderType, GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + GLchar **shaderSourceData; + GLint *shaderDataLengths; + GLint success; + GLsizei count; + size_t i; + + count = 1 + shaderEnv.size(); + shaderSourceData = new GLchar *[count]; + shaderDataLengths = new GLint[count]; + + // Prefix defines before main body of shader + for (i = 0; i < shaderEnv.size(); i++) { + shaderSourceData[i] = (GLchar *)shaderEnv[i].c_str(); + shaderDataLengths[i] = shaderEnv[i].length(); + } + + // Now the main body of the shader program + shaderSourceData[i] = (GLchar *)shaderData; + shaderDataLengths[i] = shaderDataLen; + + // And create the shader + shaderId = glCreateShader(shaderType); + glShaderSource(shaderId, count, shaderSourceData, shaderDataLengths); + glCompileShader(shaderId); + + // Check status + glGetShaderiv(shaderId, GL_COMPILE_STATUS, &success); + if (success == GL_FALSE) { + GLint sizeLog = 0; + GLchar *infoLog; + + glGetShaderiv(shaderId, GL_INFO_LOG_LENGTH, &sizeLog); + infoLog = new GLchar[sizeLog]; + + glGetShaderInfoLog(shaderId, sizeLog, &sizeLog, infoLog); + LOG(eGL, Error) << infoLog; + + delete[] infoLog; + } + + delete[] shaderSourceData; + delete[] shaderDataLengths; + + return !(success == GL_TRUE); +} + +void eGL::dumpShaderSource(GLuint shaderId) +{ + GLint shaderLength = 0; + GLchar *shaderSource; + + glGetShaderiv(shaderId, GL_SHADER_SOURCE_LENGTH, &shaderLength); + + LOG(eGL, Debug) << "Shader length is " << shaderLength; + + if (shaderLength > 0) { + shaderSource = new GLchar[shaderLength]; + if (!shaderSource) + return; + + glGetShaderSource(shaderId, shaderLength, &shaderLength, shaderSource); + if (shaderLength) { + LOG(eGL, Debug) << "Shader source = " << shaderSource; + } + delete[] shaderSource; + } +} + +int eGL::linkProgram(GLuint &programId, GLuint vertexshaderId, GLuint fragmentshaderId) +{ + GLint success; + GLenum err; + + programId = glCreateProgram(); + if (!programId) + goto fail; + + glAttachShader(programId, vertexshaderId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Attach compute vertex shader fail"; + goto fail; + } + + glAttachShader(programId, fragmentshaderId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Attach compute vertex shader fail"; + goto fail; + } + + glLinkProgram(programId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Link program fail"; + goto fail; + } + + glDetachShader(programId, fragmentshaderId); + glDetachShader(programId, vertexshaderId); + + // Check status + glGetProgramiv(programId, GL_LINK_STATUS, &success); + if (success == GL_FALSE) { + GLint sizeLog = 0; + GLchar *infoLog; + + glGetProgramiv(programId, GL_INFO_LOG_LENGTH, &sizeLog); + infoLog = new GLchar[sizeLog]; + + glGetProgramInfoLog(programId, sizeLog, &sizeLog, infoLog); + LOG(eGL, Error) << infoLog; + + delete[] infoLog; + goto fail; + } + + return 0; +fail: + return -ENODEV; +} +} // namespace libcamera diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 0d004694e..491eb7340 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -77,6 +77,27 @@ if libgbm.found() and gbm_works ]) endif +libegl = cc.find_library('EGL', required : false) +libglesv2 = cc.find_library('GLESv2', required : false) +mesa_works = cc.check_header('EGL/egl.h', required: false) + +if libegl.found() and mesa_works + config_h.set('HAVE_LIBEGL', 1) +endif + +if libglesv2.found() and mesa_works + config_h.set('HAVE_GLESV2', 1) +endif + +if mesa_works and gbm_works + libcamera_internal_sources += files([ + 'egl.cpp', + ]) + gles_headless_enabled = true +else + gles_headless_enabled = false +endif + subdir('base') subdir('converter') subdir('ipa') @@ -198,7 +219,9 @@ libcamera_deps += [ libcamera_base_private, libcrypto, libdl, + libegl, libgbm, + libglesv2, liblttng, libudev, libyaml, From eba39200919934ab0221224819c181fbd97b4eeb Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 31 Jan 2024 02:48:39 +0000 Subject: [PATCH 23/42] libcamera: software_isp: debayer_egl: Add an eGL debayer class Add a class to run the existing glsl debayer shaders on a GBM surface. Signed-off-by: Bryan O'Donoghue libcamera: software_isp: debayer_egl: Extend logic to enable application of softISP RGB debayer params The existing SoftISP calculates RGB gain values as a lookup table of 256 values which shifts for each frame depending on the required correction. We can pass the required tables into the debayer shaders as textures, one texture for R, G and B respectively. The debayer shader will do its debayer interpolation and then if the appropriate define is specified use the calculated R, G and B values as indexes into our bayer colour gain table. Signed-off-by: Bryan O'Donoghue --- src/libcamera/egl.cpp | 4 +- src/libcamera/software_isp/debayer_egl.cpp | 587 +++++++++++++++++++++ src/libcamera/software_isp/debayer_egl.h | 164 ++++++ src/libcamera/software_isp/meson.build | 8 + 4 files changed, 761 insertions(+), 2 deletions(-) create mode 100644 src/libcamera/software_isp/debayer_egl.cpp create mode 100644 src/libcamera/software_isp/debayer_egl.h diff --git a/src/libcamera/egl.cpp b/src/libcamera/egl.cpp index 89ece148b..c6b0f9a56 100644 --- a/src/libcamera/egl.cpp +++ b/src/libcamera/egl.cpp @@ -82,13 +82,13 @@ void eGL::destroyDMABufTexture(eGLImage *eglImage) // // Generate a 2D texture from an input buffer directly -void eGL::createTexture2D(eGLImage *eglImage, uint32_t width, uint32_t height, void *data) +void eGL::createTexture2D(eGLImage *eglImage, GLint format, uint32_t width, uint32_t height, void *data) { glActiveTexture(eglImage->texture_unit_); glBindTexture(GL_TEXTURE_2D, eglImage->texture_); // Generate texture, bind, associate image to texture, configure, unbind - glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, data); + glTexImage2D(GL_TEXTURE_2D, 0, format, width, height, 0, format, GL_UNSIGNED_BYTE, data); // Nearest filtering glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp new file mode 100644 index 000000000..3fb155119 --- /dev/null +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -0,0 +1,587 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * debayer_cpu.cpp - EGL based debayering class + */ + +#include +#include +#include + +#include + +#include "libcamera/internal/glsl_shaders.h" +#include "debayer_egl.h" + +namespace libcamera { + +DebayerEGL::DebayerEGL(std::unique_ptr stats) + : Debayer(), stats_(std::move(stats)) +{ + eglImageBayerIn_ = eglImageRedLookup_ = eglImageBlueLookup_ = eglImageGreenLookup_ = NULL; +} + +DebayerEGL::~DebayerEGL() +{ + if (eglImageBlueLookup_) + delete eglImageBlueLookup_; + + if (eglImageGreenLookup_) + delete eglImageGreenLookup_; + + if (eglImageRedLookup_) + delete eglImageRedLookup_; + + if (eglImageBayerIn_) + delete eglImageBayerIn_; +} + +int DebayerEGL::getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config) +{ + BayerFormat bayerFormat = + BayerFormat::fromPixelFormat(inputFormat); + + if ((bayerFormat.bitDepth == 8 || bayerFormat.bitDepth == 10) && + bayerFormat.packing == BayerFormat::Packing::None && + isStandardBayerOrder(bayerFormat.order)) { + config.bpp = (bayerFormat.bitDepth + 7) & ~7; + config.patternSize.width = 2; + config.patternSize.height = 2; + config.outputFormats = std::vector({ formats::XRGB8888, + formats::ARGB8888, + formats::XBGR8888, + formats::ABGR8888 }); + return 0; + } + + if (bayerFormat.bitDepth == 10 && + bayerFormat.packing == BayerFormat::Packing::CSI2 && + isStandardBayerOrder(bayerFormat.order)) { + config.bpp = 10; + config.patternSize.width = 4; /* 5 bytes per *4* pixels */ + config.patternSize.height = 2; + config.outputFormats = std::vector({ formats::XRGB8888, + formats::ARGB8888, + formats::XBGR8888, + formats::ABGR8888 }); + return 0; + } + + LOG(Debayer, Info) + << "Unsupported input format " << inputFormat.toString(); + return -EINVAL; +} + +int DebayerEGL::getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config) +{ + if (outputFormat == formats::XRGB8888 || outputFormat == formats::ARGB8888 || + outputFormat == formats::XBGR8888 || outputFormat == formats::ABGR8888) { + config.bpp = 32; + return 0; + } + + LOG(Debayer, Error) + << "Unsupported output format " << outputFormat.toString(); + + return -EINVAL; +} + +int DebayerEGL::getShaderVariableLocations(void) +{ + attributeVertex_ = glGetAttribLocation(programId_, "vertexIn"); + attributeTexture_ = glGetAttribLocation(programId_, "textureIn"); + + textureUniformBayerDataIn_ = glGetUniformLocation(programId_, "tex_y"); + textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); + textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); + textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); + + textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); + textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); + textureUniformStrideFactor_ = glGetUniformLocation(programId_, "stride_factor"); + textureUniformBayerFirstRed_ = glGetUniformLocation(programId_, "tex_bayer_first_red"); + textureUniformProjMatrix_ = glGetUniformLocation(programId_, "proj_matrix"); + + LOG(Debayer, Debug) << "vertexIn " << attributeVertex_ << " textureIn " << attributeTexture_ + << " tex_y " << textureUniformBayerDataIn_ + << " red_param " << textureUniformRedLookupDataIn_ + << " green_param " << textureUniformGreenLookupDataIn_ + << " blue_param " << textureUniformBlueLookupDataIn_ + << " tex_step " << textureUniformStep_ + << " tex_size " << textureUniformSize_ + << " stride_factor " << textureUniformStrideFactor_ + << " tex_bayer_first_red " << textureUniformBayerFirstRed_ + << " proj_matrix " << textureUniformProjMatrix_; + return 0; +} + +int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputFormat) +{ + std::vector shaderEnv; + unsigned int fragmentShaderDataLen; + unsigned char *fragmentShaderData; + unsigned int vertexShaderDataLen; + unsigned char *vertexShaderData; + GLenum err; + + // Target gles 100 glsl requires "#version x" as first directive in shader + egl_.pushEnv(shaderEnv, "#version 100"); + + // Specify GL_OES_EGL_image_external + egl_.pushEnv(shaderEnv, "#extension GL_OES_EGL_image_external: enable"); + + // Tell shaders how to re-order output taking account of how the + // pixels are actually stored by GBM + switch (outputFormat) { + case formats::ARGB8888: + case formats::XRGB8888: + break; + case formats::ABGR8888: + case formats::XBGR8888: + egl_.pushEnv(shaderEnv, "#define SWAP_BLUE"); + break; + default: + goto invalid_fmt; + } + + // Pixel location parameters + switch (inputFormat) { + case libcamera::formats::SBGGR8: + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SBGGR12_CSI2P: + firstRed_x_ = 1.0; + firstRed_y_ = 1.0; + break; + case libcamera::formats::SGBRG8: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + firstRed_x_ = 0.0; + firstRed_y_ = 1.0; + break; + case libcamera::formats::SGRBG8: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + firstRed_x_ = 1.0; + firstRed_y_ = 0.0; + break; + case libcamera::formats::SRGGB8: + case libcamera::formats::SRGGB10_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: + firstRed_x_ = 0.0; + firstRed_y_ = 0.0; + break; + default: + goto invalid_fmt; + break; + }; + + // Shader selection + switch (inputFormat) { + case libcamera::formats::SBGGR8: + case libcamera::formats::SGBRG8: + case libcamera::formats::SGRBG8: + case libcamera::formats::SRGGB8: + fragmentShaderData = bayer_8_frag; + fragmentShaderDataLen = bayer_8_frag_len; + vertexShaderData = bayer_8_vert; + vertexShaderDataLen = bayer_8_vert_len; + break; + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SRGGB10_CSI2P: + egl_.pushEnv(shaderEnv, "#define RAW10P"); + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + break; + case libcamera::formats::SBGGR12_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: + egl_.pushEnv(shaderEnv, "#define RAW12P"); + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + break; + default: + goto invalid_fmt; + break; + }; + + // Flag to shaders that we have parameter gain tables + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + + if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) + goto compile_fail; + + if (egl_.compileFragmentShader(fragmentShaderId_, fragmentShaderData, fragmentShaderDataLen, shaderEnv)) + goto compile_fail; + + if (egl_.linkProgram(programId_, vertexShaderId_, fragmentShaderId_)) + goto link_fail; + + egl_.dumpShaderSource(vertexShaderId_); + egl_.dumpShaderSource(fragmentShaderId_); + + /* Ensure we set the programId_ */ + egl_.useProgram(programId_); + err = glGetError(); + if (err != GL_NO_ERROR) + goto program_fail; + + if (getShaderVariableLocations()) + goto parameters_fail; + + return 0; + +parameters_fail: + LOG(Debayer, Error) << "Program parameters fail"; + return -ENODEV; + +program_fail: + LOG(Debayer, Error) << "Use program error " << err; + return -ENODEV; + +link_fail: + LOG(Debayer, Error) << "Linking program fail"; + return -ENODEV; + +compile_fail: + LOG(Debayer, Error) << "Compile debayer shaders fail"; + return -ENODEV; + +invalid_fmt: + LOG(Debayer, Error) << "Unsupported input output format combination"; + return -EINVAL; +} + +int DebayerEGL::configure(const StreamConfiguration &inputCfg, + const std::vector> &outputCfgs, + bool ccmEnabled) +{ + if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) + return -EINVAL; + + if (stats_->configure(inputCfg) != 0) + return -EINVAL; + + const Size &stats_pattern_size = stats_->patternSize(); + if (inputConfig_.patternSize.width != stats_pattern_size.width || + inputConfig_.patternSize.height != stats_pattern_size.height) { + LOG(Debayer, Error) + << "mismatching stats and debayer pattern sizes for " + << inputCfg.pixelFormat.toString(); + return -EINVAL; + } + + inputConfig_.stride = inputCfg.stride; + width_ = inputCfg.size.width; + height_ = inputCfg.size.height; + ccmEnabled_ = ccmEnabled = false; + + if (outputCfgs.size() != 1) { + LOG(Debayer, Error) + << "Unsupported number of output streams: " + << outputCfgs.size(); + return -EINVAL; + } + + LOG(Debayer, Info) << "Input size " << inputCfg.size << " stride " << inputCfg.stride; + + if (gbmSurface_.initSurface(inputCfg.size.width, inputCfg.size.height)) + return -ENODEV; + + if (egl_.initEGLContext(&gbmSurface_)) + return -ENODEV; + + glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); + LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; + + if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { + LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits + << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS + << " try using an identity CCM "; + return -ENODEV; + } + // Raw bayer input as texture + eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); + if (!eglImageBayerIn_) + return -ENOMEM; + + /// RGB correction tables as 2d textures + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); + if (!eglImageRedLookup_) + return -ENOMEM; + + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); + if (!eglImageGreenLookup_) + return -ENOMEM; + + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); + if (!eglImageBlueLookup_) + return -ENOMEM; + + // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) + if (gbmSurface_.mapSurface()) + return -ENODEV; + + StreamConfiguration &outputCfg = outputCfgs[0]; + SizeRange outSizeRange = sizes(inputCfg.pixelFormat, inputCfg.size); + + outputConfig_.stride = gbmSurface_.getStride(); + outputConfig_.frameSize = gbmSurface_.getFrameSize(); + + LOG(Debayer, Debug) << "Overriding stream config stride " + << outputCfg.stride << " with GBM surface stride " + << outputConfig_.stride; + outputCfg.stride = outputConfig_.stride; + + if (!outSizeRange.contains(outputCfg.size) || outputConfig_.stride != outputCfg.stride) { + LOG(Debayer, Error) + << "Invalid output size/stride: " + << "\n " << outputCfg.size << " (" << outSizeRange << ")" + << "\n " << outputCfg.stride << " (" << outputConfig_.stride << ")"; + return -EINVAL; + } + + window_.x = ((inputCfg.size.width - outputCfg.size.width) / 2) & + ~(inputConfig_.patternSize.width - 1); + window_.y = ((inputCfg.size.height - outputCfg.size.height) / 2) & + ~(inputConfig_.patternSize.height - 1); + window_.width = outputCfg.size.width; + window_.height = outputCfg.size.height; + + /* Don't pass x,y since process() already adjusts src before passing it */ + stats_->setWindow(Rectangle(window_.size())); + + LOG(Debayer, Debug) << "Input width " << inputCfg.size.width << " height " << inputCfg.size.height; + LOG(Debayer, Debug) << "Output width " << outputCfg.size.width << " height " << outputCfg.size.height; + LOG(Debayer, Debug) << "Output stride " << outputCfg.size.width << " height " << outputCfg.size.height; + + if (initBayerShaders(inputCfg.pixelFormat, outputCfg.pixelFormat)) + return -EINVAL; + + return 0; +} + +Size DebayerEGL::patternSize(PixelFormat inputFormat) +{ + DebayerEGL::DebayerInputConfig config; + + if (getInputConfig(inputFormat, config) != 0) + return {}; + + return config.patternSize; +} + +std::vector DebayerEGL::formats(PixelFormat inputFormat) +{ + DebayerEGL::DebayerInputConfig config; + + if (getInputConfig(inputFormat, config) != 0) + return std::vector(); + + return config.outputFormats; +} + +std::tuple +DebayerEGL::strideAndFrameSize(const PixelFormat &outputFormat, const Size &size) +{ + DebayerEGL::DebayerOutputConfig config; + + if (getOutputConfig(outputFormat, config) != 0) + return std::make_tuple(0, 0); + + /* round up to multiple of 8 for 64 bits alignment */ + unsigned int stride = (size.width * config.bpp / 8 + 7) & ~7; + + return std::make_tuple(stride, stride * size.height); +} + +void DebayerEGL::setShaderVariableValues(void) +{ + /* + * Raw Bayer 8-bit, and packed raw Bayer 10-bit/12-bit formats + * are stored in a GL_LUMINANCE texture. The texture width is + * equal to the stride. + */ + GLfloat firstRed[] = { firstRed_x_, firstRed_y_ }; + GLfloat imgSize[] = { (GLfloat)width_, + (GLfloat)height_ }; + GLfloat Step[] = { 1.0f / (inputConfig_.stride - 1), + 1.0f / (height_ - 1) }; + GLfloat Stride = 1.0f; + GLfloat projIdentityMatrix[] = { + 1, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0, + 0, 0, 0, 1 + }; + + // vertexIn - bayer_8.vert + glEnableVertexAttribArray(attributeVertex_); + glVertexAttribPointer(attributeVertex_, 2, GL_FLOAT, GL_TRUE, + 2 * sizeof(GLfloat), vcoordinates); + + // textureIn - bayer_8.vert + glEnableVertexAttribArray(attributeTexture_); + glVertexAttribPointer(attributeTexture_, 2, GL_FLOAT, GL_TRUE, + 2 * sizeof(GLfloat), tcoordinates); + + // Set the sampler2D to the respective texture unit for each texutre + // To simultaneously sample multiple textures we need to use multiple + // texture units + glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + + // These values are: + // firstRed = tex_bayer_first_red - bayer_8.vert + // imgSize = tex_size - bayer_8.vert + // step = tex_step - bayer_8.vert + // Stride = stride_factor identity.vert + // textureUniformProjMatri = No scaling + glUniform2fv(textureUniformBayerFirstRed_, 1, firstRed); + glUniform2fv(textureUniformSize_, 1, imgSize); + glUniform2fv(textureUniformStep_, 1, Step); + glUniform1f(textureUniformStrideFactor_, Stride); + glUniformMatrix4fv(textureUniformProjMatrix_, 1, + GL_FALSE, projIdentityMatrix); + + LOG(Debayer, Debug) << "vertexIn " << attributeVertex_ << " textureIn " << attributeTexture_ + << " tex_y " << textureUniformBayerDataIn_ + << " red_param " << textureUniformRedLookupDataIn_ + << " green_param " << textureUniformGreenLookupDataIn_ + << " blue_param " << textureUniformBlueLookupDataIn_ + << " tex_step " << textureUniformStep_ + << " tex_size " << textureUniformSize_ + << " stride_factor " << textureUniformStrideFactor_ + << " tex_bayer_first_red " << textureUniformBayerFirstRed_; + + LOG (Debayer, Debug) << "textureUniformY_ = 0 " << + " firstRed.x " << firstRed[0] << + " firstRed.y " << firstRed[1] << + " textureUniformSize_.width " << imgSize[0] << " " + " textureUniformSize_.height " << imgSize[1] << + " textureUniformStep_.x " << Step[0] << + " textureUniformStep_.y " << Step[1] << + " textureUniformStrideFactor_ " << Stride << + " textureUniformProjMatrix_ " << textureUniformProjMatrix_; + return; +} + +void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, DebayerParams ¶ms) +{ + LOG(Debayer, Debug) + << "Input height " << height_ + << " width " << width_ + << " fd " << in.getPlaneFD(0); + + // eGL context switch + egl_.makeCurrent(); + + // Greate a standard texture + // we will replace this with the DMA version at some point + egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); + + // Populate bayer parameters + egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + + // Setup the scene + setShaderVariableValues(); + glViewport(0, 0, width_, height_); + glClear(GL_COLOR_BUFFER_BIT); + glDisable(GL_BLEND); + + // Draw the scene + glDrawArrays(GL_TRIANGLE_FAN, 0, DEBAYER_OPENGL_COORDS); + + // eglclientWaitScynKhr / eglwaitsynckr ? + egl_.swapBuffers(); + + // Copy from the output GBM buffer to our output plane + // once we get render to texture working the + // explicit lock ioctl, memcpy and unlock ioctl won't be required + gbmSurface_.getFrameBufferData(out.planes()[0].data(), out.planes()[0].size()); +} + +void DebayerEGL::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params) +{ + bench_.startFrame(); + + std::vector dmaSyncers; + + dmaSyncBegin(dmaSyncers, input, output); + + setParams(params); + + /* Copy metadata from the input buffer */ + FrameMetadata &metadata = output->_d()->metadata(); + metadata.status = input->metadata().status; + metadata.sequence = input->metadata().sequence; + metadata.timestamp = input->metadata().timestamp; + + MappedFrameBuffer in(input, MappedFrameBuffer::MapFlag::Read); + MappedFrameBuffer out(output, MappedFrameBuffer::MapFlag::Write); + if (!in.isValid() || !out.isValid()) { + LOG(Debayer, Error) << "mmap-ing buffer(s) failed"; + metadata.status = FrameMetadata::FrameError; + return; + } + + debayerGPU(in, out, params); + + dmaSyncers.clear(); + + bench_.finishFrame(); + + metadata.planes()[0].bytesused = out.planes()[0].size(); + + // Calculate stats for the whole frame + stats_->processFrame(frame, 0, input); + + outputBufferReady.emit(output); + inputBufferReady.emit(input); +} + +SizeRange DebayerEGL::sizes(PixelFormat inputFormat, const Size &inputSize) +{ + Size patternSize = this->patternSize(inputFormat); + unsigned int borderHeight = patternSize.height; + + if (patternSize.isNull()) + return {}; + + /* No need for top/bottom border with a pattern height of 2 */ + if (patternSize.height == 2) + borderHeight = 0; + + /* + * For debayer interpolation a border is kept around the entire image + * and the minimum output size is pattern-height x pattern-width. + */ + if (inputSize.width < (3 * patternSize.width) || + inputSize.height < (2 * borderHeight + patternSize.height)) { + LOG(Debayer, Warning) + << "Input format size too small: " << inputSize.toString(); + return {}; + } + + return SizeRange(Size(patternSize.width, patternSize.height), + Size((inputSize.width - 2 * patternSize.width) & ~(patternSize.width - 1), + (inputSize.height - 2 * borderHeight) & ~(patternSize.height - 1)), + patternSize.width, patternSize.height); +} + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h new file mode 100644 index 000000000..c0fc220f8 --- /dev/null +++ b/src/libcamera/software_isp/debayer_egl.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025, Bryan O'Donoghue. + * + * Authors: + * Bryan O'Donoghue + * + * debayer_opengl.h - EGL debayer header + */ + +#pragma once + +#include +#include +#include + +#define GL_GLEXT_PROTOTYPES +#define EGL_EGLEXT_PROTOTYPES +#include +#include +#include + +#include + +#include "debayer.h" + +#include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/egl.h" +#include "libcamera/internal/framebuffer.h" +#include "libcamera/internal/mapped_framebuffer.h" +#include "libcamera/internal/software_isp/benchmark.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" + +namespace libcamera { + +#define DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS 4 +#define DEBAYER_OPENGL_COORDS 4 + +/** + * \class DebayerEGL + * \brief Class for debayering using an EGL Shader + * + * Implements an EGL shader based debayering solution. + */ +class DebayerEGL : public Debayer +{ +public: + /** + * \brief Constructs a DebayerEGL object. + * \param[in] stats Pointer to the stats object to use. + */ + DebayerEGL(std::unique_ptr stats); + ~DebayerEGL(); + + /* + * Setup the Debayer object according to the passed in parameters. + * Return 0 on success, a negative errno value on failure + * (unsupported parameters). + */ + int configure(const StreamConfiguration &inputCfg, + const std::vector> &outputCfgs, + bool ccmEnabled); + + /* + * Get width and height at which the bayer-pattern repeats. + * Return pattern-size or an empty Size for an unsupported inputFormat. + */ + Size patternSize(PixelFormat inputFormat); + + std::vector formats(PixelFormat input); + std::tuple strideAndFrameSize(const PixelFormat &outputFormat, const Size &size); + + void process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params); + + /** + * \brief Get the file descriptor for the statistics. + * + * \return the file descriptor pointing to the statistics. + */ + const SharedFD &getStatsFD() { return stats_->getStatsFD(); } + + /** + * \brief Get the output frame size. + * + * \return The output frame size. + */ + unsigned int frameSize() { return outputConfig_.frameSize; } + + SizeRange sizes(PixelFormat inputFormat, const Size &inputSize); + +private: + static int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); + static int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); + int setupStandardBayerOrder(BayerFormat::Order order); + void pushEnv(std::vector &shaderEnv, const char *str); + int initBayerShaders(PixelFormat inputFormat, PixelFormat outputFormat); + int initEGLContext(); + int generateTextures(); + int compileShaderProgram(GLuint &shaderId, GLenum shaderType, + unsigned char *shaderData, int shaderDataLen, + std::vector shaderEnv); + int linkShaderProgram(void); + int getShaderVariableLocations(); + void setShaderVariableValues(void); + void configureTexture(GLuint &texture); + void debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, DebayerParams ¶ms); + + // Shader program identifiers + GLuint vertexShaderId_; + GLuint fragmentShaderId_; + GLuint programId_; + enum { + BAYER_INPUT_INDEX = 0, + BAYER_OUTPUT_INDEX, + BAYER_BUF_NUM, + }; + + // Pointer to object representing input texture + eGLImage *eglImageBayerIn_; + + eGLImage *eglImageRedLookup_; + eGLImage *eglImageGreenLookup_; + eGLImage *eglImageBlueLookup_; + + // Shader parameters + float firstRed_x_; + float firstRed_y_; + GLint attributeVertex_; + GLint attributeTexture_; + GLint textureUniformStep_; + GLint textureUniformSize_; + GLint textureUniformStrideFactor_; + GLint textureUniformBayerFirstRed_; + GLint textureUniformProjMatrix_; + + GLint textureUniformBayerDataIn_; + GLint textureUniformRedLookupDataIn_; + GLint textureUniformGreenLookupDataIn_; + GLint textureUniformBlueLookupDataIn_; + + Rectangle window_; + std::unique_ptr stats_; + eGL egl_; + GBM gbmSurface_; + uint32_t width_; + uint32_t height_; + bool ccmEnabled_; + + GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { + { -1.0f, -1.0f }, + { -1.0f, +1.0f }, + { +1.0f, +1.0f }, + { +1.0f, -1.0f }, + }; + + GLfloat tcoordinates[DEBAYER_OPENGL_COORDS][2] = { + { 0.0f, 1.0f }, + { 0.0f, 0.0f }, + { 1.0f, 0.0f }, + { 1.0f, 1.0f }, + }; +}; + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/meson.build b/src/libcamera/software_isp/meson.build index 59fa5f02a..c61ac7d59 100644 --- a/src/libcamera/software_isp/meson.build +++ b/src/libcamera/software_isp/meson.build @@ -2,6 +2,7 @@ softisp_enabled = pipelines.contains('simple') summary({'SoftISP support' : softisp_enabled}, section : 'Configuration') +summary({'SoftISP GPU acceleration' : gles_headless_enabled}, section : 'Configuration') if not softisp_enabled subdir_done() @@ -14,3 +15,10 @@ libcamera_internal_sources += files([ 'software_isp.cpp', 'swstats_cpu.cpp', ]) + +if softisp_enabled and gles_headless_enabled + config_h.set('HAVE_DEBAYER_EGL', 1) + libcamera_internal_sources += files([ + 'debayer_egl.cpp', + ]) +endif From a9fa1ff3c65f9e7f8a1ac201a2befb0f49008885 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 12 Jun 2024 02:08:02 +0100 Subject: [PATCH 24/42] libcamera: software_isp: debayer_egl: Make DebayerEGL an environment option If GPUISP support is available make it so an environment variable can switch it on. Given we don't have full feature parity with CPUISP just yet on pixel format output, we should default to CPUISP mode giving the user the option to switch on GPUISP by setting LIBCAMERA_SOFTISP_MODE=gpu Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/software_isp.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index e8fa8a17a..ff4471c05 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -25,6 +26,9 @@ #include "libcamera/internal/software_isp/debayer_params.h" #include "debayer_cpu.h" +#if HAVE_DEBAYER_EGL +#include "debayer_egl.h" +#endif /** * \file software_isp.cpp @@ -114,7 +118,20 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor, } stats->statsReady.connect(this, &SoftwareIsp::statsReady); - debayer_ = std::make_unique(std::move(stats)); +#if HAVE_DEBAYER_EGL + const char *softISPMode = utils::secure_getenv("LIBCAMERA_SOFTISP_MODE"); + + if (softISPMode && !strcmp(softISPMode, "gpu")) + debayer_ = std::make_unique(std::move(stats)); +#endif + if (!debayer_) + debayer_ = std::make_unique(std::move(stats)); + + if (!debayer_) { + LOG(SoftwareIsp, Error) << "Failed to create Debayer object"; + return; + } + debayer_->inputBufferReady.connect(this, &SoftwareIsp::inputReady); debayer_->outputBufferReady.connect(this, &SoftwareIsp::outputReady); From 60082dd56fd6f0d56ccf8852c8459a40420a9396 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sun, 1 Dec 2024 13:27:12 +0000 Subject: [PATCH 25/42] libcamera: shaders: Use highp not mediump for float precision We get better sample resolution with highp instead of mediump. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/shaders/RGB.frag | 2 +- include/libcamera/internal/shaders/YUV_2_planes.frag | 2 +- include/libcamera/internal/shaders/YUV_3_planes.frag | 2 +- include/libcamera/internal/shaders/YUV_packed.frag | 2 +- include/libcamera/internal/shaders/bayer_1x_packed.frag | 2 +- include/libcamera/internal/shaders/bayer_8.frag | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libcamera/internal/shaders/RGB.frag b/include/libcamera/internal/shaders/RGB.frag index 4c374ac98..724395894 100644 --- a/include/libcamera/internal/shaders/RGB.frag +++ b/include/libcamera/internal/shaders/RGB.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/include/libcamera/internal/shaders/YUV_2_planes.frag b/include/libcamera/internal/shaders/YUV_2_planes.frag index 1d5d12062..d286f1179 100644 --- a/include/libcamera/internal/shaders/YUV_2_planes.frag +++ b/include/libcamera/internal/shaders/YUV_2_planes.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/include/libcamera/internal/shaders/YUV_3_planes.frag b/include/libcamera/internal/shaders/YUV_3_planes.frag index 8f788e90a..8e3e0b4a5 100644 --- a/include/libcamera/internal/shaders/YUV_3_planes.frag +++ b/include/libcamera/internal/shaders/YUV_3_planes.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/include/libcamera/internal/shaders/YUV_packed.frag b/include/libcamera/internal/shaders/YUV_packed.frag index b9ef9d41b..3c9e3e397 100644 --- a/include/libcamera/internal/shaders/YUV_packed.frag +++ b/include/libcamera/internal/shaders/YUV_packed.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index f53f55758..19b13ad08 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -20,7 +20,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif /* diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index 7e35ca88e..aa7a1b004 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -16,7 +16,7 @@ Copyright (C) 2021, Linaro //Pixel Shader #ifdef GL_ES -precision mediump float; +precision highp float; #endif /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ From 60394c45dc6d8a226b537ba2020b8b25e1cb7cbb Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 22 Apr 2025 02:08:59 +0100 Subject: [PATCH 26/42] libcamera: shaders: Extend debayer shaders to apply RGB gain values on output Extend out the bayer fragment shaders to take 3 x 256 byte inputs as textures from the CPU. We then use an index to the table to recover the colour-corrected values provided by the SoftIPA thread. Signed-off-by: Bryan O'Donoghue --- .../internal/shaders/bayer_1x_packed.frag | 56 ++++++++++++++++ .../libcamera/internal/shaders/bayer_8.frag | 62 ++++++++++++++++- src/libcamera/software_isp/debayer_egl.cpp | 67 +++++++++++++------ src/libcamera/software_isp/debayer_egl.h | 7 +- 4 files changed, 169 insertions(+), 23 deletions(-) diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index 19b13ad08..90bd64570 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -65,6 +65,10 @@ uniform vec2 tex_step; uniform vec2 tex_bayer_first_red; uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; +uniform mat3 ccm; void main(void) { @@ -212,5 +216,57 @@ void main(void) vec3(patterns.y, C, patterns.x) : vec3(patterns.wz, C)); +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; +#endif + gl_FragColor = vec4(rgb, 1.0); } diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index aa7a1b004..5955c2eaf 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -21,11 +21,17 @@ precision highp float; /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; varying vec4 center; varying vec4 yCoord; varying vec4 xCoord; +uniform mat3 ccm; void main(void) { + vec3 rgb; + #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r float C = texture2D(tex_y, center.xy).r; // ( 0, 0) @@ -97,11 +103,65 @@ void main(void) { PATTERN.xw += kB.xw * B; PATTERN.xz += kF.xz * F; - gl_FragColor.rgb = (alternate.y == 0.0) ? + rgb = (alternate.y == 0.0) ? ((alternate.x == 0.0) ? vec3(C, PATTERN.xy) : vec3(PATTERN.z, C, PATTERN.w)) : ((alternate.x == 0.0) ? vec3(PATTERN.w, C, PATTERN.z) : vec3(PATTERN.yx, C)); + +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; +#endif + + gl_FragColor.rgb = rgb; } diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 3fb155119..824cd6d3f 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void) textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); + ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm"); textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); @@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void) << " red_param " << textureUniformRedLookupDataIn_ << " green_param " << textureUniformGreenLookupDataIn_ << " blue_param " << textureUniformBlueLookupDataIn_ + << " ccm " << ccmUniformDataIn_ << " tex_step " << textureUniformStep_ << " tex_size " << textureUniformSize_ << " stride_factor " << textureUniformStrideFactor_ @@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm break; }; - // Flag to shaders that we have parameter gain tables - egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + if (ccmEnabled_) { + // Run the CCM if available + egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS"); + } else { + // Flag to shaders that we have parameter gain tables + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + } if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) goto compile_fail; @@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, const std::vector> &outputCfgs, bool ccmEnabled) { + GLint maxTextureImageUnits; + if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) return -EINVAL; @@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, inputConfig_.stride = inputCfg.stride; width_ = inputCfg.size.width; height_ = inputCfg.size.height; - ccmEnabled_ = ccmEnabled = false; + ccmEnabled_ = ccmEnabled = true; if (outputCfgs.size() != 1) { LOG(Debayer, Error) @@ -304,30 +313,35 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; - if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { + if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS << " try using an identity CCM "; return -ENODEV; } + // Raw bayer input as texture eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); if (!eglImageBayerIn_) return -ENOMEM; - /// RGB correction tables as 2d textures - // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate - eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); - if (!eglImageRedLookup_) - return -ENOMEM; + // Only do the RGB lookup table textures if CCM is disabled + if (!ccmEnabled_) { - eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); - if (!eglImageGreenLookup_) - return -ENOMEM; + /// RGB correction tables as 2d textures + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); + if (!eglImageRedLookup_) + return -ENOMEM; - eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); - if (!eglImageBlueLookup_) - return -ENOMEM; + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); + if (!eglImageGreenLookup_) + return -ENOMEM; + + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); + if (!eglImageBlueLookup_) + return -ENOMEM; + } // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) if (gbmSurface_.mapSurface()) @@ -440,9 +454,11 @@ void DebayerEGL::setShaderVariableValues(void) // To simultaneously sample multiple textures we need to use multiple // texture units glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); - glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + if (!ccmEnabled_) { + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + } // These values are: // firstRed = tex_bayer_first_red - bayer_8.vert @@ -494,9 +510,18 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); // Populate bayer parameters - egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); - egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); - egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + if (ccmEnabled_) { + GLfloat ccm[] = { + 1, 0, 0, + 0, 1, 0, + 0, 0, 1, + }; + glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); + } else { + egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + } // Setup the scene setShaderVariableValues(); diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h index c0fc220f8..94bc6fc41 100644 --- a/src/libcamera/software_isp/debayer_egl.h +++ b/src/libcamera/software_isp/debayer_egl.h @@ -134,17 +134,22 @@ private: GLint textureUniformProjMatrix_; GLint textureUniformBayerDataIn_; + + // These textures will either point to simple RGB gains or to CCM lookup tables GLint textureUniformRedLookupDataIn_; GLint textureUniformGreenLookupDataIn_; GLint textureUniformBlueLookupDataIn_; + // Represent per-frame CCM as a uniform vector of floats 3 x 3 + GLint ccmUniformDataIn_; + bool ccmEnabled_; + Rectangle window_; std::unique_ptr stats_; eGL egl_; GBM gbmSurface_; uint32_t width_; uint32_t height_; - bool ccmEnabled_; GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { { -1.0f, -1.0f }, From 2b7bca4a149bcb03a487dcb4ebb46c4c17f241ca Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 10 May 2025 06:25:46 +0100 Subject: [PATCH 27/42] libcamera: software_isp: debayer_egl: Convert from identity CCM to CCM calculated by SoftIPA Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer_egl.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 824cd6d3f..b30d21077 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -293,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, inputConfig_.stride = inputCfg.stride; width_ = inputCfg.size.width; height_ = inputCfg.size.height; - ccmEnabled_ = ccmEnabled = true; + ccmEnabled_ = ccmEnabled; if (outputCfgs.size() != 1) { LOG(Debayer, Error) @@ -511,10 +511,10 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay // Populate bayer parameters if (ccmEnabled_) { - GLfloat ccm[] = { - 1, 0, 0, - 0, 1, 0, - 0, 0, 1, + GLfloat ccm[9] = { + params.ccm[0][0], params.ccm[0][1], params.ccm[0][2], + params.ccm[1][0], params.ccm[1][1], params.ccm[1][2], + params.ccm[2][0], params.ccm[2][1], params.ccm[2][2], }; glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); } else { From c7346319bd90fdb4aabf2674c1afdfa5985a6ac3 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Sat, 10 May 2025 06:26:30 +0100 Subject: [PATCH 28/42] libcamera: software_isp: Switch on uncalibrated CCM to validate eGLDebayer Signed-off-by: Bryan O'Donoghue --- src/ipa/simple/data/uncalibrated.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index 5508e6686..8b6df9afc 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -8,12 +8,12 @@ algorithms: # Color correction matrices can be defined here. The CCM algorithm # has a significant performance impact, and should only be enabled # if tuned. - # - Ccm: - # ccms: - # - ct: 6500 - # ccm: [ 1, 0, 0, - # 0, 1, 0, - # 0, 0, 1] + - Ccm: + ccms: + - ct: 6500 + ccm: [ 1, 0, 0, + 0, 1, 0, + 0, 0, 1] - Lut: - Agc: ... From 53930ee1d723ca1ba9ea8bcd7b211662fb71670d Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 30 May 2025 11:43:28 +0100 Subject: [PATCH 29/42] libcamera: software_isp: Make isStandardBayerOrder static Make this member function static so that it may be called from static members of derived classes. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index 214bcdd3c..3893318b2 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -88,7 +88,7 @@ private: protected: void setParams(DebayerParams ¶ms); void dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output); - bool isStandardBayerOrder(BayerFormat::Order order); + static bool isStandardBayerOrder(BayerFormat::Order order); }; } /* namespace libcamera */ From 4a9a1dff88e13d3b547082c61047709d317d7d27 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Fri, 30 May 2025 11:50:47 +0100 Subject: [PATCH 30/42] libcamera: software_isp: debayer_cpu: Make getInputConfig and getOutputConfig static Make getInputConfig and getOutputConfig static so as to allow for interrogation of the supported pixel formats prior to object instantiation. Do this so as to allow the higher level logic make an informed choice between CPU and GPU ISP based on which pixel formats are supported. Curretnly CPU ISP supports more diverse input and output schemes. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/debayer_cpu.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index a043a9541..999e3421c 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -102,8 +102,8 @@ private: template void debayer10P_RGRG_BGR888(uint8_t *dst, const uint8_t *src[]); - int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); - int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); + static int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); + static int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); int setupStandardBayerOrder(BayerFormat::Order order); int setDebayerFunctions(PixelFormat inputFormat, PixelFormat outputFormat, From 6637b468b70be2827b648b4b1bb6156c92384bda Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Wed, 4 Jun 2025 16:51:16 +0100 Subject: [PATCH 31/42] libcamera: shaders: Extend bayer shaders to support swapping R and B on output We can easily facilitate swapping R and B on output. Pivot on an environment define for this purpose. Signed-off-by: Bryan O'Donoghue --- include/libcamera/internal/shaders/bayer_1x_packed.frag | 4 ++++ include/libcamera/internal/shaders/bayer_8.frag | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index 90bd64570..c0632eb1f 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -268,5 +268,9 @@ void main(void) rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; #endif +#if defined (SWAP_BLUE) + gl_FragColor = vec4(rgb.bgr, 1.0); +#else gl_FragColor = vec4(rgb, 1.0); +#endif } diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index 5955c2eaf..74ce15096 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -163,5 +163,9 @@ void main(void) { rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; #endif - gl_FragColor.rgb = rgb; +#if defined (SWAP_BLUE) + gl_FragColor = vec4(rgb.bgr, 1.0); +#else + gl_FragColor = vec4(rgb, 1.0); +#endif } From 9b66144aad4088e99cc5fc3ad215f8f1620c78cb Mon Sep 17 00:00:00 2001 From: Milan Zamazal Date: Tue, 29 Apr 2025 13:56:25 +0200 Subject: [PATCH 32/42] libcamera: shaders: Fix neighbouring positions in 8-bit debayering When accessing a texture position in a shader, the pixel with the nearest centre to the specified texture coordinates (as mandated by specifying GL_NEAREST parameter) is taken. The current vertex shader determines the positions of the neighbouring pixels by adding the provided texture steps to the exact centre pixel coordinates. But this places the computed coordinates, from the point of view of GL_NEAREST, exactly between the pixels and is thus prone to floating point inaccuracies. Wrong neighbouring pixel coordinates may be used, resulting in artefacts in the output image. Let's fix the problem by shifting the initial coordinates a bit from the pixel border. Signed-off-by: Milan Zamazal --- include/libcamera/internal/shaders/bayer_8.vert | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/libcamera/internal/shaders/bayer_8.vert b/include/libcamera/internal/shaders/bayer_8.vert index fb5109eee..fc1cf89f2 100644 --- a/include/libcamera/internal/shaders/bayer_8.vert +++ b/include/libcamera/internal/shaders/bayer_8.vert @@ -44,10 +44,10 @@ void main(void) { center.xy = textureIn; center.zw = textureIn * tex_size + tex_bayer_first_red; - xCoord = center.x + vec4(-2.0 * tex_step.x, - -tex_step.x, tex_step.x, 2.0 * tex_step.x); - yCoord = center.y + vec4(-2.0 * tex_step.y, - -tex_step.y, tex_step.y, 2.0 * tex_step.y); + xCoord = center.x + 0.1 * tex_step.x + + vec4(-2.0 * tex_step.x, -tex_step.x, tex_step.x, 2.0 * tex_step.x); + yCoord = center.y + 0.1 * tex_step.y + + vec4(-2.0 * tex_step.y, -tex_step.y, tex_step.y, 2.0 * tex_step.y); gl_Position = proj_matrix * vertexIn; } From f28498a2fb878d1786de5cc5ce1dd16af0ee597e Mon Sep 17 00:00:00 2001 From: Milan Zamazal Date: Tue, 29 Apr 2025 14:19:59 +0200 Subject: [PATCH 33/42] libcamera: software_isp: GPU support for unpacked 10/12-bit formats The GPU processing supports 8-bit sensor formats and 10/12-bit packed formats. Support for 10/12-bit unpacked formats is missing, let's add it. 10/12-bit unpacked formats use two adjacent bytes to store the value. This means the 8-bit shaders can be used if we can modify them for additional support of 16-bit addressing. This requires the following modifications: - Using GL_RG (two bytes per pixel) instead of GL_LUMINANCE (one byte per pixel) as the texture format for the given input formats. - Setting the texture width to the number of pixels rather than the number of bytes. - Making the definition of `fetch' macro variable, according to the pixel format. - Using only `fetch' for accessing the texture. Signed-off-by: Milan Zamazal Signed-off-by: Bryan O'Donoghue --- .../libcamera/internal/shaders/bayer_8.frag | 10 +++- src/libcamera/software_isp/debayer_egl.cpp | 46 +++++++++++++------ src/libcamera/software_isp/debayer_egl.h | 2 + 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index 74ce15096..78c2609c2 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -32,9 +32,17 @@ uniform mat3 ccm; void main(void) { vec3 rgb; + #if defined(RAW10P) + #define pixel(p) p.r / 4.0 + p.g * 64.0 + #define fetch(x, y) pixel(texture2D(tex_y, vec2(x, y))) + #elif defined(RAW12P) + #define pixel(p) p.r / 16.0 + p.g * 16.0 + #define fetch(x, y) pixel(texture2D(tex_y, vec2(x, y))) + #else #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r + #endif - float C = texture2D(tex_y, center.xy).r; // ( 0, 0) + float C = fetch(center.x, center.y); // ( 0, 0) const vec4 kC = vec4( 4.0, 6.0, 5.0, 5.0) / 8.0; // Determine which of four types of pixels we are on. diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index b30d21077..71742d849 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -151,6 +151,8 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm } // Pixel location parameters + glFormat_ = GL_LUMINANCE; + bytesPerPixel_ = 1; switch (inputFormat) { case libcamera::formats::SBGGR8: case libcamera::formats::SBGGR10_CSI2P: @@ -197,20 +199,38 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm case libcamera::formats::SGRBG10_CSI2P: case libcamera::formats::SRGGB10_CSI2P: egl_.pushEnv(shaderEnv, "#define RAW10P"); - fragmentShaderData = bayer_1x_packed_frag; - fragmentShaderDataLen = bayer_1x_packed_frag_len; - vertexShaderData = identity_vert; - vertexShaderDataLen = identity_vert_len; + if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { + fragmentShaderData = bayer_8_frag; + fragmentShaderDataLen = bayer_8_frag_len; + vertexShaderData = bayer_8_vert; + vertexShaderDataLen = bayer_8_vert_len; + glFormat_ = GL_RG; + bytesPerPixel_ = 2; + } else { + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + } break; case libcamera::formats::SBGGR12_CSI2P: case libcamera::formats::SGBRG12_CSI2P: case libcamera::formats::SGRBG12_CSI2P: case libcamera::formats::SRGGB12_CSI2P: egl_.pushEnv(shaderEnv, "#define RAW12P"); - fragmentShaderData = bayer_1x_packed_frag; - fragmentShaderDataLen = bayer_1x_packed_frag_len; - vertexShaderData = identity_vert; - vertexShaderDataLen = identity_vert_len; + if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { + fragmentShaderData = bayer_8_frag; + fragmentShaderDataLen = bayer_8_frag_len; + vertexShaderData = bayer_8_vert; + vertexShaderDataLen = bayer_8_vert_len; + glFormat_ = GL_RG; + bytesPerPixel_ = 2; + } else { + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + } break; default: goto invalid_fmt; @@ -430,7 +450,7 @@ void DebayerEGL::setShaderVariableValues(void) GLfloat firstRed[] = { firstRed_x_, firstRed_y_ }; GLfloat imgSize[] = { (GLfloat)width_, (GLfloat)height_ }; - GLfloat Step[] = { 1.0f / (inputConfig_.stride - 1), + GLfloat Step[] = { static_cast(bytesPerPixel_) / (inputConfig_.stride - 1), 1.0f / (height_ - 1) }; GLfloat Stride = 1.0f; GLfloat projIdentityMatrix[] = { @@ -507,7 +527,7 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay // Greate a standard texture // we will replace this with the DMA version at some point - egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); + egl_.createTexture2D(eglImageBayerIn_, glFormat_, inputConfig_.stride / bytesPerPixel_, height_, in.planes()[0].data()); // Populate bayer parameters if (ccmEnabled_) { @@ -518,9 +538,9 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay }; glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); } else { - egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); - egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); - egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + egl_.createTexture2D(eglImageRedLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); } // Setup the scene diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h index 94bc6fc41..56f5434ac 100644 --- a/src/libcamera/software_isp/debayer_egl.h +++ b/src/libcamera/software_isp/debayer_egl.h @@ -150,6 +150,8 @@ private: GBM gbmSurface_; uint32_t width_; uint32_t height_; + GLint glFormat_; + unsigned int bytesPerPixel_; GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { { -1.0f, -1.0f }, From a2b7c63532670b3817b0719c1723ecc1d3f08daa Mon Sep 17 00:00:00 2001 From: Milan Zamazal Date: Tue, 29 Apr 2025 14:37:05 +0200 Subject: [PATCH 34/42] libcamera: shaders: Rename bayer_8 to bayer_unpacked bayer_8.* shaders are now used for all unpacked sensor data formats, regardless of the pixel bit width. Let's rename the "8-bit" shaders to avoid confusion. Signed-off-by: Milan Zamazal Signed-off-by: Bryan O'Donoghue --- .../{bayer_8.frag => bayer_unpacked.frag} | 0 .../{bayer_8.vert => bayer_unpacked.vert} | 0 .../libcamera/internal/shaders/meson.build | 4 ++-- src/apps/qcam/assets/shader/shaders.qrc | 4 ++-- src/apps/qcam/viewfinder_gl.cpp | 16 ++++++------- src/libcamera/software_isp/debayer_egl.cpp | 24 +++++++++---------- 6 files changed, 24 insertions(+), 24 deletions(-) rename include/libcamera/internal/shaders/{bayer_8.frag => bayer_unpacked.frag} (100%) rename include/libcamera/internal/shaders/{bayer_8.vert => bayer_unpacked.vert} (100%) diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_unpacked.frag similarity index 100% rename from include/libcamera/internal/shaders/bayer_8.frag rename to include/libcamera/internal/shaders/bayer_unpacked.frag diff --git a/include/libcamera/internal/shaders/bayer_8.vert b/include/libcamera/internal/shaders/bayer_unpacked.vert similarity index 100% rename from include/libcamera/internal/shaders/bayer_8.vert rename to include/libcamera/internal/shaders/bayer_unpacked.vert diff --git a/include/libcamera/internal/shaders/meson.build b/include/libcamera/internal/shaders/meson.build index 386b342d0..dd441a577 100644 --- a/include/libcamera/internal/shaders/meson.build +++ b/include/libcamera/internal/shaders/meson.build @@ -4,7 +4,7 @@ # for the purposes of inclusion in OpenGL debayering shader_files = files([ 'bayer_1x_packed.frag', - 'bayer_8.frag', - 'bayer_8.vert', + 'bayer_unpacked.frag', + 'bayer_unpacked.vert', 'identity.vert', ]) diff --git a/src/apps/qcam/assets/shader/shaders.qrc b/src/apps/qcam/assets/shader/shaders.qrc index 04f9d7061..32dfa51bf 100644 --- a/src/apps/qcam/assets/shader/shaders.qrc +++ b/src/apps/qcam/assets/shader/shaders.qrc @@ -6,8 +6,8 @@ ../../../../../include/libcamera/internal/shaders/YUV_3_planes.frag ../../../../../include/libcamera/internal/shaders/YUV_packed.frag ../../../../../include/libcamera/internal/shaders/bayer_1x_packed.frag - ../../../../../include/libcamera/internal/shaders/bayer_8.frag - ../../../../../include/libcamera/internal/shaders/bayer_8.vert + ../../../../../include/libcamera/internal/shaders/bayer_unpacked.frag + ../../../../../include/libcamera/internal/shaders/bayer_unpacked.vert ../../../../../include/libcamera/internal/shaders/identity.vert diff --git a/src/apps/qcam/viewfinder_gl.cpp b/src/apps/qcam/viewfinder_gl.cpp index 70f600650..95965ab71 100644 --- a/src/apps/qcam/viewfinder_gl.cpp +++ b/src/apps/qcam/viewfinder_gl.cpp @@ -235,29 +235,29 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) case libcamera::formats::SBGGR8: firstRed_.setX(1.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; - fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG8: firstRed_.setX(0.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; - fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG8: firstRed_.setX(1.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; - fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB8: firstRed_.setX(0.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.vert"; - fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SBGGR10_CSI2P: diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 71742d849..9ec966608 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -189,10 +189,10 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm case libcamera::formats::SGBRG8: case libcamera::formats::SGRBG8: case libcamera::formats::SRGGB8: - fragmentShaderData = bayer_8_frag; - fragmentShaderDataLen = bayer_8_frag_len; - vertexShaderData = bayer_8_vert; - vertexShaderDataLen = bayer_8_vert_len; + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; break; case libcamera::formats::SBGGR10_CSI2P: case libcamera::formats::SGBRG10_CSI2P: @@ -200,10 +200,10 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm case libcamera::formats::SRGGB10_CSI2P: egl_.pushEnv(shaderEnv, "#define RAW10P"); if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { - fragmentShaderData = bayer_8_frag; - fragmentShaderDataLen = bayer_8_frag_len; - vertexShaderData = bayer_8_vert; - vertexShaderDataLen = bayer_8_vert_len; + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; glFormat_ = GL_RG; bytesPerPixel_ = 2; } else { @@ -219,10 +219,10 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm case libcamera::formats::SRGGB12_CSI2P: egl_.pushEnv(shaderEnv, "#define RAW12P"); if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { - fragmentShaderData = bayer_8_frag; - fragmentShaderDataLen = bayer_8_frag_len; - vertexShaderData = bayer_8_vert; - vertexShaderDataLen = bayer_8_vert_len; + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; glFormat_ = GL_RG; bytesPerPixel_ = 2; } else { From e0c8eb0913ccc716d0f06596f9929e16ae285e69 Mon Sep 17 00:00:00 2001 From: Bryan O'Donoghue Date: Tue, 10 Jun 2025 16:06:01 +0100 Subject: [PATCH 35/42] libcamera: software_isp: Add a gpuisp todo list List the series of things to do in GPU ISP in perceived order of difficulty. Signed-off-by: Bryan O'Donoghue --- src/libcamera/software_isp/gpuisp-todo.txt | 42 ++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/libcamera/software_isp/gpuisp-todo.txt diff --git a/src/libcamera/software_isp/gpuisp-todo.txt b/src/libcamera/software_isp/gpuisp-todo.txt new file mode 100644 index 000000000..0ff82f81e --- /dev/null +++ b/src/libcamera/software_isp/gpuisp-todo.txt @@ -0,0 +1,42 @@ +List the TODOs in perceived order of ease. + +24 bit output support: + - Take the BPP we already capture and get a 24 bit GBM surface + - Pass a compile-time parameter to the shaders to tell them to do + gl_FragColor = rgb not gl_FragColor = rgba + +Make GPUISP default: + - Right now the environment variable allows over-riding to swtich + from CPU to GPU. + - Once we support 24 BPP output on GPUISP we will have the same + pixel format support as CPU and can set the default to GPU without + regressing functionality + +glTexture1D: + - Initial code was developed for < GLES 2.O but since we have fixed + on GLES >= 2.0 this means we can use glTexture1D + - Provided this is so amend the shaders to do val = texture(x, y, 0); + not texture(x, y, 0.5) the 0.5 is because of using glTexture2D + +Surfaceless GBM: + - We get a GBM surface and then have to swap buffers + If we rework for surfaceless GBM and EGL then the swap buffer can + be dropped. + +dma-buf texture upload: + - Currently we pass the input buffer to glCreateTexture2D. + We should be able to make the upload of the input buffer go faster + by using eglCreateImageKHR and enumerated the dma-buf contents. + +Render-to-texture: + - Right now we render to the GBM provided surface framebuffer + and then memcpy from that buffer to the target output buffer. + This necessitates flushing the cache on the target buffer in + addition to the memcpy(). + - Render-to-texture where we generate the target framebuffer + directly from a dma-buf handle will mitigate the memcpy() phase. + - It should be the case then that the consumer of the output buffer + i.e. the thing that's not libcamera is responsible to flush the cache + if-and-only-if that user writes to the buffer. + - We need to flush the cache on the buffer because we are memcpying() to it. + From 9613459d30c1504c8c4ebd6b7241fa169f2eefd1 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sun, 23 Feb 2025 23:03:48 +0300 Subject: [PATCH 36/42] libcamera: software_isp: Add brightness control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 29 +++++++++++++++++++++++++++++ src/ipa/simple/algorithms/agc.h | 7 +++++++ src/ipa/simple/ipa_context.h | 2 ++ 3 files changed, 38 insertions(+) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index c46bb0ebe..249caa264 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -41,6 +41,33 @@ Agc::Agc() { } +int Agc::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + return 0; +} + +int Agc::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.brightness = std::optional(); + + return 0; +} + +void Agc::queueRequest(typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &brightness = controls.get(controls::Brightness); + if (brightness.has_value()) { + context.activeState.knobs.brightness = brightness; + LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); + } +} + void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) { /* @@ -54,6 +81,8 @@ void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, dou double next; int32_t &exposure = frameContext.sensor.exposure; double &again = frameContext.sensor.gain; + const auto brightness = context.activeState.knobs.brightness.value_or(1.0); + exposureMSV /= brightness; if (exposureMSV < kExposureOptimal - kExposureSatisfactory) { next = exposure * kExpNumeratorUp / kExpDenominator; diff --git a/src/ipa/simple/algorithms/agc.h b/src/ipa/simple/algorithms/agc.h index 112d9f5a1..00bc101e7 100644 --- a/src/ipa/simple/algorithms/agc.h +++ b/src/ipa/simple/algorithms/agc.h @@ -19,6 +19,13 @@ public: Agc(); ~Agc() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; void process(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const SwIspStats *stats, diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index a471b80ae..afc28ba22 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -64,6 +64,8 @@ struct IPAActiveState { /* 0..2 range, 1.0 = normal */ std::optional contrast; std::optional saturation; + /* 0..2 range, 1.0 = normal */ + std::optional brightness; } knobs; }; From 9f2edfa764612db4b8a6afb42ba8344ab78f24a1 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sun, 16 Mar 2025 22:12:10 +0300 Subject: [PATCH 37/42] libcamera: software_isp: Add AGC disable control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 11 +++++++++++ src/ipa/simple/ipa_context.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index 249caa264..de20863d8 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -44,6 +44,7 @@ Agc::Agc() int Agc::init(IPAContext &context, [[maybe_unused]] const YamlObject &tuningData) { + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); return 0; } @@ -52,6 +53,7 @@ int Agc::configure(IPAContext &context, [[maybe_unused]] const IPAConfigInfo &configInfo) { context.activeState.knobs.brightness = std::optional(); + context.activeState.knobs.ae_enabled = std::optional(); return 0; } @@ -62,10 +64,15 @@ void Agc::queueRequest(typename Module::Context &context, const ControlList &controls) { const auto &brightness = controls.get(controls::Brightness); + const auto &ae_enabled = controls.get(controls::AeEnable); if (brightness.has_value()) { context.activeState.knobs.brightness = brightness; LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); } + if (ae_enabled.has_value()) { + context.activeState.knobs.ae_enabled = ae_enabled; + LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); + } } void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) @@ -132,6 +139,10 @@ void Agc::process(IPAContext &context, const SwIspStats *stats, ControlList &metadata) { + const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); + if (!ae_enable) + return; + utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index afc28ba22..8e8add4c3 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -66,6 +66,8 @@ struct IPAActiveState { std::optional saturation; /* 0..2 range, 1.0 = normal */ std::optional brightness; + /* 0..1 range, 1 = normal */ + std::optional ae_enabled; } knobs; }; From 37aa6edbf3be46a718dc215061d75e3cf79bd4c5 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Mon, 17 Mar 2025 04:24:56 +0300 Subject: [PATCH 38/42] libcamera: software_isp: Add focus control Signed-off-by: Vasiliy Doylov --- .../internal/software_isp/software_isp.h | 4 +- include/libcamera/ipa/soft.mojom | 3 +- src/ipa/simple/algorithms/af.cpp | 71 +++++++++++++++++++ src/ipa/simple/algorithms/af.h | 40 +++++++++++ src/ipa/simple/algorithms/meson.build | 1 + src/ipa/simple/data/uncalibrated.yaml | 1 + src/ipa/simple/ipa_context.h | 9 +++ src/ipa/simple/soft_simple.cpp | 18 ++++- src/libcamera/pipeline/simple/simple.cpp | 30 ++++++-- src/libcamera/software_isp/software_isp.cpp | 4 +- 10 files changed, 169 insertions(+), 12 deletions(-) create mode 100644 src/ipa/simple/algorithms/af.cpp create mode 100644 src/ipa/simple/algorithms/af.h diff --git a/include/libcamera/internal/software_isp/software_isp.h b/include/libcamera/internal/software_isp/software_isp.h index ad89c9b3c..9e5e05fc0 100644 --- a/include/libcamera/internal/software_isp/software_isp.h +++ b/include/libcamera/internal/software_isp/software_isp.h @@ -86,11 +86,11 @@ public: Signal outputBufferReady; Signal ispStatsReady; Signal metadataReady; - Signal setSensorControls; + Signal setSensorControls; private: void saveIspParams(); - void setSensorCtrls(const ControlList &sensorControls); + void setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls); void statsReady(uint32_t frame, uint32_t bufferId); void inputReady(FrameBuffer *input); void outputReady(FrameBuffer *output); diff --git a/include/libcamera/ipa/soft.mojom b/include/libcamera/ipa/soft.mojom index 77328c5fd..e5767532c 100644 --- a/include/libcamera/ipa/soft.mojom +++ b/include/libcamera/ipa/soft.mojom @@ -10,6 +10,7 @@ import "include/libcamera/ipa/core.mojom"; struct IPAConfigInfo { libcamera.ControlInfoMap sensorControls; + libcamera.ControlInfoMap lensControls; }; interface IPASoftInterface { @@ -32,7 +33,7 @@ interface IPASoftInterface { }; interface IPASoftEventInterface { - setSensorControls(libcamera.ControlList sensorControls); + setSensorControls(libcamera.ControlList sensorControls, libcamera.ControlList lensControls); setIspParams(); metadataReady(uint32 frame, libcamera.ControlList metadata); }; diff --git a/src/ipa/simple/algorithms/af.cpp b/src/ipa/simple/algorithms/af.cpp new file mode 100644 index 000000000..b51ed95e4 --- /dev/null +++ b/src/ipa/simple/algorithms/af.cpp @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#include "af.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftAutoFocus) + +namespace ipa::soft::algorithms { + +Af::Af() +{ +} + +int Af::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::LensPosition] = ControlInfo(0.0f, 100.0f, 50.0f); + return 0; +} + +int Af::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.focus_pos = std::optional(); + + return 0; +} + +void Af::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &focus_pos = controls.get(controls::LensPosition); + if (focus_pos.has_value()) { + context.activeState.knobs.focus_pos = focus_pos; + LOG(IPASoftAutoFocus, Debug) << "Setting focus position to " << focus_pos.value(); + } +} + +void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAFrameContext &frameContext, [[maybe_unused]] double exposureMSV) +{ + frameContext.lens.focus_pos = context.activeState.knobs.focus_pos.value_or(50.0) / 100.0 * (context.configuration.focus.focus_max - context.configuration.focus.focus_min); +} + +void Af::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const SwIspStats *stats, + [[maybe_unused]] ControlList &metadata) +{ + updateFocus(context, frameContext, 0); +} + +REGISTER_IPA_ALGORITHM(Af, "Af") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/af.h b/src/ipa/simple/algorithms/af.h new file mode 100644 index 000000000..a575ef102 --- /dev/null +++ b/src/ipa/simple/algorithms/af.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Af : public Algorithm +{ +public: + Af(); + ~Af() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateFocus(IPAContext &context, IPAFrameContext &frameContext, double focus); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build index 2d0adb059..dec59ee8c 100644 --- a/src/ipa/simple/algorithms/meson.build +++ b/src/ipa/simple/algorithms/meson.build @@ -6,4 +6,5 @@ soft_simple_ipa_algorithms = files([ 'blc.cpp', 'ccm.cpp', 'lut.cpp', + 'af.cpp', ]) diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index 8b6df9afc..9274c11e1 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -16,4 +16,5 @@ algorithms: 0, 0, 1] - Lut: - Agc: + - Af: ... diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index 8e8add4c3..71b9bb637 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -34,6 +34,9 @@ struct IPASessionConfiguration { struct { std::optional level; } black; + struct { + int32_t focus_min, focus_max; + } focus; }; struct IPAActiveState { @@ -68,6 +71,8 @@ struct IPAActiveState { std::optional brightness; /* 0..1 range, 1 = normal */ std::optional ae_enabled; + /* 0..100 range, 50.0 = normal */ + std::optional focus_pos; } knobs; }; @@ -81,6 +86,10 @@ struct IPAFrameContext : public FrameContext { double gain; } sensor; + struct { + int32_t focus_pos; + } lens; + struct { double red; double blue; diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp index c94c4cd55..53fa74c03 100644 --- a/src/ipa/simple/soft_simple.cpp +++ b/src/ipa/simple/soft_simple.cpp @@ -77,6 +77,7 @@ private: SwIspStats *stats_; std::unique_ptr camHelper_; ControlInfoMap sensorInfoMap_; + ControlInfoMap lensInfoMap_; /* Local parameter storage */ struct IPAContext context_; @@ -196,6 +197,7 @@ int IPASoftSimple::init(const IPASettings &settings, int IPASoftSimple::configure(const IPAConfigInfo &configInfo) { sensorInfoMap_ = configInfo.sensorControls; + lensInfoMap_ = configInfo.lensControls; const ControlInfo &exposureInfo = sensorInfoMap_.find(V4L2_CID_EXPOSURE)->second; const ControlInfo &gainInfo = sensorInfoMap_.find(V4L2_CID_ANALOGUE_GAIN)->second; @@ -205,6 +207,17 @@ int IPASoftSimple::configure(const IPAConfigInfo &configInfo) context_.activeState = {}; context_.frameContexts.clear(); + if (lensInfoMap_.empty()) { + LOG(IPASoft, Warning) << "No camera leans found! Focus control disabled."; + context_.configuration.focus.focus_min = 0; + context_.configuration.focus.focus_max = 0; + } else { + const ControlInfo &lensInfo = lensInfoMap_.find(V4L2_CID_FOCUS_ABSOLUTE)->second; + context_.configuration.focus.focus_min = lensInfo.min().get(); + context_.configuration.focus.focus_max = lensInfo.max().get(); + LOG(IPASoft, Warning) << "Camera leans found! Focus: " << context_.configuration.focus.focus_min << "-" << context_.configuration.focus.focus_max; + } + context_.configuration.agc.lineDuration = context_.sensorInfo.minLineLength * 1.0s / context_.sensorInfo.pixelRate; context_.configuration.agc.exposureMin = exposureInfo.min().get(); @@ -327,7 +340,10 @@ void IPASoftSimple::processStats(const uint32_t frame, ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(camHelper_ ? camHelper_->gainCode(againNew) : againNew)); - setSensorControls.emit(ctrls); + ControlList lens_ctrls(lensInfoMap_); + lens_ctrls.set(V4L2_CID_FOCUS_ABSOLUTE, frameContext.lens.focus_pos); + + setSensorControls.emit(ctrls, lens_ctrls); } std::string IPASoftSimple::logPrefix() const diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp index efb07051b..4c002ca68 100644 --- a/src/libcamera/pipeline/simple/simple.cpp +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -30,6 +30,7 @@ #include #include "libcamera/internal/camera.h" +#include "libcamera/internal/camera_lens.h" #include "libcamera/internal/camera_sensor.h" #include "libcamera/internal/camera_sensor_properties.h" #include "libcamera/internal/converter.h" @@ -41,6 +42,8 @@ #include "libcamera/internal/v4l2_subdevice.h" #include "libcamera/internal/v4l2_videodevice.h" +#include "libcamera/controls.h" + namespace libcamera { LOG_DEFINE_CATEGORY(SimplePipeline) @@ -356,7 +359,7 @@ private: void ispStatsReady(uint32_t frame, uint32_t bufferId); void metadataReady(uint32_t frame, const ControlList &metadata); - void setSensorControls(const ControlList &sensorControls); + void setSensorControls(const ControlList &sensorControls, const ControlList &lensControls); }; class SimpleCameraConfiguration : public CameraConfiguration @@ -1002,7 +1005,7 @@ void SimpleCameraData::metadataReady(uint32_t frame, const ControlList &metadata tryCompleteRequest(info->request); } -void SimpleCameraData::setSensorControls(const ControlList &sensorControls) +void SimpleCameraData::setSensorControls(const ControlList &sensorControls, const ControlList &lensControls) { delayedCtrls_->push(sensorControls); /* @@ -1013,10 +1016,21 @@ void SimpleCameraData::setSensorControls(const ControlList &sensorControls) * but it also bypasses delayedCtrls_, creating AGC regulation issues. * Both problems should be fixed. */ - if (!frameStartEmitter_) { - ControlList ctrls(sensorControls); - sensor_->setControls(&ctrls); - } + if (frameStartEmitter_) + return; + + ControlList ctrls(sensorControls); + sensor_->setControls(&ctrls); + + CameraLens *focusLens = sensor_->focusLens(); + if (!focusLens) + return; + + if (!lensControls.contains(V4L2_CID_FOCUS_ABSOLUTE)) + return; + + const ControlValue &focusValue = lensControls.get(V4L2_CID_FOCUS_ABSOLUTE); + focusLens->setFocusPosition(focusValue.get()); } /* Retrieve all source pads connected to a sink pad through active routes. */ @@ -1406,6 +1420,10 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) } else { ipa::soft::IPAConfigInfo configInfo; configInfo.sensorControls = data->sensor_->controls(); + if (data->sensor_->focusLens() != nullptr) + configInfo.lensControls = data->sensor_->focusLens()->controls(); + else + configInfo.lensControls = ControlInfoMap(); return data->swIsp_->configure(inputCfg, outputCfgs, configInfo); } } diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index ff4471c05..0e4693b77 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -420,9 +420,9 @@ void SoftwareIsp::saveIspParams() debayerParams_ = *sharedParams_; } -void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls) +void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls) { - setSensorControls.emit(sensorControls); + setSensorControls.emit(sensorControls, lensControls); } void SoftwareIsp::statsReady(uint32_t frame, uint32_t bufferId) From 21501e6e538809d95c2b315f0645b733c21e5da1 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 21 May 2025 21:24:09 +0300 Subject: [PATCH 39/42] libcamera: software_isp: Add manual exposure control Signed-off-by: Vasiliy Doylov --- src/ipa/simple/algorithms/agc.cpp | 14 ++++++++++++-- src/ipa/simple/ipa_context.h | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index de20863d8..35f4798cf 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -46,6 +46,8 @@ int Agc::init(IPAContext &context, { context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + context.ctrlMap[&controls::ExposureValue] = ControlInfo(0.0f, 0.5f, 1.0f); + return 0; } @@ -65,6 +67,7 @@ void Agc::queueRequest(typename Module::Context &context, { const auto &brightness = controls.get(controls::Brightness); const auto &ae_enabled = controls.get(controls::AeEnable); + const auto &exposure_value = controls.get(controls::ExposureValue); if (brightness.has_value()) { context.activeState.knobs.brightness = brightness; LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); @@ -73,6 +76,10 @@ void Agc::queueRequest(typename Module::Context &context, context.activeState.knobs.ae_enabled = ae_enabled; LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); } + if (exposure_value.has_value()) { + context.activeState.knobs.exposure_value = exposure_value.value(); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << exposure_value.value(); + } } void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) @@ -140,13 +147,16 @@ void Agc::process(IPAContext &context, ControlList &metadata) { const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); - if (!ae_enable) - return; + if (!ae_enable) + frameContext.sensor.exposure = (int32_t)( context.activeState.knobs.exposure_value.value_or(0.5) * (context.configuration.agc.exposureMax - context.configuration.agc.exposureMin)); utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << frameContext.sensor.exposure; + if (!ae_enable) + return; /* * Calculate Mean Sample Value (MSV) according to formula from: diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index 71b9bb637..c5b5527b5 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -71,6 +71,8 @@ struct IPAActiveState { std::optional brightness; /* 0..1 range, 1 = normal */ std::optional ae_enabled; + /* 0..1 range, 0.5 = normal */ + std::optional exposure_value; /* 0..100 range, 50.0 = normal */ std::optional focus_pos; } knobs; From 8cf5d8fa3aadcd231211e2ede2827315964be508 Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 5 Apr 2025 14:59:17 +0300 Subject: [PATCH 40/42] gitignore: ignore my setup Signed-off-by: Vasiliy Doylov --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 51d314408..656f622f0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ *.pyc __pycache__/ venv/ +.vscode/ +.cache/ +compile_commands.json From 434c9d2a4b11b912ec7e5b0acc3494469d90063a Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Sat, 7 Jun 2025 02:22:30 +0300 Subject: [PATCH 41/42] CI: Add local forgejo CI Signed-off-by: Vasiliy Doylov --- .forgejo/workflows/build-alpine.yaml | 37 +++++++++ package/alpine/APKBUILD | 117 +++++++++++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 .forgejo/workflows/build-alpine.yaml create mode 100644 package/alpine/APKBUILD diff --git a/.forgejo/workflows/build-alpine.yaml b/.forgejo/workflows/build-alpine.yaml new file mode 100644 index 000000000..335b29ad6 --- /dev/null +++ b/.forgejo/workflows/build-alpine.yaml @@ -0,0 +1,37 @@ +name: PostmarketOS Build +run-name: PostmarketOS Build +on: + push: + workflow_dispatch: + +jobs: + build: + name: Build for ${{ matrix.info.arch }} + runs-on: Pmbootstrap + strategy: + matrix: + info: + - arch: x86_64 + - arch: aarch64 + steps: + - name: Remove libcamera aport + run: rm -rf ${{env.PMB_PMAPORTS}}/temp/libcamera + - name: Check out repository code + uses: actions/checkout@v4 + - name: Build packages + id: build + uses: actions/pmbootstrap-build@main + with: + name: libcamera + aports: ${{github.workspace}}/package/alpine + arch: ${{ matrix.info.arch }} + src: ${{github.workspace}} + - name: "Upload packages" + uses: actions/upload-alpine-package@main + with: + files: ${{steps.build.outputs.packages}} + secret: ${{secrets.PACKAGE_TOKEN}} + - name: Reset pmaports changes + if: always() + continue-on-error: true + run: git -C ${{env.PMB_PMAPORTS}} reset --hard diff --git a/package/alpine/APKBUILD b/package/alpine/APKBUILD new file mode 100644 index 000000000..93ede5b0f --- /dev/null +++ b/package/alpine/APKBUILD @@ -0,0 +1,117 @@ +pkgname=libcamera +pkgver=9999999 +pkgrel=0 +pkgdesc="Linux camera framework" +url="https://libcamera.org/" +arch="all" +license="LGPL-2.1-or-later AND GPL-2.0-or-later" +depends_dev=" + eudev-dev + glib-dev + gnutls-dev + gst-plugins-bad-dev + qt6-qtbase-dev + " + +makedepends="$depends_dev + coreutils + doxygen + graphviz + gtest-dev + libevent-dev + libpisp-dev + libunwind-dev + libyuv-dev + linux-headers + meson + py3-jinja2 + py3-ply + py3-sphinx + py3-yaml + qt6-qttools-dev + yaml-dev + " +subpackages=" + $pkgname-dbg + $pkgname-dev + $pkgname-doc + $pkgname-gstreamer + $pkgname-tools + " +source="" +builddir="$srcdir/$pkgname" +# gstreamer tests fail +# manual strip because ipa .sign files depend on the file contents- have to re-sign after strip +options="!strip !check" + +case "$CARCH" in +arm*|aarch64) + subpackages="$subpackages" + ;; +esac + +case "$CARCH" in +ppc64le|s390x|riscv64|loongarch64) + # doesn't install any ipa + ;; +*) + depends="$pkgname-ipa" + subpackages="$subpackages $pkgname-ipa" + ;; +esac + +build() { + abuild-meson \ + -Dtest=false \ + -Dv4l2=false \ + -Dwerror=false \ + -Dpipelines=simple \ + -Dipas=simple \ + . output + meson compile -C output +} + +check() { + meson test -C output --print-errorlogs +} + +package() { + DESTDIR="$pkgdir" meson install --no-rebuild -C output + + # manual strip first.. + scanelf --recursive \ + --nobanner \ + --etype "ET_DYN,ET_EXEC" \ + --format "%F" \ + "$pkgdir" \ + | while read -r file; do + strip "$file" + done +} + +ipa() { + depends="" + amove usr/lib/libcamera + # then sign ipa's + local ipa + for ipa in "$subpkgdir"/usr/lib/libcamera/ipa/ipa*.so; do + msg "signing $ipa" + "$builddir"/src/ipa/ipa-sign.sh \ + "$(find "$builddir"/output -type f -iname "*ipa-priv-key.pem")" \ + "$ipa" \ + "$ipa".sign + done +} + +gstreamer() { + depends="" + amove usr/lib/gstreamer-1.0 +} + +tools() { + depends="" + amove usr/bin/cam + amove usr/bin/lc-compliance +} + +sha512sums="" From a450911b98888abea6596e6595ae2e46b671506a Mon Sep 17 00:00:00 2001 From: Vasiliy Doylov Date: Wed, 11 Jun 2025 22:43:24 +0300 Subject: [PATCH 42/42] CI: change pkgname Signed-off-by: Vasiliy Doylov --- .forgejo/workflows/build-alpine.yaml | 2 +- package/alpine/APKBUILD | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/build-alpine.yaml b/.forgejo/workflows/build-alpine.yaml index 335b29ad6..5f779d3a9 100644 --- a/.forgejo/workflows/build-alpine.yaml +++ b/.forgejo/workflows/build-alpine.yaml @@ -22,7 +22,7 @@ jobs: id: build uses: actions/pmbootstrap-build@main with: - name: libcamera + name: libcamera-neko-gpu aports: ${{github.workspace}}/package/alpine arch: ${{ matrix.info.arch }} src: ${{github.workspace}} diff --git a/package/alpine/APKBUILD b/package/alpine/APKBUILD index 93ede5b0f..1f31843a2 100644 --- a/package/alpine/APKBUILD +++ b/package/alpine/APKBUILD @@ -1,4 +1,5 @@ -pkgname=libcamera +basepkgname=libcamera +pkgname=$basepkgname-neko-gpu pkgver=9999999 pkgrel=0 pkgdesc="Linux camera framework" @@ -38,6 +39,14 @@ subpackages=" $pkgname-gstreamer $pkgname-tools " +provides=" + $basepkgname + $basepkgname-dbg + $basepkgname-dev + $basepkgname-doc + $basepkgname-gstreamer + $basepkgname-tools +" source="" builddir="$srcdir/$pkgname" # gstreamer tests fail