diff --git a/.forgejo/workflows/build-alpine.yaml b/.forgejo/workflows/build-alpine.yaml new file mode 100644 index 000000000..5f779d3a9 --- /dev/null +++ b/.forgejo/workflows/build-alpine.yaml @@ -0,0 +1,37 @@ +name: PostmarketOS Build +run-name: PostmarketOS Build +on: + push: + workflow_dispatch: + +jobs: + build: + name: Build for ${{ matrix.info.arch }} + runs-on: Pmbootstrap + strategy: + matrix: + info: + - arch: x86_64 + - arch: aarch64 + steps: + - name: Remove libcamera aport + run: rm -rf ${{env.PMB_PMAPORTS}}/temp/libcamera + - name: Check out repository code + uses: actions/checkout@v4 + - name: Build packages + id: build + uses: actions/pmbootstrap-build@main + with: + name: libcamera-neko-gpu + aports: ${{github.workspace}}/package/alpine + arch: ${{ matrix.info.arch }} + src: ${{github.workspace}} + - name: "Upload packages" + uses: actions/upload-alpine-package@main + with: + files: ${{steps.build.outputs.packages}} + secret: ${{secrets.PACKAGE_TOKEN}} + - name: Reset pmaports changes + if: always() + continue-on-error: true + run: git -C ${{env.PMB_PMAPORTS}} reset --hard diff --git a/.gitignore b/.gitignore index 51d314408..656f622f0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ *.pyc __pycache__/ venv/ +.vscode/ +.cache/ +compile_commands.json diff --git a/include/libcamera/internal/egl.h b/include/libcamera/internal/egl.h new file mode 100644 index 000000000..04d637d88 --- /dev/null +++ b/include/libcamera/internal/egl.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl_context.cpp - Helper class for managing eGL interactions. + */ + +#pragma once + +#include + +#include + +#include "libcamera/internal/gbm.h" + +#define EGL_EGLEXT_PROTOTYPES +#include +#include +#define GL_GLEXT_PROTOTYPES +#include +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(eGL) + +class eGLImage +{ +public: + eGLImage(uint32_t width, uint32_t height, uint32_t bpp, GLenum texture_unit, uint32_t texture_unit_uniform_id) + { + image_ = EGL_NO_IMAGE_KHR; + width_ = width; + height_ = height; + bpp_ = bpp; + stride_ = width_ * bpp_ / 4; + framesize_ = stride_ * height_; + texture_unit_ = texture_unit; + texture_unit_uniform_id_ = texture_unit_uniform_id; + + glGenTextures(1, &texture_); + } + + ~eGLImage() + { + glDeleteTextures(1, &texture_); + } + + uint32_t width_; + uint32_t height_; + uint32_t stride_; + uint32_t offset_; + uint32_t framesize_; + uint32_t bpp_; + uint32_t texture_unit_uniform_id_; + GLenum texture_unit_; + GLuint texture_; + EGLImageKHR image_; +}; + +class eGL +{ +public: + eGL(); + ~eGL(); + + int initEGLContext(GBM *gbmContext); + int createDMABufTexture2D(eGLImage *eglImage, int fd); + void destroyDMABufTexture(eGLImage *eglImage); + void createTexture2D(eGLImage *eglImage, GLint format, uint32_t width, uint32_t height, void *data); + void createTexture1D(eGLImage *eglImage, GLint format, uint32_t width, void *data); + + void pushEnv(std::vector &shaderEnv, const char *str); + void makeCurrent(); + void swapBuffers(); + + int compileVertexShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + int compileFragmentShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + int linkProgram(GLuint &programIdd, GLuint fragmentshaderId, GLuint vertexshaderId); + void dumpShaderSource(GLuint shaderId); + void useProgram(GLuint programId); + +private: + int fd_; + + EGLDisplay display_; + EGLContext context_; + EGLSurface surface_; + + int compileShader(int shaderType, GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv); + + PFNEGLEXPORTDMABUFIMAGEMESAPROC eglExportDMABUFImageMESA; + PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOES; + + PFNEGLCREATEIMAGEKHRPROC eglCreateImageKHR; + PFNEGLDESTROYIMAGEKHRPROC eglDestroyImageKHR; + + PFNEGLCLIENTWAITSYNCKHRPROC eglClientWaitSyncKHR; + PFNEGLCREATESYNCKHRPROC eglCreateSyncKHR; +}; +} //namespace libcamera diff --git a/include/libcamera/internal/gbm.h b/include/libcamera/internal/gbm.h new file mode 100644 index 000000000..a5486cc94 --- /dev/null +++ b/include/libcamera/internal/gbm.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * gbm.h - Helper class for managing GBM interactions. + */ + +#pragma once + +#include + +#include + +#include + +namespace libcamera { + +LOG_DECLARE_CATEGORY(GBM) + +class GBM +{ +public: + GBM(); + ~GBM(); + + int initSurface(uint32_t width, uint32_t height); + int mapSurface(); + int getFrameBufferData(uint8_t *data_out, size_t data_len); + struct gbm_device *getDevice() { return gbm_device_; } + struct gbm_surface *getSurface() { return gbm_surface_; } + uint32_t getFrameSize() { return framesize_; } + uint32_t getStride() { return stride_; } + PixelFormat getPixelFormat() { return format_; } + +private: + int fd_; + struct gbm_device *gbm_device_; + struct gbm_surface *gbm_surface_; + + struct gbm_bo *gbm_bo_; + uint32_t width_; + uint32_t height_; + uint32_t stride_; + uint32_t offset_; + uint32_t framesize_; + void *map_; + int bo_fd_; + + PixelFormat format_; +}; + +} // namespace libcamera diff --git a/include/libcamera/internal/mapped_framebuffer.h b/include/libcamera/internal/mapped_framebuffer.h index 6aaabf508..9a5355c76 100644 --- a/include/libcamera/internal/mapped_framebuffer.h +++ b/include/libcamera/internal/mapped_framebuffer.h @@ -55,6 +55,10 @@ public: using MapFlags = Flags; MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags); + int getPlaneFD(int plane); + +private: + const FrameBuffer *buffer_; }; LIBCAMERA_FLAGS_ENABLE_OPERATORS(MappedFrameBuffer::MapFlag) diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index 33f318b2b..4a2919f61 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -1,6 +1,7 @@ # SPDX-License-Identifier: CC0-1.0 subdir('tracepoints') +subdir('shaders') libcamera_internal_headers = files([ 'bayer_format.h', @@ -22,6 +23,7 @@ libcamera_internal_headers = files([ 'dma_buf_allocator.h', 'formats.h', 'framebuffer.h', + 'gbm.h', 'ipa_data_serializer.h', 'ipa_manager.h', 'ipa_module.h', @@ -57,5 +59,14 @@ tracepoints_h = custom_target( libcamera_internal_headers += tracepoints_h +libcamera_shader_headers = custom_target( + 'gen-shader-headers', + input : [shader_files], + output : 'glsl_shaders.h', + command : [gen_shader_headers, meson.project_source_root(), meson.project_build_root(), '@OUTPUT@', '@INPUT@'], +) + +libcamera_internal_headers += libcamera_shader_headers + subdir('converter') subdir('software_isp') diff --git a/src/apps/qcam/assets/shader/RGB.frag b/include/libcamera/internal/shaders/RGB.frag similarity index 93% rename from src/apps/qcam/assets/shader/RGB.frag rename to include/libcamera/internal/shaders/RGB.frag index 4c374ac98..724395894 100644 --- a/src/apps/qcam/assets/shader/RGB.frag +++ b/include/libcamera/internal/shaders/RGB.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/src/apps/qcam/assets/shader/YUV_2_planes.frag b/include/libcamera/internal/shaders/YUV_2_planes.frag similarity index 97% rename from src/apps/qcam/assets/shader/YUV_2_planes.frag rename to include/libcamera/internal/shaders/YUV_2_planes.frag index 1d5d12062..d286f1179 100644 --- a/src/apps/qcam/assets/shader/YUV_2_planes.frag +++ b/include/libcamera/internal/shaders/YUV_2_planes.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/src/apps/qcam/assets/shader/YUV_3_planes.frag b/include/libcamera/internal/shaders/YUV_3_planes.frag similarity index 96% rename from src/apps/qcam/assets/shader/YUV_3_planes.frag rename to include/libcamera/internal/shaders/YUV_3_planes.frag index 8f788e90a..8e3e0b4a5 100644 --- a/src/apps/qcam/assets/shader/YUV_3_planes.frag +++ b/include/libcamera/internal/shaders/YUV_3_planes.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/src/apps/qcam/assets/shader/YUV_packed.frag b/include/libcamera/internal/shaders/YUV_packed.frag similarity index 99% rename from src/apps/qcam/assets/shader/YUV_packed.frag rename to include/libcamera/internal/shaders/YUV_packed.frag index b9ef9d41b..3c9e3e397 100644 --- a/src/apps/qcam/assets/shader/YUV_packed.frag +++ b/include/libcamera/internal/shaders/YUV_packed.frag @@ -6,7 +6,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif varying vec2 textureOut; diff --git a/src/apps/qcam/assets/shader/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag similarity index 76% rename from src/apps/qcam/assets/shader/bayer_1x_packed.frag rename to include/libcamera/internal/shaders/bayer_1x_packed.frag index f53f55758..c0632eb1f 100644 --- a/src/apps/qcam/assets/shader/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -20,7 +20,7 @@ */ #ifdef GL_ES -precision mediump float; +precision highp float; #endif /* @@ -65,6 +65,10 @@ uniform vec2 tex_step; uniform vec2 tex_bayer_first_red; uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; +uniform mat3 ccm; void main(void) { @@ -212,5 +216,61 @@ void main(void) vec3(patterns.y, C, patterns.x) : vec3(patterns.wz, C)); +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; +#endif + +#if defined (SWAP_BLUE) + gl_FragColor = vec4(rgb.bgr, 1.0); +#else gl_FragColor = vec4(rgb, 1.0); +#endif } diff --git a/src/apps/qcam/assets/shader/bayer_8.frag b/include/libcamera/internal/shaders/bayer_unpacked.frag similarity index 56% rename from src/apps/qcam/assets/shader/bayer_8.frag rename to include/libcamera/internal/shaders/bayer_unpacked.frag index 7e35ca88e..78c2609c2 100644 --- a/src/apps/qcam/assets/shader/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_unpacked.frag @@ -16,19 +16,33 @@ Copyright (C) 2021, Linaro //Pixel Shader #ifdef GL_ES -precision mediump float; +precision highp float; #endif /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; varying vec4 center; varying vec4 yCoord; varying vec4 xCoord; +uniform mat3 ccm; void main(void) { - #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r + vec3 rgb; - float C = texture2D(tex_y, center.xy).r; // ( 0, 0) + #if defined(RAW10P) + #define pixel(p) p.r / 4.0 + p.g * 64.0 + #define fetch(x, y) pixel(texture2D(tex_y, vec2(x, y))) + #elif defined(RAW12P) + #define pixel(p) p.r / 16.0 + p.g * 16.0 + #define fetch(x, y) pixel(texture2D(tex_y, vec2(x, y))) + #else + #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r + #endif + + float C = fetch(center.x, center.y); // ( 0, 0) const vec4 kC = vec4( 4.0, 6.0, 5.0, 5.0) / 8.0; // Determine which of four types of pixels we are on. @@ -97,11 +111,69 @@ void main(void) { PATTERN.xw += kB.xw * B; PATTERN.xz += kF.xz * F; - gl_FragColor.rgb = (alternate.y == 0.0) ? + rgb = (alternate.y == 0.0) ? ((alternate.x == 0.0) ? vec3(C, PATTERN.xy) : vec3(PATTERN.z, C, PATTERN.w)) : ((alternate.x == 0.0) ? vec3(PATTERN.w, C, PATTERN.z) : vec3(PATTERN.yx, C)); + +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; +#endif + +#if defined (SWAP_BLUE) + gl_FragColor = vec4(rgb.bgr, 1.0); +#else + gl_FragColor = vec4(rgb, 1.0); +#endif } diff --git a/src/apps/qcam/assets/shader/bayer_8.vert b/include/libcamera/internal/shaders/bayer_unpacked.vert similarity index 85% rename from src/apps/qcam/assets/shader/bayer_8.vert rename to include/libcamera/internal/shaders/bayer_unpacked.vert index fb5109eee..fc1cf89f2 100644 --- a/src/apps/qcam/assets/shader/bayer_8.vert +++ b/include/libcamera/internal/shaders/bayer_unpacked.vert @@ -44,10 +44,10 @@ void main(void) { center.xy = textureIn; center.zw = textureIn * tex_size + tex_bayer_first_red; - xCoord = center.x + vec4(-2.0 * tex_step.x, - -tex_step.x, tex_step.x, 2.0 * tex_step.x); - yCoord = center.y + vec4(-2.0 * tex_step.y, - -tex_step.y, tex_step.y, 2.0 * tex_step.y); + xCoord = center.x + 0.1 * tex_step.x + + vec4(-2.0 * tex_step.x, -tex_step.x, tex_step.x, 2.0 * tex_step.x); + yCoord = center.y + 0.1 * tex_step.y + + vec4(-2.0 * tex_step.y, -tex_step.y, tex_step.y, 2.0 * tex_step.y); gl_Position = proj_matrix * vertexIn; } diff --git a/src/apps/qcam/assets/shader/identity.vert b/include/libcamera/internal/shaders/identity.vert similarity index 100% rename from src/apps/qcam/assets/shader/identity.vert rename to include/libcamera/internal/shaders/identity.vert diff --git a/include/libcamera/internal/shaders/meson.build b/include/libcamera/internal/shaders/meson.build new file mode 100644 index 000000000..dd441a577 --- /dev/null +++ b/include/libcamera/internal/shaders/meson.build @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: CC0-1.0 + +# List of shader files to convert to header hex +# for the purposes of inclusion in OpenGL debayering +shader_files = files([ + 'bayer_1x_packed.frag', + 'bayer_unpacked.frag', + 'bayer_unpacked.vert', + 'identity.vert', +]) diff --git a/include/libcamera/internal/software_isp/benchmark.h b/include/libcamera/internal/software_isp/benchmark.h new file mode 100644 index 000000000..8af250154 --- /dev/null +++ b/include/libcamera/internal/software_isp/benchmark.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Authors: + * Hans de Goede + * + * Simple builtin benchmark to measure software ISP processing times + */ + +#pragma once + +#include +#include + +namespace libcamera { + +class Benchmark +{ +public: + Benchmark(); + ~Benchmark(); + + void startFrame(void); + void finishFrame(void); + +private: + unsigned int measuredFrames_; + int64_t frameProcessTime_; + timespec frameStartTime_; + /* Skip 30 frames for things to stabilize then measure 30 frames */ + static constexpr unsigned int kFramesToSkip = 30; + static constexpr unsigned int kLastFrameToMeasure = 60; +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/software_isp/debayer_params.h b/include/libcamera/internal/software_isp/debayer_params.h index 217cd5d92..56803195e 100644 --- a/include/libcamera/internal/software_isp/debayer_params.h +++ b/include/libcamera/internal/software_isp/debayer_params.h @@ -13,6 +13,8 @@ #include #include +#include "libcamera/internal/matrix.h" + namespace libcamera { struct DebayerParams { @@ -49,6 +51,11 @@ struct DebayerParams { CcmLookupTable greenCcm; CcmLookupTable blueCcm; LookupTable gammaLut; + + /* + * Per frame CCM values as calcualted by the IPA + */ + Matrix ccm; }; } /* namespace libcamera */ diff --git a/include/libcamera/internal/software_isp/meson.build b/include/libcamera/internal/software_isp/meson.build index 508ddddca..df7c3b97d 100644 --- a/include/libcamera/internal/software_isp/meson.build +++ b/include/libcamera/internal/software_isp/meson.build @@ -1,7 +1,9 @@ # SPDX-License-Identifier: CC0-1.0 libcamera_internal_headers += files([ + 'benchmark.h', 'debayer_params.h', 'software_isp.h', 'swisp_stats.h', + 'swstats_cpu.h', ]) diff --git a/include/libcamera/internal/software_isp/software_isp.h b/include/libcamera/internal/software_isp/software_isp.h index 786246592..9e5e05fc0 100644 --- a/include/libcamera/internal/software_isp/software_isp.h +++ b/include/libcamera/internal/software_isp/software_isp.h @@ -37,7 +37,7 @@ namespace libcamera { -class DebayerCpu; +class Debayer; class FrameBuffer; class PixelFormat; class Stream; @@ -86,16 +86,15 @@ public: Signal outputBufferReady; Signal ispStatsReady; Signal metadataReady; - Signal setSensorControls; + Signal setSensorControls; private: void saveIspParams(); - void setSensorCtrls(const ControlList &sensorControls); + void setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls); void statsReady(uint32_t frame, uint32_t bufferId); void inputReady(FrameBuffer *input); void outputReady(FrameBuffer *output); - - std::unique_ptr debayer_; + std::unique_ptr debayer_; Thread ispWorkerThread_; SharedMemObject sharedParams_; DebayerParams debayerParams_; diff --git a/src/libcamera/software_isp/swstats_cpu.h b/include/libcamera/internal/software_isp/swstats_cpu.h similarity index 86% rename from src/libcamera/software_isp/swstats_cpu.h rename to include/libcamera/internal/software_isp/swstats_cpu.h index 26a2f462e..fa47cec91 100644 --- a/src/libcamera/software_isp/swstats_cpu.h +++ b/include/libcamera/internal/software_isp/swstats_cpu.h @@ -18,12 +18,16 @@ #include #include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/framebuffer.h" #include "libcamera/internal/shared_mem_object.h" #include "libcamera/internal/software_isp/swisp_stats.h" +#include "benchmark.h" + namespace libcamera { class PixelFormat; +class MappedFrameBuffer; struct StreamConfiguration; class SwStatsCpu @@ -42,6 +46,7 @@ public: void setWindow(const Rectangle &window); void startFrame(); void finishFrame(uint32_t frame, uint32_t bufferId); + void processFrame(uint32_t frame, uint32_t bufferId, FrameBuffer *input); void processLine0(unsigned int y, const uint8_t *src[]) { @@ -65,6 +70,7 @@ public: private: using statsProcessFn = void (SwStatsCpu::*)(const uint8_t *src[]); + using processFrameFn = void (SwStatsCpu::*)(MappedFrameBuffer &in); int setupStandardBayerOrder(BayerFormat::Order order); /* Bayer 8 bpp unpacked */ @@ -77,6 +83,10 @@ private: void statsBGGR10PLine0(const uint8_t *src[]); void statsGBRG10PLine0(const uint8_t *src[]); + void processBayerFrame2(MappedFrameBuffer &in); + + processFrameFn processFrame_; + /* Variables set by configure(), used every line */ statsProcessFn stats0_; statsProcessFn stats2_; @@ -89,9 +99,11 @@ private: Size patternSize_; unsigned int xShift_; + unsigned int stride_; SharedMemObject sharedStats_; SwIspStats stats_; + Benchmark bench_; }; } /* namespace libcamera */ diff --git a/include/libcamera/ipa/soft.mojom b/include/libcamera/ipa/soft.mojom index 77328c5fd..e5767532c 100644 --- a/include/libcamera/ipa/soft.mojom +++ b/include/libcamera/ipa/soft.mojom @@ -10,6 +10,7 @@ import "include/libcamera/ipa/core.mojom"; struct IPAConfigInfo { libcamera.ControlInfoMap sensorControls; + libcamera.ControlInfoMap lensControls; }; interface IPASoftInterface { @@ -32,7 +33,7 @@ interface IPASoftInterface { }; interface IPASoftEventInterface { - setSensorControls(libcamera.ControlList sensorControls); + setSensorControls(libcamera.ControlList sensorControls, libcamera.ControlList lensControls); setIspParams(); metadataReady(uint32 frame, libcamera.ControlList metadata); }; diff --git a/package/alpine/APKBUILD b/package/alpine/APKBUILD new file mode 100644 index 000000000..1f31843a2 --- /dev/null +++ b/package/alpine/APKBUILD @@ -0,0 +1,126 @@ +basepkgname=libcamera +pkgname=$basepkgname-neko-gpu +pkgver=9999999 +pkgrel=0 +pkgdesc="Linux camera framework" +url="https://libcamera.org/" +arch="all" +license="LGPL-2.1-or-later AND GPL-2.0-or-later" +depends_dev=" + eudev-dev + glib-dev + gnutls-dev + gst-plugins-bad-dev + qt6-qtbase-dev + " + +makedepends="$depends_dev + coreutils + doxygen + graphviz + gtest-dev + libevent-dev + libpisp-dev + libunwind-dev + libyuv-dev + linux-headers + meson + py3-jinja2 + py3-ply + py3-sphinx + py3-yaml + qt6-qttools-dev + yaml-dev + " +subpackages=" + $pkgname-dbg + $pkgname-dev + $pkgname-doc + $pkgname-gstreamer + $pkgname-tools + " +provides=" + $basepkgname + $basepkgname-dbg + $basepkgname-dev + $basepkgname-doc + $basepkgname-gstreamer + $basepkgname-tools +" +source="" +builddir="$srcdir/$pkgname" +# gstreamer tests fail +# manual strip because ipa .sign files depend on the file contents- have to re-sign after strip +options="!strip !check" + +case "$CARCH" in +arm*|aarch64) + subpackages="$subpackages" + ;; +esac + +case "$CARCH" in +ppc64le|s390x|riscv64|loongarch64) + # doesn't install any ipa + ;; +*) + depends="$pkgname-ipa" + subpackages="$subpackages $pkgname-ipa" + ;; +esac + +build() { + abuild-meson \ + -Dtest=false \ + -Dv4l2=false \ + -Dwerror=false \ + -Dpipelines=simple \ + -Dipas=simple \ + . output + meson compile -C output +} + +check() { + meson test -C output --print-errorlogs +} + +package() { + DESTDIR="$pkgdir" meson install --no-rebuild -C output + + # manual strip first.. + scanelf --recursive \ + --nobanner \ + --etype "ET_DYN,ET_EXEC" \ + --format "%F" \ + "$pkgdir" \ + | while read -r file; do + strip "$file" + done +} + +ipa() { + depends="" + amove usr/lib/libcamera + # then sign ipa's + local ipa + for ipa in "$subpkgdir"/usr/lib/libcamera/ipa/ipa*.so; do + msg "signing $ipa" + "$builddir"/src/ipa/ipa-sign.sh \ + "$(find "$builddir"/output -type f -iname "*ipa-priv-key.pem")" \ + "$ipa" \ + "$ipa".sign + done +} + +gstreamer() { + depends="" + amove usr/lib/gstreamer-1.0 +} + +tools() { + depends="" + amove usr/bin/cam + amove usr/bin/lc-compliance +} + +sha512sums="" diff --git a/src/apps/qcam/assets/shader/shaders.qrc b/src/apps/qcam/assets/shader/shaders.qrc index 96c709f92..32dfa51bf 100644 --- a/src/apps/qcam/assets/shader/shaders.qrc +++ b/src/apps/qcam/assets/shader/shaders.qrc @@ -1,13 +1,13 @@ - RGB.frag - YUV_2_planes.frag - YUV_3_planes.frag - YUV_packed.frag - bayer_1x_packed.frag - bayer_8.frag - bayer_8.vert - identity.vert + ../../../../../include/libcamera/internal/shaders/RGB.frag + ../../../../../include/libcamera/internal/shaders/YUV_2_planes.frag + ../../../../../include/libcamera/internal/shaders/YUV_3_planes.frag + ../../../../../include/libcamera/internal/shaders/YUV_packed.frag + ../../../../../include/libcamera/internal/shaders/bayer_1x_packed.frag + ../../../../../include/libcamera/internal/shaders/bayer_unpacked.frag + ../../../../../include/libcamera/internal/shaders/bayer_unpacked.vert + ../../../../../include/libcamera/internal/shaders/identity.vert diff --git a/src/apps/qcam/viewfinder_gl.cpp b/src/apps/qcam/viewfinder_gl.cpp index f31956ff0..95965ab71 100644 --- a/src/apps/qcam/viewfinder_gl.cpp +++ b/src/apps/qcam/viewfinder_gl.cpp @@ -141,7 +141,7 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) textureMinMagFilters_ = GL_LINEAR; /* Use identity.vert as the default vertex shader. */ - vertexShaderFile_ = ":identity.vert"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/identity.vert"; fragmentShaderDefines_.clear(); @@ -150,170 +150,170 @@ bool ViewFinderGL::selectFormat(const libcamera::PixelFormat &format) horzSubSample_ = 2; vertSubSample_ = 2; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV21: horzSubSample_ = 2; vertSubSample_ = 2; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV16: horzSubSample_ = 2; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV61: horzSubSample_ = 2; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV24: horzSubSample_ = 1; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_UV"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::NV42: horzSubSample_ = 1; vertSubSample_ = 1; fragmentShaderDefines_.append("#define YUV_PATTERN_VU"); - fragmentShaderFile_ = ":YUV_2_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_2_planes.frag"; break; case libcamera::formats::YUV420: horzSubSample_ = 2; vertSubSample_ = 2; - fragmentShaderFile_ = ":YUV_3_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_3_planes.frag"; break; case libcamera::formats::YVU420: horzSubSample_ = 2; vertSubSample_ = 2; - fragmentShaderFile_ = ":YUV_3_planes.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_3_planes.frag"; break; case libcamera::formats::UYVY: fragmentShaderDefines_.append("#define YUV_PATTERN_UYVY"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::VYUY: fragmentShaderDefines_.append("#define YUV_PATTERN_VYUY"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::YUYV: fragmentShaderDefines_.append("#define YUV_PATTERN_YUYV"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::YVYU: fragmentShaderDefines_.append("#define YUV_PATTERN_YVYU"); - fragmentShaderFile_ = ":YUV_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/YUV_packed.frag"; break; case libcamera::formats::ABGR8888: fragmentShaderDefines_.append("#define RGB_PATTERN rgb"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::ARGB8888: fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::BGRA8888: fragmentShaderDefines_.append("#define RGB_PATTERN gba"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::RGBA8888: fragmentShaderDefines_.append("#define RGB_PATTERN abg"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::BGR888: fragmentShaderDefines_.append("#define RGB_PATTERN rgb"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::RGB888: fragmentShaderDefines_.append("#define RGB_PATTERN bgr"); - fragmentShaderFile_ = ":RGB.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/RGB.frag"; break; case libcamera::formats::SBGGR8: firstRed_.setX(1.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG8: firstRed_.setX(0.0); firstRed_.setY(1.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG8: firstRed_.setX(1.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB8: firstRed_.setX(0.0); firstRed_.setY(0.0); - vertexShaderFile_ = ":bayer_8.vert"; - fragmentShaderFile_ = ":bayer_8.frag"; + vertexShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.vert"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_unpacked.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SBGGR10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG10_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB10_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW10P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SBGGR12_CSI2P: firstRed_.setX(1.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGBRG12_CSI2P: firstRed_.setX(0.0); firstRed_.setY(1.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SGRBG12_CSI2P: firstRed_.setX(1.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; case libcamera::formats::SRGGB12_CSI2P: firstRed_.setX(0.0); firstRed_.setY(0.0); fragmentShaderDefines_.append("#define RAW12P"); - fragmentShaderFile_ = ":bayer_1x_packed.frag"; + fragmentShaderFile_ = ":include/libcamera/internal/shaders/bayer_1x_packed.frag"; textureMinMagFilters_ = GL_NEAREST; break; default: diff --git a/src/ipa/simple/algorithms/af.cpp b/src/ipa/simple/algorithms/af.cpp new file mode 100644 index 000000000..b51ed95e4 --- /dev/null +++ b/src/ipa/simple/algorithms/af.cpp @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#include "af.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftAutoFocus) + +namespace ipa::soft::algorithms { + +Af::Af() +{ +} + +int Af::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::LensPosition] = ControlInfo(0.0f, 100.0f, 50.0f); + return 0; +} + +int Af::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.focus_pos = std::optional(); + + return 0; +} + +void Af::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &focus_pos = controls.get(controls::LensPosition); + if (focus_pos.has_value()) { + context.activeState.knobs.focus_pos = focus_pos; + LOG(IPASoftAutoFocus, Debug) << "Setting focus position to " << focus_pos.value(); + } +} + +void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAFrameContext &frameContext, [[maybe_unused]] double exposureMSV) +{ + frameContext.lens.focus_pos = context.activeState.knobs.focus_pos.value_or(50.0) / 100.0 * (context.configuration.focus.focus_max - context.configuration.focus.focus_min); +} + +void Af::process([[maybe_unused]] IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] IPAFrameContext &frameContext, + [[maybe_unused]] const SwIspStats *stats, + [[maybe_unused]] ControlList &metadata) +{ + updateFocus(context, frameContext, 0); +} + +REGISTER_IPA_ALGORITHM(Af, "Af") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/af.h b/src/ipa/simple/algorithms/af.h new file mode 100644 index 000000000..a575ef102 --- /dev/null +++ b/src/ipa/simple/algorithms/af.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Af : public Algorithm +{ +public: + Af(); + ~Af() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateFocus(IPAContext &context, IPAFrameContext &frameContext, double focus); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index c46bb0ebe..35f4798cf 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -41,6 +41,47 @@ Agc::Agc() { } +int Agc::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); + context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + context.ctrlMap[&controls::ExposureValue] = ControlInfo(0.0f, 0.5f, 1.0f); + + return 0; +} + +int Agc::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.brightness = std::optional(); + context.activeState.knobs.ae_enabled = std::optional(); + + return 0; +} + +void Agc::queueRequest(typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &brightness = controls.get(controls::Brightness); + const auto &ae_enabled = controls.get(controls::AeEnable); + const auto &exposure_value = controls.get(controls::ExposureValue); + if (brightness.has_value()) { + context.activeState.knobs.brightness = brightness; + LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); + } + if (ae_enabled.has_value()) { + context.activeState.knobs.ae_enabled = ae_enabled; + LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); + } + if (exposure_value.has_value()) { + context.activeState.knobs.exposure_value = exposure_value.value(); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << exposure_value.value(); + } +} + void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) { /* @@ -54,6 +95,8 @@ void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, dou double next; int32_t &exposure = frameContext.sensor.exposure; double &again = frameContext.sensor.gain; + const auto brightness = context.activeState.knobs.brightness.value_or(1.0); + exposureMSV /= brightness; if (exposureMSV < kExposureOptimal - kExposureSatisfactory) { next = exposure * kExpNumeratorUp / kExpDenominator; @@ -103,10 +146,17 @@ void Agc::process(IPAContext &context, const SwIspStats *stats, ControlList &metadata) { + const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); + + if (!ae_enable) + frameContext.sensor.exposure = (int32_t)( context.activeState.knobs.exposure_value.value_or(0.5) * (context.configuration.agc.exposureMax - context.configuration.agc.exposureMin)); utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << frameContext.sensor.exposure; + if (!ae_enable) + return; /* * Calculate Mean Sample Value (MSV) according to formula from: diff --git a/src/ipa/simple/algorithms/agc.h b/src/ipa/simple/algorithms/agc.h index 112d9f5a1..00bc101e7 100644 --- a/src/ipa/simple/algorithms/agc.h +++ b/src/ipa/simple/algorithms/agc.h @@ -19,6 +19,13 @@ public: Agc(); ~Agc() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; void process(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const SwIspStats *stats, diff --git a/src/ipa/simple/algorithms/ccm.cpp b/src/ipa/simple/algorithms/ccm.cpp index 0a98406c1..cb023878d 100644 --- a/src/ipa/simple/algorithms/ccm.cpp +++ b/src/ipa/simple/algorithms/ccm.cpp @@ -14,6 +14,7 @@ #include #include "libcamera/internal/matrix.h" +#include "libcamera/internal/software_isp/debayer_params.h" namespace { @@ -84,7 +85,7 @@ void Ccm::applySaturation(Matrix &ccm, float saturation) } void Ccm::prepare(IPAContext &context, const uint32_t frame, - IPAFrameContext &frameContext, [[maybe_unused]] DebayerParams *params) + IPAFrameContext &frameContext, DebayerParams *params) { auto &saturation = context.activeState.knobs.saturation; @@ -108,6 +109,7 @@ void Ccm::prepare(IPAContext &context, const uint32_t frame, context.activeState.ccm.ccm = ccm; frameContext.ccm.ccm = ccm; frameContext.saturation = saturation; + params->ccm = ccm; context.activeState.ccm.changed = true; } diff --git a/src/ipa/simple/algorithms/lut.cpp b/src/ipa/simple/algorithms/lut.cpp index d1d5f7271..a161adb1a 100644 --- a/src/ipa/simple/algorithms/lut.cpp +++ b/src/ipa/simple/algorithms/lut.cpp @@ -126,6 +126,7 @@ void Lut::prepare(IPAContext &context, auto &red = params->redCcm; auto &green = params->greenCcm; auto &blue = params->blueCcm; + params->ccm = ccm; for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { red[i].r = ccmValue(i, ccm[0][0]); red[i].g = ccmValue(i, ccm[1][0]); diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build index 2d0adb059..dec59ee8c 100644 --- a/src/ipa/simple/algorithms/meson.build +++ b/src/ipa/simple/algorithms/meson.build @@ -6,4 +6,5 @@ soft_simple_ipa_algorithms = files([ 'blc.cpp', 'ccm.cpp', 'lut.cpp', + 'af.cpp', ]) diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index 5508e6686..9274c11e1 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -8,12 +8,13 @@ algorithms: # Color correction matrices can be defined here. The CCM algorithm # has a significant performance impact, and should only be enabled # if tuned. - # - Ccm: - # ccms: - # - ct: 6500 - # ccm: [ 1, 0, 0, - # 0, 1, 0, - # 0, 0, 1] + - Ccm: + ccms: + - ct: 6500 + ccm: [ 1, 0, 0, + 0, 1, 0, + 0, 0, 1] - Lut: - Agc: + - Af: ... diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index a471b80ae..c5b5527b5 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -34,6 +34,9 @@ struct IPASessionConfiguration { struct { std::optional level; } black; + struct { + int32_t focus_min, focus_max; + } focus; }; struct IPAActiveState { @@ -64,6 +67,14 @@ struct IPAActiveState { /* 0..2 range, 1.0 = normal */ std::optional contrast; std::optional saturation; + /* 0..2 range, 1.0 = normal */ + std::optional brightness; + /* 0..1 range, 1 = normal */ + std::optional ae_enabled; + /* 0..1 range, 0.5 = normal */ + std::optional exposure_value; + /* 0..100 range, 50.0 = normal */ + std::optional focus_pos; } knobs; }; @@ -77,6 +88,10 @@ struct IPAFrameContext : public FrameContext { double gain; } sensor; + struct { + int32_t focus_pos; + } lens; + struct { double red; double blue; diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp index c94c4cd55..53fa74c03 100644 --- a/src/ipa/simple/soft_simple.cpp +++ b/src/ipa/simple/soft_simple.cpp @@ -77,6 +77,7 @@ private: SwIspStats *stats_; std::unique_ptr camHelper_; ControlInfoMap sensorInfoMap_; + ControlInfoMap lensInfoMap_; /* Local parameter storage */ struct IPAContext context_; @@ -196,6 +197,7 @@ int IPASoftSimple::init(const IPASettings &settings, int IPASoftSimple::configure(const IPAConfigInfo &configInfo) { sensorInfoMap_ = configInfo.sensorControls; + lensInfoMap_ = configInfo.lensControls; const ControlInfo &exposureInfo = sensorInfoMap_.find(V4L2_CID_EXPOSURE)->second; const ControlInfo &gainInfo = sensorInfoMap_.find(V4L2_CID_ANALOGUE_GAIN)->second; @@ -205,6 +207,17 @@ int IPASoftSimple::configure(const IPAConfigInfo &configInfo) context_.activeState = {}; context_.frameContexts.clear(); + if (lensInfoMap_.empty()) { + LOG(IPASoft, Warning) << "No camera leans found! Focus control disabled."; + context_.configuration.focus.focus_min = 0; + context_.configuration.focus.focus_max = 0; + } else { + const ControlInfo &lensInfo = lensInfoMap_.find(V4L2_CID_FOCUS_ABSOLUTE)->second; + context_.configuration.focus.focus_min = lensInfo.min().get(); + context_.configuration.focus.focus_max = lensInfo.max().get(); + LOG(IPASoft, Warning) << "Camera leans found! Focus: " << context_.configuration.focus.focus_min << "-" << context_.configuration.focus.focus_max; + } + context_.configuration.agc.lineDuration = context_.sensorInfo.minLineLength * 1.0s / context_.sensorInfo.pixelRate; context_.configuration.agc.exposureMin = exposureInfo.min().get(); @@ -327,7 +340,10 @@ void IPASoftSimple::processStats(const uint32_t frame, ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(camHelper_ ? camHelper_->gainCode(againNew) : againNew)); - setSensorControls.emit(ctrls); + ControlList lens_ctrls(lensInfoMap_); + lens_ctrls.set(V4L2_CID_FOCUS_ABSOLUTE, frameContext.lens.focus_pos); + + setSensorControls.emit(ctrls, lens_ctrls); } std::string IPASoftSimple::logPrefix() const diff --git a/src/libcamera/egl.cpp b/src/libcamera/egl.cpp new file mode 100644 index 000000000..c6b0f9a56 --- /dev/null +++ b/src/libcamera/egl.cpp @@ -0,0 +1,369 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl.cpp - Helper class for managing eGL interactions. + */ + +#include "libcamera/internal/egl.h" + +#include +#include +#include +#include + +#include +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(eGL) + +eGL::eGL() +{ +} + +eGL::~eGL() +{ +} + +// Create linear image attached to previous BO object +int eGL::createDMABufTexture2D(eGLImage *eglImage, int fd) +{ + int ret = 0; + + eglImage->stride_ = eglImage->width_ * eglImage->height_; + eglImage->offset_ = 0; + eglImage->framesize_ = eglImage->height_ * eglImage->stride_; + + LOG(eGL, Info) + << " stride " << eglImage->stride_ << " width " << eglImage->width_ << " height " << eglImage->height_ << " offset " << eglImage->offset_ << " framesize " << eglImage->framesize_; + + // TODO: use the dma buf handle from udma heap here directly + // should work for both input and output with fencing + EGLint image_attrs[] = { + EGL_WIDTH, (EGLint)eglImage->width_, + EGL_HEIGHT, (EGLint)eglImage->height_, + EGL_LINUX_DRM_FOURCC_EXT, (int)GBM_FORMAT_ARGB8888, + EGL_DMA_BUF_PLANE0_FD_EXT, fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0, + EGL_DMA_BUF_PLANE0_PITCH_EXT, (EGLint)eglImage->framesize_, + EGL_NONE, EGL_NONE, /* modifier lo */ + EGL_NONE, EGL_NONE, /* modifier hi */ + EGL_NONE, + }; + + eglImage->image_ = eglCreateImageKHR(display_, EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + NULL, image_attrs); + + if (eglImage->image_ == EGL_NO_IMAGE_KHR) { + LOG(eGL, Error) << "eglCreateImageKHR fail"; + ret = -ENODEV; + goto done; + } + + // Generate texture, bind, associate image to texture, configure, unbind + glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, eglImage->image_); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + +done: + return ret; +} + +void eGL::destroyDMABufTexture(eGLImage *eglImage) +{ + eglDestroyImage(display_, eglImage->image_); +} + +// +// Generate a 2D texture from an input buffer directly +void eGL::createTexture2D(eGLImage *eglImage, GLint format, uint32_t width, uint32_t height, void *data) +{ + glActiveTexture(eglImage->texture_unit_); + glBindTexture(GL_TEXTURE_2D, eglImage->texture_); + + // Generate texture, bind, associate image to texture, configure, unbind + glTexImage2D(GL_TEXTURE_2D, 0, format, width, height, 0, format, GL_UNSIGNED_BYTE, data); + + // Nearest filtering + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + + // Wrap to edge to avoid edge artifacts + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); +} + +int eGL::initEGLContext(GBM *gbmContext) +{ + EGLint configAttribs[] = { + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_NONE + }; + + EGLint contextAttribs[] = { + EGL_CONTEXT_MAJOR_VERSION, 2, + EGL_NONE + }; + + EGLint numConfigs; + EGLConfig config; + EGLint major; + EGLint minor; + + if (!eglBindAPI(EGL_OPENGL_ES_API)) { + LOG(eGL, Error) << "API bind fail"; + goto fail; + } + + //TODO: use optional eglGetPlatformDisplayEXT ? + display_ = eglGetDisplay(gbmContext->getDevice()); + if (display_ == EGL_NO_DISPLAY) { + LOG(eGL, Error) << "Unable to get EGL display"; + goto fail; + } + + if (eglInitialize(display_, &major, &minor) != EGL_TRUE) { + LOG(eGL, Error) << "eglInitialize fail"; + goto fail; + } + + LOG(eGL, Info) << "EGL: version " << major << "." << minor; + LOG(eGL, Info) << "EGL: EGL_VERSION: " << eglQueryString(display_, EGL_VERSION); + LOG(eGL, Info) << "EGL: EGL_VENDOR: " << eglQueryString(display_, EGL_VENDOR); + LOG(eGL, Info) << "EGL: EGL_CLIENT_APIS: " << eglQueryString(display_, EGL_CLIENT_APIS); + LOG(eGL, Info) << "EGL: EGL_EXTENSIONS: " << eglQueryString(display_, EGL_EXTENSIONS); + + //TODO: interrogate strings to make sure we aren't hooking unsupported functions + // and remember to error out if a function we depend on isn't found. + // we don't use these functions right now but expect to for DMA backed + // texture generation and render-to-texture. One thing we can do is differentiate + // between DMA and non-DMA texture generation based on the presence of these functions + // In reality most - all ? - mesa implementations have these extensions so + // probably no fallback will be required + eglCreateImageKHR = (PFNEGLCREATEIMAGEKHRPROC)eglGetProcAddress("eglCreateImageKHR"); + if (!eglCreateImageKHR) + LOG(eGL, Warning) << "eglCreateImageKHR not found"; + + eglDestroyImageKHR = (PFNEGLDESTROYIMAGEKHRPROC)eglGetProcAddress("eglDestroyImageKHR"); + if (!eglDestroyImageKHR) + LOG(eGL, Warning) << "eglDestroyImageKHR not found"; + + eglExportDMABUFImageMESA = (PFNEGLEXPORTDMABUFIMAGEMESAPROC)eglGetProcAddress("eglExportDMABUFImageMESA"); + if (!eglExportDMABUFImageMESA) + LOG(eGL, Warning) << "eglExportDMABUFImageMESA not found"; + + glEGLImageTargetTexture2DOES = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglGetProcAddress("glEGLImageTargetTexture2DOES"); + if (!glEGLImageTargetTexture2DOES) + LOG(eGL, Warning) << "glEGLImageTargetTexture2DOES not found"; + + eglClientWaitSyncKHR = (PFNEGLCLIENTWAITSYNCKHRPROC)eglGetProcAddress("eglClientWaitSyncKHR"); + if (!eglClientWaitSyncKHR) + LOG(eGL, Warning) << "eglClientWaitSyncKHR not found"; + + eglCreateSyncKHR = (PFNEGLCREATESYNCKHRPROC)eglGetProcAddress("eglCreateSyncKHR"); + if (!eglCreateSyncKHR) + LOG(eGL, Warning) << "eglCreateSyncKHR not found"; + + if (eglChooseConfig(display_, configAttribs, &config, 1, &numConfigs) != EGL_TRUE) { + LOG(eGL, Error) << "eglChooseConfig fail"; + goto fail; + } + + context_ = eglCreateContext(display_, config, EGL_NO_CONTEXT, contextAttribs); + if (context_ == EGL_NO_CONTEXT) { + LOG(eGL, Error) << "eglContext returned EGL_NO_CONTEXT"; + goto fail; + } + + surface_ = eglCreateWindowSurface(display_, config, + (EGLNativeWindowType)gbmContext->getSurface(), + NULL); + if (surface_ == EGL_NO_SURFACE) { + LOG(eGL, Error) << "eglCreateWindowSurface fail"; + goto fail; + } + + makeCurrent(); + swapBuffers(); + + return 0; +fail: + + return -ENODEV; +} + +void eGL::makeCurrent(void) +{ + if (eglMakeCurrent(display_, surface_, surface_, context_) != EGL_TRUE) { + LOG(eGL, Error) << "eglMakeCurrent fail"; + } +} + +void eGL::swapBuffers(void) +{ + if (eglSwapBuffers(display_, surface_) != EGL_TRUE) { + LOG(eGL, Error) << "eglSwapBuffers fail"; + } +} + +void eGL::useProgram(GLuint programId) +{ + glUseProgram(programId); +} + +void eGL::pushEnv(std::vector &shaderEnv, const char *str) +{ + std::string addStr = str; + + addStr.push_back('\n'); + shaderEnv.push_back(addStr); +} + +int eGL::compileVertexShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + return compileShader(GL_VERTEX_SHADER, shaderId, shaderData, shaderDataLen, shaderEnv); +} + +int eGL::compileFragmentShader(GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + return compileShader(GL_FRAGMENT_SHADER, shaderId, shaderData, shaderDataLen, shaderEnv); +} + +int eGL::compileShader(int shaderType, GLuint &shaderId, unsigned char *shaderData, + unsigned int shaderDataLen, + std::vector shaderEnv) +{ + GLchar **shaderSourceData; + GLint *shaderDataLengths; + GLint success; + GLsizei count; + size_t i; + + count = 1 + shaderEnv.size(); + shaderSourceData = new GLchar *[count]; + shaderDataLengths = new GLint[count]; + + // Prefix defines before main body of shader + for (i = 0; i < shaderEnv.size(); i++) { + shaderSourceData[i] = (GLchar *)shaderEnv[i].c_str(); + shaderDataLengths[i] = shaderEnv[i].length(); + } + + // Now the main body of the shader program + shaderSourceData[i] = (GLchar *)shaderData; + shaderDataLengths[i] = shaderDataLen; + + // And create the shader + shaderId = glCreateShader(shaderType); + glShaderSource(shaderId, count, shaderSourceData, shaderDataLengths); + glCompileShader(shaderId); + + // Check status + glGetShaderiv(shaderId, GL_COMPILE_STATUS, &success); + if (success == GL_FALSE) { + GLint sizeLog = 0; + GLchar *infoLog; + + glGetShaderiv(shaderId, GL_INFO_LOG_LENGTH, &sizeLog); + infoLog = new GLchar[sizeLog]; + + glGetShaderInfoLog(shaderId, sizeLog, &sizeLog, infoLog); + LOG(eGL, Error) << infoLog; + + delete[] infoLog; + } + + delete[] shaderSourceData; + delete[] shaderDataLengths; + + return !(success == GL_TRUE); +} + +void eGL::dumpShaderSource(GLuint shaderId) +{ + GLint shaderLength = 0; + GLchar *shaderSource; + + glGetShaderiv(shaderId, GL_SHADER_SOURCE_LENGTH, &shaderLength); + + LOG(eGL, Debug) << "Shader length is " << shaderLength; + + if (shaderLength > 0) { + shaderSource = new GLchar[shaderLength]; + if (!shaderSource) + return; + + glGetShaderSource(shaderId, shaderLength, &shaderLength, shaderSource); + if (shaderLength) { + LOG(eGL, Debug) << "Shader source = " << shaderSource; + } + delete[] shaderSource; + } +} + +int eGL::linkProgram(GLuint &programId, GLuint vertexshaderId, GLuint fragmentshaderId) +{ + GLint success; + GLenum err; + + programId = glCreateProgram(); + if (!programId) + goto fail; + + glAttachShader(programId, vertexshaderId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Attach compute vertex shader fail"; + goto fail; + } + + glAttachShader(programId, fragmentshaderId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Attach compute vertex shader fail"; + goto fail; + } + + glLinkProgram(programId); + if ((err = glGetError()) != GL_NO_ERROR) { + LOG(eGL, Error) << "Link program fail"; + goto fail; + } + + glDetachShader(programId, fragmentshaderId); + glDetachShader(programId, vertexshaderId); + + // Check status + glGetProgramiv(programId, GL_LINK_STATUS, &success); + if (success == GL_FALSE) { + GLint sizeLog = 0; + GLchar *infoLog; + + glGetProgramiv(programId, GL_INFO_LOG_LENGTH, &sizeLog); + infoLog = new GLchar[sizeLog]; + + glGetProgramInfoLog(programId, sizeLog, &sizeLog, infoLog); + LOG(eGL, Error) << infoLog; + + delete[] infoLog; + goto fail; + } + + return 0; +fail: + return -ENODEV; +} +} // namespace libcamera diff --git a/src/libcamera/gbm.cpp b/src/libcamera/gbm.cpp new file mode 100644 index 000000000..43032093e --- /dev/null +++ b/src/libcamera/gbm.cpp @@ -0,0 +1,137 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * egl.cpp - Helper class for managing GBM interactions. + */ + +#include "libcamera/internal/gbm.h" + +#include +#include +#include +#include + +#include +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(GBM) + +GBM::GBM() +{ + fd_ = 0; +} + +GBM::~GBM() +{ + if (gbm_surface_) + gbm_surface_destroy(gbm_surface_); + + if (gbm_device_) + gbm_device_destroy(gbm_device_); + + if (fd_ >= 0) + close(fd_); +} + +// this should probably go into its own class to deal with the +// allocation and deletion of frambuffers attached to GBM devices/objects +int GBM::initSurface(uint32_t width, uint32_t height) +{ + const char *dri_node = "/dev/dri/renderD128"; //TODO: get from an env or config setting + + fd_ = open(dri_node, O_RDWR | O_CLOEXEC); //TODO: CLOEXEC ? + if (fd_ < 0) { + LOG(GBM, Error) << "Open " << dri_node << " fail " << fd_; + return fd_; + } + + gbm_device_ = gbm_create_device(fd_); + if (!gbm_device_) { + LOG(GBM, Error) << "gbm_crate_device fail"; + goto fail; + } + + // GBM_FORMAT_RGBA8888 is not supported mesa::src/gbm/dri/gbm_dri.c::gbm_dri_visuals_table[] + // This means we need to choose XRGB8888 or ARGB8888 as the raw buffer format + gbm_surface_ = gbm_surface_create(gbm_device_, width, height, GBM_FORMAT_ARGB8888, + GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR); + if (!gbm_surface_) { + LOG(GBM, Error) << "Unable to create linear gbm surface"; + goto fail; + } + + format_ = libcamera::formats::ARGB8888; + + return 0; +fail: + return -ENODEV; +} + +int GBM::mapSurface() +{ + gbm_bo_ = gbm_surface_lock_front_buffer(gbm_surface_); + if (!gbm_bo_) { + LOG(GBM, Error) << "GBM input buffer object create fail"; + return -ENODEV; + } + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + + bo_fd_ = gbm_bo_get_fd(gbm_bo_); + + if (!bo_fd_) { + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + LOG(GBM, Error) << "Unable to get fd for bo: " << bo_fd_; + return -ENODEV; + } + + stride_ = gbm_bo_get_stride(gbm_bo_); + width_ = gbm_bo_get_width(gbm_bo_); + height_ = gbm_bo_get_height(gbm_bo_); + offset_ = gbm_bo_get_offset(gbm_bo_, 0); + framesize_ = height_ * stride_; + + map_ = mmap(NULL, height_ * stride_, PROT_READ, MAP_SHARED, bo_fd_, 0); + if (map_ == MAP_FAILED) { + LOG(GBM, Error) << "mmap gbm_bo_ fail"; + return -ENODEV; + } + + LOG(GBM, Debug) << " stride " << stride_ + << " width " << width_ + << " height " << height_ + << " offset " << offset_ + << " framesize " << framesize_; + + return 0; +} + +int GBM::getFrameBufferData(uint8_t *data, size_t data_len) +{ + struct dma_buf_sync sync; + + gbm_surface_lock_front_buffer(gbm_surface_); + + sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ; + ioctl(bo_fd_, DMA_BUF_IOCTL_SYNC, &sync); + + if (data_len > framesize_) { + LOG(GBM, Error) << "Invalid read size " << data_len << " max is " << framesize_; + return -EINVAL; + } + + memcpy(data, map_, data_len); + + sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ; + ioctl(bo_fd_, DMA_BUF_IOCTL_SYNC, &sync); + + gbm_surface_release_buffer(gbm_surface_, gbm_bo_); + + return 0; +} +} //namespace libcamera diff --git a/src/libcamera/mapped_framebuffer.cpp b/src/libcamera/mapped_framebuffer.cpp index f54bbf21f..d5f347d4d 100644 --- a/src/libcamera/mapped_framebuffer.cpp +++ b/src/libcamera/mapped_framebuffer.cpp @@ -238,6 +238,13 @@ MappedFrameBuffer::MappedFrameBuffer(const FrameBuffer *buffer, MapFlags flags) planes_.emplace_back(info.address + plane.offset, plane.length); } + + buffer_ = buffer; +} + +int MappedFrameBuffer::getPlaneFD(int plane) +{ + return buffer_->planes()[plane].fd.get(); } } /* namespace libcamera */ diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 202db1efe..491eb7340 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -67,6 +67,37 @@ libcamera_deps = [] libatomic = cc.find_library('atomic', required : false) libthreads = dependency('threads') +libgbm = cc.find_library('gbm', required: false) +gbm_works = cc.check_header('gbm.h', required: false) + +if libgbm.found() and gbm_works + config_h.set('HAVE_GBM', 1) + libcamera_internal_sources += files([ + 'gbm.cpp', + ]) +endif + +libegl = cc.find_library('EGL', required : false) +libglesv2 = cc.find_library('GLESv2', required : false) +mesa_works = cc.check_header('EGL/egl.h', required: false) + +if libegl.found() and mesa_works + config_h.set('HAVE_LIBEGL', 1) +endif + +if libglesv2.found() and mesa_works + config_h.set('HAVE_GLESV2', 1) +endif + +if mesa_works and gbm_works + libcamera_internal_sources += files([ + 'egl.cpp', + ]) + gles_headless_enabled = true +else + gles_headless_enabled = false +endif + subdir('base') subdir('converter') subdir('ipa') @@ -188,6 +219,9 @@ libcamera_deps += [ libcamera_base_private, libcrypto, libdl, + libegl, + libgbm, + libglesv2, liblttng, libudev, libyaml, diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp index efb07051b..4c002ca68 100644 --- a/src/libcamera/pipeline/simple/simple.cpp +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -30,6 +30,7 @@ #include #include "libcamera/internal/camera.h" +#include "libcamera/internal/camera_lens.h" #include "libcamera/internal/camera_sensor.h" #include "libcamera/internal/camera_sensor_properties.h" #include "libcamera/internal/converter.h" @@ -41,6 +42,8 @@ #include "libcamera/internal/v4l2_subdevice.h" #include "libcamera/internal/v4l2_videodevice.h" +#include "libcamera/controls.h" + namespace libcamera { LOG_DEFINE_CATEGORY(SimplePipeline) @@ -356,7 +359,7 @@ private: void ispStatsReady(uint32_t frame, uint32_t bufferId); void metadataReady(uint32_t frame, const ControlList &metadata); - void setSensorControls(const ControlList &sensorControls); + void setSensorControls(const ControlList &sensorControls, const ControlList &lensControls); }; class SimpleCameraConfiguration : public CameraConfiguration @@ -1002,7 +1005,7 @@ void SimpleCameraData::metadataReady(uint32_t frame, const ControlList &metadata tryCompleteRequest(info->request); } -void SimpleCameraData::setSensorControls(const ControlList &sensorControls) +void SimpleCameraData::setSensorControls(const ControlList &sensorControls, const ControlList &lensControls) { delayedCtrls_->push(sensorControls); /* @@ -1013,10 +1016,21 @@ void SimpleCameraData::setSensorControls(const ControlList &sensorControls) * but it also bypasses delayedCtrls_, creating AGC regulation issues. * Both problems should be fixed. */ - if (!frameStartEmitter_) { - ControlList ctrls(sensorControls); - sensor_->setControls(&ctrls); - } + if (frameStartEmitter_) + return; + + ControlList ctrls(sensorControls); + sensor_->setControls(&ctrls); + + CameraLens *focusLens = sensor_->focusLens(); + if (!focusLens) + return; + + if (!lensControls.contains(V4L2_CID_FOCUS_ABSOLUTE)) + return; + + const ControlValue &focusValue = lensControls.get(V4L2_CID_FOCUS_ABSOLUTE); + focusLens->setFocusPosition(focusValue.get()); } /* Retrieve all source pads connected to a sink pad through active routes. */ @@ -1406,6 +1420,10 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) } else { ipa::soft::IPAConfigInfo configInfo; configInfo.sensorControls = data->sensor_->controls(); + if (data->sensor_->focusLens() != nullptr) + configInfo.lensControls = data->sensor_->focusLens()->controls(); + else + configInfo.lensControls = ControlInfoMap(); return data->swIsp_->configure(inputCfg, outputCfgs, configInfo); } } diff --git a/src/libcamera/software_isp/benchmark.cpp b/src/libcamera/software_isp/benchmark.cpp new file mode 100644 index 000000000..b3da3c416 --- /dev/null +++ b/src/libcamera/software_isp/benchmark.cpp @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Red Hat Inc. + * + * Authors: + * Hans de Goede + * + * Simple builtin benchmark to measure software ISP processing times + */ + +#include "libcamera/internal/software_isp/benchmark.h" + +#include + +namespace libcamera { + +LOG_DEFINE_CATEGORY(Benchmark) + +/** + * \class Benchmark + * \brief Simple builtin benchmark + * + * Simple builtin benchmark to measure software ISP processing times. + */ + +/** + * \brief Constructs a Benchmark object + */ +Benchmark::Benchmark() + : measuredFrames_(0), frameProcessTime_(0) +{ +} + +Benchmark::~Benchmark() +{ +} + +static inline int64_t timeDiff(timespec &after, timespec &before) +{ + return (after.tv_sec - before.tv_sec) * 1000000000LL + + (int64_t)after.tv_nsec - (int64_t)before.tv_nsec; +} + +/** + * \brief Start measuring process time for a single frame + * + * Call this function before processing frame data to start measuring + * the process time for a frame. + */ +void Benchmark::startFrame(void) +{ + if (measuredFrames_ >= Benchmark::kLastFrameToMeasure) + return; + + frameStartTime_ = {}; + clock_gettime(CLOCK_MONOTONIC_RAW, &frameStartTime_); +} + +/** + * \brief Finish measuring process time for a single frame + * + * Call this function after processing frame data to finish measuring + * the process time for a frame. + * + * This function will log frame processing time information after + * Benchmark::kLastFrameToMeasure frames have been processed. + */ +void Benchmark::finishFrame(void) +{ + if (measuredFrames_ >= Benchmark::kLastFrameToMeasure) + return; + + measuredFrames_++; + + if (measuredFrames_ <= Benchmark::kFramesToSkip) + return; + + timespec frameEndTime = {}; + clock_gettime(CLOCK_MONOTONIC_RAW, &frameEndTime); + frameProcessTime_ += timeDiff(frameEndTime, frameStartTime_); + + if (measuredFrames_ == Benchmark::kLastFrameToMeasure) { + const unsigned int measuredFrames = Benchmark::kLastFrameToMeasure - + Benchmark::kFramesToSkip; + LOG(Benchmark, Info) + << "Processed " << measuredFrames + << " frames in " << frameProcessTime_ / 1000 << "us, " + << frameProcessTime_ / (1000 * measuredFrames) + << " us/frame"; + } +} + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer.cpp b/src/libcamera/software_isp/debayer.cpp index e9e18c488..d0e17d20b 100644 --- a/src/libcamera/software_isp/debayer.cpp +++ b/src/libcamera/software_isp/debayer.cpp @@ -103,6 +103,17 @@ namespace libcamera { LOG_DEFINE_CATEGORY(Debayer) +Debayer::Debayer() +{ + /* Initialize color lookup tables */ + for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { + red_[i] = green_[i] = blue_[i] = i; + redCcm_[i] = { static_cast(i), 0, 0 }; + greenCcm_[i] = { 0, static_cast(i), 0 }; + blueCcm_[i] = { 0, 0, static_cast(i) }; + } +} + Debayer::~Debayer() { } @@ -176,4 +187,54 @@ Debayer::~Debayer() * \brief Signals when the output buffer is ready */ +/** + * \fn void Debayer::setParams(DebayerParams ¶ms) + * \brief Select the bayer params to use for the next frame debayer + * \param[in] params The parameters to be used in debayering + */ +void Debayer::setParams(DebayerParams ¶ms) +{ + green_ = params.green; + greenCcm_ = params.greenCcm; + if (swapRedBlueGains_) { + red_ = params.blue; + blue_ = params.red; + redCcm_ = params.blueCcm; + blueCcm_ = params.redCcm; + for (unsigned int i = 0; i < 256; i++) { + std::swap(redCcm_[i].r, redCcm_[i].b); + std::swap(blueCcm_[i].r, blueCcm_[i].b); + } + } else { + red_ = params.red; + blue_ = params.blue; + redCcm_ = params.redCcm; + blueCcm_ = params.blueCcm; + } + gammaLut_ = params.gammaLut; +} + +/** + * \fn void Debayer::dmaSyncBegin(DebayerParams ¶ms) + * \brief Common CPU/GPU Dma Sync Buffer begin + */ +void Debayer::dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output) +{ + for (const FrameBuffer::Plane &plane : input->planes()) + dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Read); + + for (const FrameBuffer::Plane &plane : output->planes()) + dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); +} + +/** + * \fn void Debayer::isStandardBayerOrder(BayerFormat::Order order) + * \brief Common method to validate standard Bayer order + */ +bool Debayer::isStandardBayerOrder(BayerFormat::Order order) +{ + return order == BayerFormat::BGGR || order == BayerFormat::GBRG || + order == BayerFormat::GRBG || order == BayerFormat::RGGB; +} + } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer.h b/src/libcamera/software_isp/debayer.h index ba033d440..3893318b2 100644 --- a/src/libcamera/software_isp/debayer.h +++ b/src/libcamera/software_isp/debayer.h @@ -14,11 +14,15 @@ #include #include +#include #include #include #include +#include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/dma_buf_allocator.h" +#include "libcamera/internal/software_isp/benchmark.h" #include "libcamera/internal/software_isp/debayer_params.h" namespace libcamera { @@ -27,9 +31,10 @@ class FrameBuffer; LOG_DECLARE_CATEGORY(Debayer) -class Debayer +class Debayer : public Object { public: + Debayer(); virtual ~Debayer() = 0; virtual int configure(const StreamConfiguration &inputCfg, @@ -45,11 +50,45 @@ public: virtual SizeRange sizes(PixelFormat inputFormat, const Size &inputSize) = 0; + virtual const SharedFD &getStatsFD() = 0; + + unsigned int frameSize() { return outputConfig_.frameSize; } + Signal inputBufferReady; Signal outputBufferReady; + struct DebayerInputConfig { + Size patternSize; + unsigned int bpp; /* Memory used per pixel, not precision */ + unsigned int stride; + std::vector outputFormats; + }; + + struct DebayerOutputConfig { + unsigned int bpp; /* Memory used per pixel, not precision */ + unsigned int stride; + unsigned int frameSize; + }; + + DebayerInputConfig inputConfig_; + DebayerOutputConfig outputConfig_; + DebayerParams::LookupTable red_; + DebayerParams::LookupTable green_; + DebayerParams::LookupTable blue_; + DebayerParams::CcmLookupTable redCcm_; + DebayerParams::CcmLookupTable greenCcm_; + DebayerParams::CcmLookupTable blueCcm_; + DebayerParams::LookupTable gammaLut_; + bool swapRedBlueGains_; + Benchmark bench_; + private: virtual Size patternSize(PixelFormat inputFormat) = 0; + +protected: + void setParams(DebayerParams ¶ms); + void dmaSyncBegin(std::vector &dmaSyncers, FrameBuffer *input, FrameBuffer *output); + static bool isStandardBayerOrder(BayerFormat::Order order); }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 66f6038c1..e56492848 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -22,7 +22,6 @@ #include #include "libcamera/internal/bayer_format.h" -#include "libcamera/internal/dma_buf_allocator.h" #include "libcamera/internal/framebuffer.h" #include "libcamera/internal/mapped_framebuffer.h" @@ -40,7 +39,7 @@ namespace libcamera { * \param[in] stats Pointer to the stats object to use */ DebayerCpu::DebayerCpu(std::unique_ptr stats) - : stats_(std::move(stats)) + : Debayer(), stats_(std::move(stats)) { /* * Reading from uncached buffers may be very slow. @@ -51,14 +50,6 @@ DebayerCpu::DebayerCpu(std::unique_ptr stats) * future. */ enableInputMemcpy_ = true; - - /* Initialize color lookup tables */ - for (unsigned int i = 0; i < DebayerParams::kRGBLookupSize; i++) { - red_[i] = green_[i] = blue_[i] = i; - redCcm_[i] = { static_cast(i), 0, 0 }; - greenCcm_[i] = { 0, static_cast(i), 0 }; - blueCcm_[i] = { 0, 0, static_cast(i) }; - } } DebayerCpu::~DebayerCpu() = default; @@ -291,12 +282,6 @@ void DebayerCpu::debayer10P_RGRG_BGR888(uint8_t *dst, const uint8_t *src[]) } } -static bool isStandardBayerOrder(BayerFormat::Order order) -{ - return order == BayerFormat::BGGR || order == BayerFormat::GBRG || - order == BayerFormat::GRBG || order == BayerFormat::RGGB; -} - /* * Setup the Debayer object according to the passed in parameters. * Return 0 on success, a negative errno value on failure @@ -554,9 +539,6 @@ int DebayerCpu::configure(const StreamConfiguration &inputCfg, lineBuffers_[i].resize(lineBufferLength_); } - measuredFrames_ = 0; - frameProcessTime_ = 0; - return 0; } @@ -746,50 +728,15 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) } } -namespace { - -inline int64_t timeDiff(timespec &after, timespec &before) -{ - return (after.tv_sec - before.tv_sec) * 1000000000LL + - (int64_t)after.tv_nsec - (int64_t)before.tv_nsec; -} - -} /* namespace */ - void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params) { - timespec frameStartTime; - - if (measuredFrames_ < DebayerCpu::kLastFrameToMeasure) { - frameStartTime = {}; - clock_gettime(CLOCK_MONOTONIC_RAW, &frameStartTime); - } + bench_.startFrame(); std::vector dmaSyncers; - for (const FrameBuffer::Plane &plane : input->planes()) - dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Read); - for (const FrameBuffer::Plane &plane : output->planes()) - dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); + dmaSyncBegin(dmaSyncers, input, output); - green_ = params.green; - greenCcm_ = params.greenCcm; - if (swapRedBlueGains_) { - red_ = params.blue; - blue_ = params.red; - redCcm_ = params.blueCcm; - blueCcm_ = params.redCcm; - for (unsigned int i = 0; i < 256; i++) { - std::swap(redCcm_[i].r, redCcm_[i].b); - std::swap(blueCcm_[i].r, blueCcm_[i].b); - } - } else { - red_ = params.red; - blue_ = params.blue; - redCcm_ = params.redCcm; - blueCcm_ = params.blueCcm; - } - gammaLut_ = params.gammaLut; + setParams(params); /* Copy metadata from the input buffer */ FrameMetadata &metadata = output->_d()->metadata(); @@ -817,21 +764,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output dmaSyncers.clear(); /* Measure before emitting signals */ - if (measuredFrames_ < DebayerCpu::kLastFrameToMeasure && - ++measuredFrames_ > DebayerCpu::kFramesToSkip) { - timespec frameEndTime = {}; - clock_gettime(CLOCK_MONOTONIC_RAW, &frameEndTime); - frameProcessTime_ += timeDiff(frameEndTime, frameStartTime); - if (measuredFrames_ == DebayerCpu::kLastFrameToMeasure) { - const unsigned int measuredFrames = DebayerCpu::kLastFrameToMeasure - - DebayerCpu::kFramesToSkip; - LOG(Debayer, Info) - << "Processed " << measuredFrames - << " frames in " << frameProcessTime_ / 1000 << "us, " - << frameProcessTime_ / (1000 * measuredFrames) - << " us/frame"; - } - } + bench_.finishFrame(); /* * Buffer ids are currently not used, so pass zeros as its parameter. diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 926195e98..999e3421c 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -17,14 +17,13 @@ #include -#include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" #include "debayer.h" -#include "swstats_cpu.h" namespace libcamera { -class DebayerCpu : public Debayer, public Object +class DebayerCpu : public Debayer { public: DebayerCpu(std::unique_ptr stats); @@ -47,13 +46,6 @@ public: */ const SharedFD &getStatsFD() { return stats_->getStatsFD(); } - /** - * \brief Get the output frame size - * - * \return The output frame size - */ - unsigned int frameSize() { return outputConfig_.frameSize; } - private: /** * \brief Called to debayer 1 line of Bayer input data to output format @@ -110,21 +102,8 @@ private: template void debayer10P_RGRG_BGR888(uint8_t *dst, const uint8_t *src[]); - struct DebayerInputConfig { - Size patternSize; - unsigned int bpp; /* Memory used per pixel, not precision */ - unsigned int stride; - std::vector outputFormats; - }; - - struct DebayerOutputConfig { - unsigned int bpp; /* Memory used per pixel, not precision */ - unsigned int stride; - unsigned int frameSize; - }; - - int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); - int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); + static int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); + static int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); int setupStandardBayerOrder(BayerFormat::Order order); int setDebayerFunctions(PixelFormat inputFormat, PixelFormat outputFormat, @@ -138,20 +117,11 @@ private: /* Max. supported Bayer pattern height is 4, debayering this requires 5 lines */ static constexpr unsigned int kMaxLineBuffers = 5; - DebayerParams::LookupTable red_; - DebayerParams::LookupTable green_; - DebayerParams::LookupTable blue_; - DebayerParams::CcmLookupTable redCcm_; - DebayerParams::CcmLookupTable greenCcm_; - DebayerParams::CcmLookupTable blueCcm_; - DebayerParams::LookupTable gammaLut_; debayerFn debayer0_; debayerFn debayer1_; debayerFn debayer2_; debayerFn debayer3_; Rectangle window_; - DebayerInputConfig inputConfig_; - DebayerOutputConfig outputConfig_; std::unique_ptr stats_; std::vector lineBuffers_[kMaxLineBuffers]; unsigned int lineBufferLength_; @@ -159,12 +129,6 @@ private: unsigned int lineBufferIndex_; unsigned int xShift_; /* Offset of 0/1 applied to window_.x */ bool enableInputMemcpy_; - bool swapRedBlueGains_; - unsigned int measuredFrames_; - int64_t frameProcessTime_; - /* Skip 30 frames for things to stabilize then measure 30 frames */ - static constexpr unsigned int kFramesToSkip = 30; - static constexpr unsigned int kLastFrameToMeasure = 60; }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp new file mode 100644 index 000000000..9ec966608 --- /dev/null +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -0,0 +1,632 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Linaro Ltd. + * + * Authors: + * Bryan O'Donoghue + * + * debayer_cpu.cpp - EGL based debayering class + */ + +#include +#include +#include + +#include + +#include "libcamera/internal/glsl_shaders.h" +#include "debayer_egl.h" + +namespace libcamera { + +DebayerEGL::DebayerEGL(std::unique_ptr stats) + : Debayer(), stats_(std::move(stats)) +{ + eglImageBayerIn_ = eglImageRedLookup_ = eglImageBlueLookup_ = eglImageGreenLookup_ = NULL; +} + +DebayerEGL::~DebayerEGL() +{ + if (eglImageBlueLookup_) + delete eglImageBlueLookup_; + + if (eglImageGreenLookup_) + delete eglImageGreenLookup_; + + if (eglImageRedLookup_) + delete eglImageRedLookup_; + + if (eglImageBayerIn_) + delete eglImageBayerIn_; +} + +int DebayerEGL::getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config) +{ + BayerFormat bayerFormat = + BayerFormat::fromPixelFormat(inputFormat); + + if ((bayerFormat.bitDepth == 8 || bayerFormat.bitDepth == 10) && + bayerFormat.packing == BayerFormat::Packing::None && + isStandardBayerOrder(bayerFormat.order)) { + config.bpp = (bayerFormat.bitDepth + 7) & ~7; + config.patternSize.width = 2; + config.patternSize.height = 2; + config.outputFormats = std::vector({ formats::XRGB8888, + formats::ARGB8888, + formats::XBGR8888, + formats::ABGR8888 }); + return 0; + } + + if (bayerFormat.bitDepth == 10 && + bayerFormat.packing == BayerFormat::Packing::CSI2 && + isStandardBayerOrder(bayerFormat.order)) { + config.bpp = 10; + config.patternSize.width = 4; /* 5 bytes per *4* pixels */ + config.patternSize.height = 2; + config.outputFormats = std::vector({ formats::XRGB8888, + formats::ARGB8888, + formats::XBGR8888, + formats::ABGR8888 }); + return 0; + } + + LOG(Debayer, Info) + << "Unsupported input format " << inputFormat.toString(); + return -EINVAL; +} + +int DebayerEGL::getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config) +{ + if (outputFormat == formats::XRGB8888 || outputFormat == formats::ARGB8888 || + outputFormat == formats::XBGR8888 || outputFormat == formats::ABGR8888) { + config.bpp = 32; + return 0; + } + + LOG(Debayer, Error) + << "Unsupported output format " << outputFormat.toString(); + + return -EINVAL; +} + +int DebayerEGL::getShaderVariableLocations(void) +{ + attributeVertex_ = glGetAttribLocation(programId_, "vertexIn"); + attributeTexture_ = glGetAttribLocation(programId_, "textureIn"); + + textureUniformBayerDataIn_ = glGetUniformLocation(programId_, "tex_y"); + textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); + textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); + textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); + ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm"); + + textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); + textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); + textureUniformStrideFactor_ = glGetUniformLocation(programId_, "stride_factor"); + textureUniformBayerFirstRed_ = glGetUniformLocation(programId_, "tex_bayer_first_red"); + textureUniformProjMatrix_ = glGetUniformLocation(programId_, "proj_matrix"); + + LOG(Debayer, Debug) << "vertexIn " << attributeVertex_ << " textureIn " << attributeTexture_ + << " tex_y " << textureUniformBayerDataIn_ + << " red_param " << textureUniformRedLookupDataIn_ + << " green_param " << textureUniformGreenLookupDataIn_ + << " blue_param " << textureUniformBlueLookupDataIn_ + << " ccm " << ccmUniformDataIn_ + << " tex_step " << textureUniformStep_ + << " tex_size " << textureUniformSize_ + << " stride_factor " << textureUniformStrideFactor_ + << " tex_bayer_first_red " << textureUniformBayerFirstRed_ + << " proj_matrix " << textureUniformProjMatrix_; + return 0; +} + +int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputFormat) +{ + std::vector shaderEnv; + unsigned int fragmentShaderDataLen; + unsigned char *fragmentShaderData; + unsigned int vertexShaderDataLen; + unsigned char *vertexShaderData; + GLenum err; + + // Target gles 100 glsl requires "#version x" as first directive in shader + egl_.pushEnv(shaderEnv, "#version 100"); + + // Specify GL_OES_EGL_image_external + egl_.pushEnv(shaderEnv, "#extension GL_OES_EGL_image_external: enable"); + + // Tell shaders how to re-order output taking account of how the + // pixels are actually stored by GBM + switch (outputFormat) { + case formats::ARGB8888: + case formats::XRGB8888: + break; + case formats::ABGR8888: + case formats::XBGR8888: + egl_.pushEnv(shaderEnv, "#define SWAP_BLUE"); + break; + default: + goto invalid_fmt; + } + + // Pixel location parameters + glFormat_ = GL_LUMINANCE; + bytesPerPixel_ = 1; + switch (inputFormat) { + case libcamera::formats::SBGGR8: + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SBGGR12_CSI2P: + firstRed_x_ = 1.0; + firstRed_y_ = 1.0; + break; + case libcamera::formats::SGBRG8: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + firstRed_x_ = 0.0; + firstRed_y_ = 1.0; + break; + case libcamera::formats::SGRBG8: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + firstRed_x_ = 1.0; + firstRed_y_ = 0.0; + break; + case libcamera::formats::SRGGB8: + case libcamera::formats::SRGGB10_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: + firstRed_x_ = 0.0; + firstRed_y_ = 0.0; + break; + default: + goto invalid_fmt; + break; + }; + + // Shader selection + switch (inputFormat) { + case libcamera::formats::SBGGR8: + case libcamera::formats::SGBRG8: + case libcamera::formats::SGRBG8: + case libcamera::formats::SRGGB8: + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; + break; + case libcamera::formats::SBGGR10_CSI2P: + case libcamera::formats::SGBRG10_CSI2P: + case libcamera::formats::SGRBG10_CSI2P: + case libcamera::formats::SRGGB10_CSI2P: + egl_.pushEnv(shaderEnv, "#define RAW10P"); + if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; + glFormat_ = GL_RG; + bytesPerPixel_ = 2; + } else { + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + } + break; + case libcamera::formats::SBGGR12_CSI2P: + case libcamera::formats::SGBRG12_CSI2P: + case libcamera::formats::SGRBG12_CSI2P: + case libcamera::formats::SRGGB12_CSI2P: + egl_.pushEnv(shaderEnv, "#define RAW12P"); + if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) { + fragmentShaderData = bayer_unpacked_frag; + fragmentShaderDataLen = bayer_unpacked_frag_len; + vertexShaderData = bayer_unpacked_vert; + vertexShaderDataLen = bayer_unpacked_vert_len; + glFormat_ = GL_RG; + bytesPerPixel_ = 2; + } else { + fragmentShaderData = bayer_1x_packed_frag; + fragmentShaderDataLen = bayer_1x_packed_frag_len; + vertexShaderData = identity_vert; + vertexShaderDataLen = identity_vert_len; + } + break; + default: + goto invalid_fmt; + break; + }; + + if (ccmEnabled_) { + // Run the CCM if available + egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS"); + } else { + // Flag to shaders that we have parameter gain tables + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + } + + if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) + goto compile_fail; + + if (egl_.compileFragmentShader(fragmentShaderId_, fragmentShaderData, fragmentShaderDataLen, shaderEnv)) + goto compile_fail; + + if (egl_.linkProgram(programId_, vertexShaderId_, fragmentShaderId_)) + goto link_fail; + + egl_.dumpShaderSource(vertexShaderId_); + egl_.dumpShaderSource(fragmentShaderId_); + + /* Ensure we set the programId_ */ + egl_.useProgram(programId_); + err = glGetError(); + if (err != GL_NO_ERROR) + goto program_fail; + + if (getShaderVariableLocations()) + goto parameters_fail; + + return 0; + +parameters_fail: + LOG(Debayer, Error) << "Program parameters fail"; + return -ENODEV; + +program_fail: + LOG(Debayer, Error) << "Use program error " << err; + return -ENODEV; + +link_fail: + LOG(Debayer, Error) << "Linking program fail"; + return -ENODEV; + +compile_fail: + LOG(Debayer, Error) << "Compile debayer shaders fail"; + return -ENODEV; + +invalid_fmt: + LOG(Debayer, Error) << "Unsupported input output format combination"; + return -EINVAL; +} + +int DebayerEGL::configure(const StreamConfiguration &inputCfg, + const std::vector> &outputCfgs, + bool ccmEnabled) +{ + GLint maxTextureImageUnits; + + if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) + return -EINVAL; + + if (stats_->configure(inputCfg) != 0) + return -EINVAL; + + const Size &stats_pattern_size = stats_->patternSize(); + if (inputConfig_.patternSize.width != stats_pattern_size.width || + inputConfig_.patternSize.height != stats_pattern_size.height) { + LOG(Debayer, Error) + << "mismatching stats and debayer pattern sizes for " + << inputCfg.pixelFormat.toString(); + return -EINVAL; + } + + inputConfig_.stride = inputCfg.stride; + width_ = inputCfg.size.width; + height_ = inputCfg.size.height; + ccmEnabled_ = ccmEnabled; + + if (outputCfgs.size() != 1) { + LOG(Debayer, Error) + << "Unsupported number of output streams: " + << outputCfgs.size(); + return -EINVAL; + } + + LOG(Debayer, Info) << "Input size " << inputCfg.size << " stride " << inputCfg.stride; + + if (gbmSurface_.initSurface(inputCfg.size.width, inputCfg.size.height)) + return -ENODEV; + + if (egl_.initEGLContext(&gbmSurface_)) + return -ENODEV; + + glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); + LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; + + if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { + LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits + << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS + << " try using an identity CCM "; + return -ENODEV; + } + + // Raw bayer input as texture + eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); + if (!eglImageBayerIn_) + return -ENOMEM; + + // Only do the RGB lookup table textures if CCM is disabled + if (!ccmEnabled_) { + + /// RGB correction tables as 2d textures + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); + if (!eglImageRedLookup_) + return -ENOMEM; + + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); + if (!eglImageGreenLookup_) + return -ENOMEM; + + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); + if (!eglImageBlueLookup_) + return -ENOMEM; + } + + // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) + if (gbmSurface_.mapSurface()) + return -ENODEV; + + StreamConfiguration &outputCfg = outputCfgs[0]; + SizeRange outSizeRange = sizes(inputCfg.pixelFormat, inputCfg.size); + + outputConfig_.stride = gbmSurface_.getStride(); + outputConfig_.frameSize = gbmSurface_.getFrameSize(); + + LOG(Debayer, Debug) << "Overriding stream config stride " + << outputCfg.stride << " with GBM surface stride " + << outputConfig_.stride; + outputCfg.stride = outputConfig_.stride; + + if (!outSizeRange.contains(outputCfg.size) || outputConfig_.stride != outputCfg.stride) { + LOG(Debayer, Error) + << "Invalid output size/stride: " + << "\n " << outputCfg.size << " (" << outSizeRange << ")" + << "\n " << outputCfg.stride << " (" << outputConfig_.stride << ")"; + return -EINVAL; + } + + window_.x = ((inputCfg.size.width - outputCfg.size.width) / 2) & + ~(inputConfig_.patternSize.width - 1); + window_.y = ((inputCfg.size.height - outputCfg.size.height) / 2) & + ~(inputConfig_.patternSize.height - 1); + window_.width = outputCfg.size.width; + window_.height = outputCfg.size.height; + + /* Don't pass x,y since process() already adjusts src before passing it */ + stats_->setWindow(Rectangle(window_.size())); + + LOG(Debayer, Debug) << "Input width " << inputCfg.size.width << " height " << inputCfg.size.height; + LOG(Debayer, Debug) << "Output width " << outputCfg.size.width << " height " << outputCfg.size.height; + LOG(Debayer, Debug) << "Output stride " << outputCfg.size.width << " height " << outputCfg.size.height; + + if (initBayerShaders(inputCfg.pixelFormat, outputCfg.pixelFormat)) + return -EINVAL; + + return 0; +} + +Size DebayerEGL::patternSize(PixelFormat inputFormat) +{ + DebayerEGL::DebayerInputConfig config; + + if (getInputConfig(inputFormat, config) != 0) + return {}; + + return config.patternSize; +} + +std::vector DebayerEGL::formats(PixelFormat inputFormat) +{ + DebayerEGL::DebayerInputConfig config; + + if (getInputConfig(inputFormat, config) != 0) + return std::vector(); + + return config.outputFormats; +} + +std::tuple +DebayerEGL::strideAndFrameSize(const PixelFormat &outputFormat, const Size &size) +{ + DebayerEGL::DebayerOutputConfig config; + + if (getOutputConfig(outputFormat, config) != 0) + return std::make_tuple(0, 0); + + /* round up to multiple of 8 for 64 bits alignment */ + unsigned int stride = (size.width * config.bpp / 8 + 7) & ~7; + + return std::make_tuple(stride, stride * size.height); +} + +void DebayerEGL::setShaderVariableValues(void) +{ + /* + * Raw Bayer 8-bit, and packed raw Bayer 10-bit/12-bit formats + * are stored in a GL_LUMINANCE texture. The texture width is + * equal to the stride. + */ + GLfloat firstRed[] = { firstRed_x_, firstRed_y_ }; + GLfloat imgSize[] = { (GLfloat)width_, + (GLfloat)height_ }; + GLfloat Step[] = { static_cast(bytesPerPixel_) / (inputConfig_.stride - 1), + 1.0f / (height_ - 1) }; + GLfloat Stride = 1.0f; + GLfloat projIdentityMatrix[] = { + 1, 0, 0, 0, + 0, 1, 0, 0, + 0, 0, 1, 0, + 0, 0, 0, 1 + }; + + // vertexIn - bayer_8.vert + glEnableVertexAttribArray(attributeVertex_); + glVertexAttribPointer(attributeVertex_, 2, GL_FLOAT, GL_TRUE, + 2 * sizeof(GLfloat), vcoordinates); + + // textureIn - bayer_8.vert + glEnableVertexAttribArray(attributeTexture_); + glVertexAttribPointer(attributeTexture_, 2, GL_FLOAT, GL_TRUE, + 2 * sizeof(GLfloat), tcoordinates); + + // Set the sampler2D to the respective texture unit for each texutre + // To simultaneously sample multiple textures we need to use multiple + // texture units + glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); + if (!ccmEnabled_) { + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + } + + // These values are: + // firstRed = tex_bayer_first_red - bayer_8.vert + // imgSize = tex_size - bayer_8.vert + // step = tex_step - bayer_8.vert + // Stride = stride_factor identity.vert + // textureUniformProjMatri = No scaling + glUniform2fv(textureUniformBayerFirstRed_, 1, firstRed); + glUniform2fv(textureUniformSize_, 1, imgSize); + glUniform2fv(textureUniformStep_, 1, Step); + glUniform1f(textureUniformStrideFactor_, Stride); + glUniformMatrix4fv(textureUniformProjMatrix_, 1, + GL_FALSE, projIdentityMatrix); + + LOG(Debayer, Debug) << "vertexIn " << attributeVertex_ << " textureIn " << attributeTexture_ + << " tex_y " << textureUniformBayerDataIn_ + << " red_param " << textureUniformRedLookupDataIn_ + << " green_param " << textureUniformGreenLookupDataIn_ + << " blue_param " << textureUniformBlueLookupDataIn_ + << " tex_step " << textureUniformStep_ + << " tex_size " << textureUniformSize_ + << " stride_factor " << textureUniformStrideFactor_ + << " tex_bayer_first_red " << textureUniformBayerFirstRed_; + + LOG (Debayer, Debug) << "textureUniformY_ = 0 " << + " firstRed.x " << firstRed[0] << + " firstRed.y " << firstRed[1] << + " textureUniformSize_.width " << imgSize[0] << " " + " textureUniformSize_.height " << imgSize[1] << + " textureUniformStep_.x " << Step[0] << + " textureUniformStep_.y " << Step[1] << + " textureUniformStrideFactor_ " << Stride << + " textureUniformProjMatrix_ " << textureUniformProjMatrix_; + return; +} + +void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, DebayerParams ¶ms) +{ + LOG(Debayer, Debug) + << "Input height " << height_ + << " width " << width_ + << " fd " << in.getPlaneFD(0); + + // eGL context switch + egl_.makeCurrent(); + + // Greate a standard texture + // we will replace this with the DMA version at some point + egl_.createTexture2D(eglImageBayerIn_, glFormat_, inputConfig_.stride / bytesPerPixel_, height_, in.planes()[0].data()); + + // Populate bayer parameters + if (ccmEnabled_) { + GLfloat ccm[9] = { + params.ccm[0][0], params.ccm[0][1], params.ccm[0][2], + params.ccm[1][0], params.ccm[1][1], params.ccm[1][2], + params.ccm[2][0], params.ccm[2][1], params.ccm[2][2], + }; + glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); + } else { + egl_.createTexture2D(eglImageRedLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + } + + // Setup the scene + setShaderVariableValues(); + glViewport(0, 0, width_, height_); + glClear(GL_COLOR_BUFFER_BIT); + glDisable(GL_BLEND); + + // Draw the scene + glDrawArrays(GL_TRIANGLE_FAN, 0, DEBAYER_OPENGL_COORDS); + + // eglclientWaitScynKhr / eglwaitsynckr ? + egl_.swapBuffers(); + + // Copy from the output GBM buffer to our output plane + // once we get render to texture working the + // explicit lock ioctl, memcpy and unlock ioctl won't be required + gbmSurface_.getFrameBufferData(out.planes()[0].data(), out.planes()[0].size()); +} + +void DebayerEGL::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params) +{ + bench_.startFrame(); + + std::vector dmaSyncers; + + dmaSyncBegin(dmaSyncers, input, output); + + setParams(params); + + /* Copy metadata from the input buffer */ + FrameMetadata &metadata = output->_d()->metadata(); + metadata.status = input->metadata().status; + metadata.sequence = input->metadata().sequence; + metadata.timestamp = input->metadata().timestamp; + + MappedFrameBuffer in(input, MappedFrameBuffer::MapFlag::Read); + MappedFrameBuffer out(output, MappedFrameBuffer::MapFlag::Write); + if (!in.isValid() || !out.isValid()) { + LOG(Debayer, Error) << "mmap-ing buffer(s) failed"; + metadata.status = FrameMetadata::FrameError; + return; + } + + debayerGPU(in, out, params); + + dmaSyncers.clear(); + + bench_.finishFrame(); + + metadata.planes()[0].bytesused = out.planes()[0].size(); + + // Calculate stats for the whole frame + stats_->processFrame(frame, 0, input); + + outputBufferReady.emit(output); + inputBufferReady.emit(input); +} + +SizeRange DebayerEGL::sizes(PixelFormat inputFormat, const Size &inputSize) +{ + Size patternSize = this->patternSize(inputFormat); + unsigned int borderHeight = patternSize.height; + + if (patternSize.isNull()) + return {}; + + /* No need for top/bottom border with a pattern height of 2 */ + if (patternSize.height == 2) + borderHeight = 0; + + /* + * For debayer interpolation a border is kept around the entire image + * and the minimum output size is pattern-height x pattern-width. + */ + if (inputSize.width < (3 * patternSize.width) || + inputSize.height < (2 * borderHeight + patternSize.height)) { + LOG(Debayer, Warning) + << "Input format size too small: " << inputSize.toString(); + return {}; + } + + return SizeRange(Size(patternSize.width, patternSize.height), + Size((inputSize.width - 2 * patternSize.width) & ~(patternSize.width - 1), + (inputSize.height - 2 * borderHeight) & ~(patternSize.height - 1)), + patternSize.width, patternSize.height); +} + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h new file mode 100644 index 000000000..56f5434ac --- /dev/null +++ b/src/libcamera/software_isp/debayer_egl.h @@ -0,0 +1,171 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025, Bryan O'Donoghue. + * + * Authors: + * Bryan O'Donoghue + * + * debayer_opengl.h - EGL debayer header + */ + +#pragma once + +#include +#include +#include + +#define GL_GLEXT_PROTOTYPES +#define EGL_EGLEXT_PROTOTYPES +#include +#include +#include + +#include + +#include "debayer.h" + +#include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/egl.h" +#include "libcamera/internal/framebuffer.h" +#include "libcamera/internal/mapped_framebuffer.h" +#include "libcamera/internal/software_isp/benchmark.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" + +namespace libcamera { + +#define DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS 4 +#define DEBAYER_OPENGL_COORDS 4 + +/** + * \class DebayerEGL + * \brief Class for debayering using an EGL Shader + * + * Implements an EGL shader based debayering solution. + */ +class DebayerEGL : public Debayer +{ +public: + /** + * \brief Constructs a DebayerEGL object. + * \param[in] stats Pointer to the stats object to use. + */ + DebayerEGL(std::unique_ptr stats); + ~DebayerEGL(); + + /* + * Setup the Debayer object according to the passed in parameters. + * Return 0 on success, a negative errno value on failure + * (unsupported parameters). + */ + int configure(const StreamConfiguration &inputCfg, + const std::vector> &outputCfgs, + bool ccmEnabled); + + /* + * Get width and height at which the bayer-pattern repeats. + * Return pattern-size or an empty Size for an unsupported inputFormat. + */ + Size patternSize(PixelFormat inputFormat); + + std::vector formats(PixelFormat input); + std::tuple strideAndFrameSize(const PixelFormat &outputFormat, const Size &size); + + void process(uint32_t frame, FrameBuffer *input, FrameBuffer *output, DebayerParams params); + + /** + * \brief Get the file descriptor for the statistics. + * + * \return the file descriptor pointing to the statistics. + */ + const SharedFD &getStatsFD() { return stats_->getStatsFD(); } + + /** + * \brief Get the output frame size. + * + * \return The output frame size. + */ + unsigned int frameSize() { return outputConfig_.frameSize; } + + SizeRange sizes(PixelFormat inputFormat, const Size &inputSize); + +private: + static int getInputConfig(PixelFormat inputFormat, DebayerInputConfig &config); + static int getOutputConfig(PixelFormat outputFormat, DebayerOutputConfig &config); + int setupStandardBayerOrder(BayerFormat::Order order); + void pushEnv(std::vector &shaderEnv, const char *str); + int initBayerShaders(PixelFormat inputFormat, PixelFormat outputFormat); + int initEGLContext(); + int generateTextures(); + int compileShaderProgram(GLuint &shaderId, GLenum shaderType, + unsigned char *shaderData, int shaderDataLen, + std::vector shaderEnv); + int linkShaderProgram(void); + int getShaderVariableLocations(); + void setShaderVariableValues(void); + void configureTexture(GLuint &texture); + void debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, DebayerParams ¶ms); + + // Shader program identifiers + GLuint vertexShaderId_; + GLuint fragmentShaderId_; + GLuint programId_; + enum { + BAYER_INPUT_INDEX = 0, + BAYER_OUTPUT_INDEX, + BAYER_BUF_NUM, + }; + + // Pointer to object representing input texture + eGLImage *eglImageBayerIn_; + + eGLImage *eglImageRedLookup_; + eGLImage *eglImageGreenLookup_; + eGLImage *eglImageBlueLookup_; + + // Shader parameters + float firstRed_x_; + float firstRed_y_; + GLint attributeVertex_; + GLint attributeTexture_; + GLint textureUniformStep_; + GLint textureUniformSize_; + GLint textureUniformStrideFactor_; + GLint textureUniformBayerFirstRed_; + GLint textureUniformProjMatrix_; + + GLint textureUniformBayerDataIn_; + + // These textures will either point to simple RGB gains or to CCM lookup tables + GLint textureUniformRedLookupDataIn_; + GLint textureUniformGreenLookupDataIn_; + GLint textureUniformBlueLookupDataIn_; + + // Represent per-frame CCM as a uniform vector of floats 3 x 3 + GLint ccmUniformDataIn_; + bool ccmEnabled_; + + Rectangle window_; + std::unique_ptr stats_; + eGL egl_; + GBM gbmSurface_; + uint32_t width_; + uint32_t height_; + GLint glFormat_; + unsigned int bytesPerPixel_; + + GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { + { -1.0f, -1.0f }, + { -1.0f, +1.0f }, + { +1.0f, +1.0f }, + { +1.0f, -1.0f }, + }; + + GLfloat tcoordinates[DEBAYER_OPENGL_COORDS][2] = { + { 0.0f, 1.0f }, + { 0.0f, 0.0f }, + { 1.0f, 0.0f }, + { 1.0f, 1.0f }, + }; +}; + +} /* namespace libcamera */ diff --git a/src/libcamera/software_isp/gpuisp-todo.txt b/src/libcamera/software_isp/gpuisp-todo.txt new file mode 100644 index 000000000..0ff82f81e --- /dev/null +++ b/src/libcamera/software_isp/gpuisp-todo.txt @@ -0,0 +1,42 @@ +List the TODOs in perceived order of ease. + +24 bit output support: + - Take the BPP we already capture and get a 24 bit GBM surface + - Pass a compile-time parameter to the shaders to tell them to do + gl_FragColor = rgb not gl_FragColor = rgba + +Make GPUISP default: + - Right now the environment variable allows over-riding to swtich + from CPU to GPU. + - Once we support 24 BPP output on GPUISP we will have the same + pixel format support as CPU and can set the default to GPU without + regressing functionality + +glTexture1D: + - Initial code was developed for < GLES 2.O but since we have fixed + on GLES >= 2.0 this means we can use glTexture1D + - Provided this is so amend the shaders to do val = texture(x, y, 0); + not texture(x, y, 0.5) the 0.5 is because of using glTexture2D + +Surfaceless GBM: + - We get a GBM surface and then have to swap buffers + If we rework for surfaceless GBM and EGL then the swap buffer can + be dropped. + +dma-buf texture upload: + - Currently we pass the input buffer to glCreateTexture2D. + We should be able to make the upload of the input buffer go faster + by using eglCreateImageKHR and enumerated the dma-buf contents. + +Render-to-texture: + - Right now we render to the GBM provided surface framebuffer + and then memcpy from that buffer to the target output buffer. + This necessitates flushing the cache on the target buffer in + addition to the memcpy(). + - Render-to-texture where we generate the target framebuffer + directly from a dma-buf handle will mitigate the memcpy() phase. + - It should be the case then that the consumer of the output buffer + i.e. the thing that's not libcamera is responsible to flush the cache + if-and-only-if that user writes to the buffer. + - We need to flush the cache on the buffer because we are memcpying() to it. + diff --git a/src/libcamera/software_isp/meson.build b/src/libcamera/software_isp/meson.build index aac7eda7b..c61ac7d59 100644 --- a/src/libcamera/software_isp/meson.build +++ b/src/libcamera/software_isp/meson.build @@ -2,14 +2,23 @@ softisp_enabled = pipelines.contains('simple') summary({'SoftISP support' : softisp_enabled}, section : 'Configuration') +summary({'SoftISP GPU acceleration' : gles_headless_enabled}, section : 'Configuration') if not softisp_enabled subdir_done() endif libcamera_internal_sources += files([ + 'benchmark.cpp', 'debayer.cpp', 'debayer_cpu.cpp', 'software_isp.cpp', 'swstats_cpu.cpp', ]) + +if softisp_enabled and gles_headless_enabled + config_h.set('HAVE_DEBAYER_EGL', 1) + libcamera_internal_sources += files([ + 'debayer_egl.cpp', + ]) +endif diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index 28e2a360e..0e4693b77 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include @@ -25,6 +26,9 @@ #include "libcamera/internal/software_isp/debayer_params.h" #include "debayer_cpu.h" +#if HAVE_DEBAYER_EGL +#include "debayer_egl.h" +#endif /** * \file software_isp.cpp @@ -114,7 +118,20 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor, } stats->statsReady.connect(this, &SoftwareIsp::statsReady); - debayer_ = std::make_unique(std::move(stats)); +#if HAVE_DEBAYER_EGL + const char *softISPMode = utils::secure_getenv("LIBCAMERA_SOFTISP_MODE"); + + if (softISPMode && !strcmp(softISPMode, "gpu")) + debayer_ = std::make_unique(std::move(stats)); +#endif + if (!debayer_) + debayer_ = std::make_unique(std::move(stats)); + + if (!debayer_) { + LOG(SoftwareIsp, Error) << "Failed to create Debayer object"; + return; + } + debayer_->inputBufferReady.connect(this, &SoftwareIsp::inputReady); debayer_->outputBufferReady.connect(this, &SoftwareIsp::outputReady); @@ -159,8 +176,6 @@ SoftwareIsp::SoftwareIsp(PipelineHandler *pipe, const CameraSensor *sensor, metadataReady.emit(frame, metadata); }); ipa_->setSensorControls.connect(this, &SoftwareIsp::setSensorCtrls); - - debayer_->moveToThread(&ispWorkerThread_); } SoftwareIsp::~SoftwareIsp() @@ -262,7 +277,18 @@ int SoftwareIsp::configure(const StreamConfiguration &inputCfg, if (ret < 0) return ret; - return debayer_->configure(inputCfg, outputCfgs, ccmEnabled_); + debayer_->moveToThread(&ispWorkerThread_); + ispWorkerThread_.start(); + + ret = debayer_->invokeMethod(&Debayer::configure, + ConnectionTypeBlocking, inputCfg, + outputCfgs, ccmEnabled_); + if (ret) { + ispWorkerThread_.exit(); + ispWorkerThread_.wait(); + } + + return ret; } /** @@ -343,7 +369,6 @@ int SoftwareIsp::start() if (ret) return ret; - ispWorkerThread_.start(); return 0; } @@ -386,7 +411,7 @@ void SoftwareIsp::stop() void SoftwareIsp::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output) { ipa_->computeParams(frame); - debayer_->invokeMethod(&DebayerCpu::process, + debayer_->invokeMethod(&Debayer::process, ConnectionTypeQueued, frame, input, output, debayerParams_); } @@ -395,9 +420,9 @@ void SoftwareIsp::saveIspParams() debayerParams_ = *sharedParams_; } -void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls) +void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls) { - setSensorControls.emit(sensorControls); + setSensorControls.emit(sensorControls, lensControls); } void SoftwareIsp::statsReady(uint32_t frame, uint32_t bufferId) diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index c520c806e..1ff15f5b7 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -9,13 +9,14 @@ * CPU based software statistics implementation */ -#include "swstats_cpu.h" +#include "libcamera/internal/software_isp/swstats_cpu.h" #include #include #include "libcamera/internal/bayer_format.h" +#include "libcamera/internal/mapped_framebuffer.h" namespace libcamera { @@ -58,6 +59,8 @@ namespace libcamera { * also indicates if processLine2() should be called or not. * This may only be called after a successful configure() call. * + * Valid sizes are: 1x1, 2x2, 4x2 or 4x4. + * * \return The pattern size */ @@ -71,6 +74,19 @@ namespace libcamera { * patternSize height == 1. * It'll process line 0 and 1 for input formats with patternSize height >= 2. * This function may only be called after a successful setWindow() call. + * + * This function takes an array of src pointers each pointing to a line in + * the source image. + * + * Bayer input data requires (patternSize_.height + 1) src pointers, with + * the middle element of the array pointing to the actual line being processed. + * Earlier element(s) will point to the previous line(s) and later element(s) + * to the next line(s). See the DebayerCpu::debayerFn documentation for details. + * + * Planar input data requires a src pointer for each plane, with src[0] pointing + * to the line in plane 0, etc. + * + * For non Bayer single plane input data only a single src pointer is required. */ /** @@ -89,20 +105,6 @@ namespace libcamera { * \brief Signals that the statistics are ready */ -/** - * \typedef SwStatsCpu::statsProcessFn - * \brief Called when there is data to get statistics from - * \param[in] src The input data - * - * These functions take an array of (patternSize_.height + 1) src - * pointers each pointing to a line in the source image. The middle - * element of the array will point to the actual line being processed. - * Earlier element(s) will point to the previous line(s) and later - * element(s) to the next line(s). - * - * See the documentation of DebayerCpu::debayerFn for more details. - */ - /** * \var unsigned int SwStatsCpu::ySkipMask_ * \brief Skip lines where this bitmask is set in y @@ -113,13 +115,6 @@ namespace libcamera { * \brief Statistics window, set by setWindow(), used every line */ -/** - * \var Size SwStatsCpu::patternSize_ - * \brief The size of the bayer pattern - * - * Valid sizes are: 2x2, 4x2 or 4x4. - */ - /** * \var unsigned int SwStatsCpu::xShift_ * \brief The offset of x, applied to window_.x for bayer variants @@ -366,11 +361,14 @@ int SwStatsCpu::setupStandardBayerOrder(BayerFormat::Order order) */ int SwStatsCpu::configure(const StreamConfiguration &inputCfg) { + stride_ = inputCfg.stride; + BayerFormat bayerFormat = BayerFormat::fromPixelFormat(inputCfg.pixelFormat); if (bayerFormat.packing == BayerFormat::Packing::None && setupStandardBayerOrder(bayerFormat.order) == 0) { + processFrame_ = &SwStatsCpu::processBayerFrame2; switch (bayerFormat.bitDepth) { case 8: stats0_ = &SwStatsCpu::statsBGGR8Line0; @@ -391,6 +389,7 @@ int SwStatsCpu::configure(const StreamConfiguration &inputCfg) /* Skip every 3th and 4th line, sample every other 2x2 block */ ySkipMask_ = 0x02; xShift_ = 0; + processFrame_ = &SwStatsCpu::processBayerFrame2; switch (bayerFormat.order) { case BayerFormat::BGGR: @@ -431,4 +430,50 @@ void SwStatsCpu::setWindow(const Rectangle &window) window_.height &= ~(patternSize_.height - 1); } +void SwStatsCpu::processBayerFrame2(MappedFrameBuffer &in) +{ + const uint8_t *src = in.planes()[0].data(); + const uint8_t *linePointers[3]; + + /* Adjust src for starting at window_.y */ + src += window_.y * stride_; + + for (unsigned int y = 0; y < window_.height; y += 2) { + if (y & ySkipMask_) { + src += stride_ * 2; + continue; + } + + /* linePointers[0] is not used by any stats0_ functions */ + linePointers[1] = src; + linePointers[2] = src + stride_; + (this->*stats0_)(linePointers); + src += stride_ * 2; + } +} + +/** + * \brief Calculate statistics for a frame in one go + * \param[in] frame The frame number + * \param[in] bufferId ID of the statistics buffer + * \param[in] input The frame to process + * + * This may only be called after a successful setWindow() call. + */ +void SwStatsCpu::processFrame(uint32_t frame, uint32_t bufferId, FrameBuffer *input) +{ + bench_.startFrame(); + startFrame(); + + MappedFrameBuffer in(input, MappedFrameBuffer::MapFlag::Read); + if (!in.isValid()) { + LOG(SwStatsCpu, Error) << "mmap-ing buffer(s) failed"; + return; + } + + (this->*processFrame_)(in); + finishFrame(frame, bufferId); + bench_.finishFrame(); +} + } /* namespace libcamera */ diff --git a/utils/gen-shader-header.py b/utils/gen-shader-header.py new file mode 100755 index 000000000..6668e648f --- /dev/null +++ b/utils/gen-shader-header.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2025, Bryan O'Donoghue. +# +# Author: Bryan O'Donoghue +# +# A python script which takes a list of shader files and converts into a C +# header. +# +import sys + +try: + with open(sys.argv[2]) as file: + data = file.read() + data_len = len(data) + + name = sys.argv[1].replace(".", "_") + name_len = name + "_len" + + j = 0 + print("unsigned char", name, "[] = {") + for ch in data: + print(f"0x{ord(ch):02x}, ", end="") + j = j + 1 + if j == 16: + print() + j = 0 + if j != 0: + print() + print("};") + + print() + print(f"const unsigned int {name_len}={data_len};") + +except FileNotFoundError: + print(f"File {sys.argv[2]} not found", file=sys.stderr) +except IOError: + print(f"Unable to read {sys.argv[2]}", file=sys.stderr) diff --git a/utils/gen-shader-headers.sh b/utils/gen-shader-headers.sh new file mode 100755 index 000000000..ca4f19f07 --- /dev/null +++ b/utils/gen-shader-headers.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -x + +if [ $# -lt 4 ]; then + echo "Invalid arg count must be >= 5" + exit 1 +fi +src_dir="$1"; shift +build_dir="$1"; shift +build_path=$build_dir/"$1"; shift + +cat < "$build_path" +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* This file is auto-generated, do not edit! */ +/* + * Copyright (C) 2025, Linaro Ltd. + * + */ + +#pragma once + +EOF + +cat <> "$build_path" +/* + * List the names of the shaders at the top of + * header for readability's sake + * +EOF + +for file in "$@"; do + echo "file is $file" + name=$(basename "$build_dir/$file" | tr '.' '_') + echo " * unsigned char $name;" >> "$build_path" +done + +echo "*/" >> "$build_path" + +echo "/* Hex encoded shader data */" >> "$build_path" +for file in "$@"; do + name=$(basename "$build_dir/$file") + "$src_dir/utils/gen-shader-header.py" "$name" "$build_dir/$file" >> "$build_path" + echo >> "$build_path" +done diff --git a/utils/meson.build b/utils/meson.build index 95d657ac9..3deed8ad4 100644 --- a/utils/meson.build +++ b/utils/meson.build @@ -3,5 +3,7 @@ subdir('codegen') subdir('ipu3') +gen_shader_headers = files('gen-shader-headers.sh') + ## Module signing gen_ipa_priv_key = files('gen-ipa-priv-key.sh')