diff --git a/include/libcamera/internal/shaders/bayer_1x_packed.frag b/include/libcamera/internal/shaders/bayer_1x_packed.frag index 19b13ad08..90bd64570 100644 --- a/include/libcamera/internal/shaders/bayer_1x_packed.frag +++ b/include/libcamera/internal/shaders/bayer_1x_packed.frag @@ -65,6 +65,10 @@ uniform vec2 tex_step; uniform vec2 tex_bayer_first_red; uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; +uniform mat3 ccm; void main(void) { @@ -212,5 +216,57 @@ void main(void) vec3(patterns.y, C, patterns.x) : vec3(patterns.wz, C)); +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b; +#endif + gl_FragColor = vec4(rgb, 1.0); } diff --git a/include/libcamera/internal/shaders/bayer_8.frag b/include/libcamera/internal/shaders/bayer_8.frag index aa7a1b004..5955c2eaf 100644 --- a/include/libcamera/internal/shaders/bayer_8.frag +++ b/include/libcamera/internal/shaders/bayer_8.frag @@ -21,11 +21,17 @@ precision highp float; /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/ uniform sampler2D tex_y; +uniform sampler2D red_param; +uniform sampler2D green_param; +uniform sampler2D blue_param; varying vec4 center; varying vec4 yCoord; varying vec4 xCoord; +uniform mat3 ccm; void main(void) { + vec3 rgb; + #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r float C = texture2D(tex_y, center.xy).r; // ( 0, 0) @@ -97,11 +103,65 @@ void main(void) { PATTERN.xw += kB.xw * B; PATTERN.xz += kF.xz * F; - gl_FragColor.rgb = (alternate.y == 0.0) ? + rgb = (alternate.y == 0.0) ? ((alternate.x == 0.0) ? vec3(C, PATTERN.xy) : vec3(PATTERN.z, C, PATTERN.w)) : ((alternate.x == 0.0) ? vec3(PATTERN.w, C, PATTERN.z) : vec3(PATTERN.yx, C)); + +#if defined(APPLY_CCM_PARAMETERS) + /* + * CCM is a 3x3 in the format + * + * +--------------+----------------+---------------+ + * | RedRedGain | RedGreenGain | RedBlueGain | + * +--------------+----------------+---------------+ + * | GreenRedGain | GreenGreenGain | GreenBlueGain | + * +--------------+----------------+---------------+ + * | BlueRedGain | BlueGreenGain | BlueBlueGain | + * +--------------+----------------+---------------+ + * + * Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin + * Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin + * Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin + * + * We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm); + * + * CPU + * float ccm [] = { + * RedRedGain, RedGreenGain, RedBlueGain, + * GreenRedGain, GreenGreenGain, GreenBlueGain, + * BlueRedGain, BlueGreenGain, BlueBlueGain, + * }; + * + * GPU + * ccm = { + * RedRedGain, GreenRedGain, BlueRedGain, + * RedGreenGain, GreenGreenGain, BlueGreenGain, + * RedBlueGain, GreenBlueGain, BlueBlueGain, + * } + * + * However the indexing for the mat data-type is column major hence + * ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain + * + */ + float rin, gin, bin; + rin = rgb.r; + gin = rgb.g; + bin = rgb.b; + + rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]); + rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]); + rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]); + +#elif defined(APPLY_RGB_PARAMETERS) + /* Apply bayer params */ + rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r; + rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g; + rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b; +#endif + + gl_FragColor.rgb = rgb; } diff --git a/src/libcamera/software_isp/debayer_egl.cpp b/src/libcamera/software_isp/debayer_egl.cpp index 3fb155119..824cd6d3f 100644 --- a/src/libcamera/software_isp/debayer_egl.cpp +++ b/src/libcamera/software_isp/debayer_egl.cpp @@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void) textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param"); textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param"); textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param"); + ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm"); textureUniformStep_ = glGetUniformLocation(programId_, "tex_step"); textureUniformSize_ = glGetUniformLocation(programId_, "tex_size"); @@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void) << " red_param " << textureUniformRedLookupDataIn_ << " green_param " << textureUniformGreenLookupDataIn_ << " blue_param " << textureUniformBlueLookupDataIn_ + << " ccm " << ccmUniformDataIn_ << " tex_step " << textureUniformStep_ << " tex_size " << textureUniformSize_ << " stride_factor " << textureUniformStrideFactor_ @@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm break; }; - // Flag to shaders that we have parameter gain tables - egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + if (ccmEnabled_) { + // Run the CCM if available + egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS"); + } else { + // Flag to shaders that we have parameter gain tables + egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS"); + } if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv)) goto compile_fail; @@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, const std::vector> &outputCfgs, bool ccmEnabled) { + GLint maxTextureImageUnits; + if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0) return -EINVAL; @@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, inputConfig_.stride = inputCfg.stride; width_ = inputCfg.size.width; height_ = inputCfg.size.height; - ccmEnabled_ = ccmEnabled = false; + ccmEnabled_ = ccmEnabled = true; if (outputCfgs.size() != 1) { LOG(Debayer, Error) @@ -304,30 +313,35 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg, glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits); LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits; - if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { + if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) { LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS << " try using an identity CCM "; return -ENODEV; } + // Raw bayer input as texture eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0); if (!eglImageBayerIn_) return -ENOMEM; - /// RGB correction tables as 2d textures - // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate - eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); - if (!eglImageRedLookup_) - return -ENOMEM; + // Only do the RGB lookup table textures if CCM is disabled + if (!ccmEnabled_) { - eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); - if (!eglImageGreenLookup_) - return -ENOMEM; + /// RGB correction tables as 2d textures + // eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate + eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1); + if (!eglImageRedLookup_) + return -ENOMEM; - eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); - if (!eglImageBlueLookup_) - return -ENOMEM; + eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2); + if (!eglImageGreenLookup_) + return -ENOMEM; + + eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3); + if (!eglImageBlueLookup_) + return -ENOMEM; + } // Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO) if (gbmSurface_.mapSurface()) @@ -440,9 +454,11 @@ void DebayerEGL::setShaderVariableValues(void) // To simultaneously sample multiple textures we need to use multiple // texture units glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_); - glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); - glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + if (!ccmEnabled_) { + glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_); + glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_); + } // These values are: // firstRed = tex_bayer_first_red - bayer_8.vert @@ -494,9 +510,18 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data()); // Populate bayer parameters - egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); - egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); - egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + if (ccmEnabled_) { + GLfloat ccm[] = { + 1, 0, 0, + 0, 1, 0, + 0, 0, 1, + }; + glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm); + } else { + egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.red); + egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.green); + egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, ¶ms.blue); + } // Setup the scene setShaderVariableValues(); diff --git a/src/libcamera/software_isp/debayer_egl.h b/src/libcamera/software_isp/debayer_egl.h index c0fc220f8..94bc6fc41 100644 --- a/src/libcamera/software_isp/debayer_egl.h +++ b/src/libcamera/software_isp/debayer_egl.h @@ -134,17 +134,22 @@ private: GLint textureUniformProjMatrix_; GLint textureUniformBayerDataIn_; + + // These textures will either point to simple RGB gains or to CCM lookup tables GLint textureUniformRedLookupDataIn_; GLint textureUniformGreenLookupDataIn_; GLint textureUniformBlueLookupDataIn_; + // Represent per-frame CCM as a uniform vector of floats 3 x 3 + GLint ccmUniformDataIn_; + bool ccmEnabled_; + Rectangle window_; std::unique_ptr stats_; eGL egl_; GBM gbmSurface_; uint32_t width_; uint32_t height_; - bool ccmEnabled_; GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = { { -1.0f, -1.0f },