libcamera: shaders: Extend debayer shaders to apply RGB gain values on output

Extend out the bayer fragment shaders to take 3 x 256 byte inputs as
textures from the CPU.

We then use an index to the table to recover the colour-corrected values
provided by the SoftIPA thread.

Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>

include/libcamera/internal/shaders/bayer_1x_packed.frag

@@ -65,6 +65,10 @@ uniform vec2 tex_step;
 uniform vec2 tex_bayer_first_red;
 
 uniform sampler2D tex_y;
+uniform sampler2D red_param;
+uniform sampler2D green_param;
+uniform sampler2D blue_param;
+uniform mat3 ccm;
 
 void main(void)
 {
@@ -212,5 +216,57 @@ void main(void)
 			vec3(patterns.y, C, patterns.x) :
 			vec3(patterns.wz, C));
 
+#if defined(APPLY_CCM_PARAMETERS)
+	/*
+	 *   CCM is a 3x3 in the format
+	 *
+	 *   +--------------+----------------+---------------+
+	 *   | RedRedGain   | RedGreenGain   | RedBlueGain   |
+	 *   +--------------+----------------+---------------+
+	 *   | GreenRedGain | GreenGreenGain | GreenBlueGain |
+	 *   +--------------+----------------+---------------+
+	 *   | BlueRedGain  |  BlueGreenGain | BlueBlueGain  |
+	 *   +--------------+----------------+---------------+
+	 *
+	 *   Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin
+	 *   Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin
+	 *   Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin
+	 *
+	 *   We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm);
+	 *
+	 *   CPU
+	 *   float ccm [] = {
+	 *             RedRedGain,   RedGreenGain,   RedBlueGain,
+	 *             GreenRedGain, GreenGreenGain, GreenBlueGain,
+	 *             BlueRedGain,  BlueGreenGain,  BlueBlueGain,
+	 *   };
+	 *
+	 *   GPU
+	 *   ccm = {
+	 *             RedRedGain,   GreenRedGain,   BlueRedGain,
+	 *             RedGreenGain, GreenGreenGain, BlueGreenGain,
+	 *             RedBlueGain,  GreenBlueGain,  BlueBlueGain,
+	 *   }
+	 *
+	 *   However the indexing for the mat data-type is column major hence
+	 *   ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain
+	 *
+	 */
+	float rin, gin, bin;
+	rin = rgb.r;
+	gin = rgb.g;
+	bin = rgb.b;
+
+	rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]);
+	rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]);
+	rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]);
+
+#elif defined(APPLY_RGB_PARAMETERS)
+	/* Apply bayer params */
+	rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r;
+	rgb.g = texture2D(green_param, vec2(rgb.g, 0.5)).g;
+	rgb.b = texture2D(blue_param, vec2(rgb.b, 0.5)).b;
+#endif
+
 	gl_FragColor = vec4(rgb, 1.0);
 }

include/libcamera/internal/shaders/bayer_8.frag

@@ -21,11 +21,17 @@ precision highp float;
 
 /** Monochrome RGBA or GL_LUMINANCE Bayer encoded texture.*/
 uniform sampler2D       tex_y;
+uniform sampler2D	red_param;
+uniform sampler2D	green_param;
+uniform sampler2D	blue_param;
 varying vec4            center;
 varying vec4            yCoord;
 varying vec4            xCoord;
+uniform mat3		ccm;
 
 void main(void) {
+    vec3 rgb;
+
     #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r
 
     float C = texture2D(tex_y, center.xy).r; // ( 0, 0)
@@ -97,11 +103,65 @@ void main(void) {
     PATTERN.xw  += kB.xw * B;
     PATTERN.xz  += kF.xz * F;
 
-    gl_FragColor.rgb = (alternate.y == 0.0) ?
+    rgb =  (alternate.y == 0.0) ?
         ((alternate.x == 0.0) ?
             vec3(C, PATTERN.xy) :
             vec3(PATTERN.z, C, PATTERN.w)) :
         ((alternate.x == 0.0) ?
             vec3(PATTERN.w, C, PATTERN.z) :
             vec3(PATTERN.yx, C));
+
+#if defined(APPLY_CCM_PARAMETERS)
+	/*
+	 *   CCM is a 3x3 in the format
+	 *
+	 *   +--------------+----------------+---------------+
+	 *   | RedRedGain   | RedGreenGain   | RedBlueGain   |
+	 *   +--------------+----------------+---------------+
+	 *   | GreenRedGain | GreenGreenGain | GreenBlueGain |
+	 *   +--------------+----------------+---------------+
+	 *   | BlueRedGain  |  BlueGreenGain | BlueBlueGain  |
+	 *   +--------------+----------------+---------------+
+	 *
+	 *   Rout = RedRedGain * Rin + RedGreenGain * Gin + RedBlueGain * Bin
+	 *   Gout = GreenRedGain * Rin + GreenGreenGain * Gin + GreenBlueGain * Bin
+	 *   Bout = BlueRedGain * Rin + BlueGreenGain * Gin + BlueBlueGain * Bin
+	 *
+	 *   We upload to the GPU without transposition glUniformMatrix3f(.., .., GL_FALSE, ccm);
+	 *
+	 *   CPU
+	 *   float ccm [] = {
+	 *             RedRedGain,   RedGreenGain,   RedBlueGain,
+	 *             GreenRedGain, GreenGreenGain, GreenBlueGain,
+	 *             BlueRedGain,  BlueGreenGain,  BlueBlueGain,
+	 *   };
+	 *
+	 *   GPU
+	 *   ccm = {
+	 *             RedRedGain,   GreenRedGain,   BlueRedGain,
+	 *             RedGreenGain, GreenGreenGain, BlueGreenGain,
+	 *             RedBlueGain,  GreenBlueGain,  BlueBlueGain,
+	 *   }
+	 *
+	 *   However the indexing for the mat data-type is column major hence
+	 *   ccm[0][0] = RedRedGain, ccm[0][1] = RedGreenGain, ccm[0][2] = RedBlueGain
+	 *
+	 */
+	float rin, gin, bin;
+	rin = rgb.r;
+	gin = rgb.g;
+	bin = rgb.b;
+
+	rgb.r = (rin * ccm[0][0]) + (gin * ccm[0][1]) + (bin * ccm[0][2]);
+	rgb.g = (rin * ccm[1][0]) + (gin * ccm[1][1]) + (bin * ccm[1][2]);
+	rgb.b = (rin * ccm[2][0]) + (gin * ccm[2][1]) + (bin * ccm[2][2]);
+
+#elif defined(APPLY_RGB_PARAMETERS)
+	/* Apply bayer params */
+	rgb.r = texture2D(red_param, vec2(rgb.r, 0.5)).r;
+	rgb.g = texture2D(red_param, vec2(rgb.g, 0.5)).g;
+	rgb.b = texture2D(red_param, vec2(rgb.b, 0.5)).b;
+#endif
+
+    gl_FragColor.rgb = rgb;
 }

src/libcamera/software_isp/debayer_egl.cpp

@@ -99,6 +99,7 @@ int DebayerEGL::getShaderVariableLocations(void)
 	textureUniformRedLookupDataIn_ = glGetUniformLocation(programId_, "red_param");
 	textureUniformGreenLookupDataIn_ = glGetUniformLocation(programId_, "green_param");
 	textureUniformBlueLookupDataIn_ = glGetUniformLocation(programId_, "blue_param");
+	ccmUniformDataIn_ = glGetUniformLocation(programId_, "ccm");
 
 	textureUniformStep_ = glGetUniformLocation(programId_, "tex_step");
 	textureUniformSize_ = glGetUniformLocation(programId_, "tex_size");
@@ -111,6 +112,7 @@ int DebayerEGL::getShaderVariableLocations(void)
 			    << " red_param " << textureUniformRedLookupDataIn_
 			    << " green_param " << textureUniformGreenLookupDataIn_
 			    << " blue_param " << textureUniformBlueLookupDataIn_
+			    << " ccm " << ccmUniformDataIn_
 			    << " tex_step " << textureUniformStep_
 			    << " tex_size " << textureUniformSize_
 			    << " stride_factor " << textureUniformStrideFactor_
@@ -215,8 +217,13 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm
 		break;
 	};
 
-	// Flag to shaders that we have parameter gain tables
+	if (ccmEnabled_) {
-	egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS");
+		// Run the CCM if available
+		egl_.pushEnv(shaderEnv, "#define APPLY_CCM_PARAMETERS");
+	} else {
+		// Flag to shaders that we have parameter gain tables
+		egl_.pushEnv(shaderEnv, "#define APPLY_RGB_PARAMETERS");
+	}
 
 	if (egl_.compileVertexShader(vertexShaderId_, vertexShaderData, vertexShaderDataLen, shaderEnv))
 		goto compile_fail;
@@ -266,6 +273,8 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
 			  const std::vector<std::reference_wrapper<StreamConfiguration>> &outputCfgs,
 			  bool ccmEnabled)
 {
+	GLint maxTextureImageUnits;
+
 	if (getInputConfig(inputCfg.pixelFormat, inputConfig_) != 0)
 		return -EINVAL;
 
@@ -284,7 +293,7 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
 	inputConfig_.stride = inputCfg.stride;
 	width_ = inputCfg.size.width;
 	height_ = inputCfg.size.height;
-	ccmEnabled_ = ccmEnabled = false;
+	ccmEnabled_ = ccmEnabled = true;
 
 	if (outputCfgs.size() != 1) {
 		LOG(Debayer, Error)
@@ -304,30 +313,35 @@ int DebayerEGL::configure(const StreamConfiguration &inputCfg,
 	glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS, &maxTextureImageUnits);
 	LOG(Debayer, Debug) << "Fragment shader maximum texture units " << maxTextureImageUnits;
 
-	if (maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) {
+	if (!ccmEnabled && maxTextureImageUnits < DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS) {
 		LOG(Debayer, Error) << "Fragment shader texture unit count " << maxTextureImageUnits
 				    << " required minimum for RGB gain table lookup " << DEBAYER_EGL_MIN_SIMPLE_RGB_GAIN_TEXTURE_UNITS
 				    << " try using an identity CCM ";
 		return -ENODEV;
 	}
+
 	// Raw bayer input as texture
 	eglImageBayerIn_ = new eGLImage(width_, height_, 32, GL_TEXTURE0, 0);
 	if (!eglImageBayerIn_)
 		return -ENOMEM;
 
-	/// RGB correction tables as 2d textures
+	// Only do the RGB lookup table textures if CCM is disabled
-	// eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate
+	if (!ccmEnabled_) {
-	eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1);
-	if (!eglImageRedLookup_)
-		return -ENOMEM;
 
-	eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2);
+		/// RGB correction tables as 2d textures
-	if (!eglImageGreenLookup_)
+		// eGL doesn't support glTexImage1D so we do a little hack with 2D to compensate
-		return -ENOMEM;
+		eglImageRedLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE1, 1);
+		if (!eglImageRedLookup_)
+			return -ENOMEM;
 
-	eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3);
+		eglImageGreenLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE2, 2);
-	if (!eglImageBlueLookup_)
+		if (!eglImageGreenLookup_)
-		return -ENOMEM;
+			return -ENOMEM;
+
+		eglImageBlueLookup_ = new eGLImage(DebayerParams::kRGBLookupSize, 1, 32, GL_TEXTURE3, 3);
+		if (!eglImageBlueLookup_)
+			return -ENOMEM;
+	}
 
 	// Create a single BO (calling gbm_surface_lock_front_buffer() again before gbm_surface_release_buffer() would create another BO)
 	if (gbmSurface_.mapSurface())
@@ -440,9 +454,11 @@ void DebayerEGL::setShaderVariableValues(void)
 	// To simultaneously sample multiple textures we need to use multiple
 	// texture units
 	glUniform1i(textureUniformBayerDataIn_, eglImageBayerIn_->texture_unit_uniform_id_);
-	glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_);
+	if (!ccmEnabled_) {
-	glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_);
+		glUniform1i(textureUniformRedLookupDataIn_, eglImageRedLookup_->texture_unit_uniform_id_);
-	glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_);
+		glUniform1i(textureUniformGreenLookupDataIn_, eglImageGreenLookup_->texture_unit_uniform_id_);
+		glUniform1i(textureUniformBlueLookupDataIn_, eglImageBlueLookup_->texture_unit_uniform_id_);
+	}
 
 	// These values are:
 	// firstRed = tex_bayer_first_red - bayer_8.vert
@@ -494,9 +510,18 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay
 	egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data());
 
 	// Populate bayer parameters
-	egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, &params.red);
+	if (ccmEnabled_) {
-	egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, &params.green);
+		GLfloat ccm[] = {
-	egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, &params.blue);
+			1, 0, 0,
+			0, 1, 0,
+			0, 0, 1,
+		};
+		glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm);
+	} else {
+		egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, &params.red);
+		egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, &params.green);
+		egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, &params.blue);
+	}
 
 	// Setup the scene
 	setShaderVariableValues();

src/libcamera/software_isp/debayer_egl.h

@@ -134,17 +134,22 @@ private:
 	GLint textureUniformProjMatrix_;
 
 	GLint textureUniformBayerDataIn_;
+
+	// These textures will either point to simple RGB gains or to CCM lookup tables
 	GLint textureUniformRedLookupDataIn_;
 	GLint textureUniformGreenLookupDataIn_;
 	GLint textureUniformBlueLookupDataIn_;
 
+	// Represent per-frame CCM as a uniform vector of floats 3 x 3
+	GLint ccmUniformDataIn_;
+	bool ccmEnabled_;
+
 	Rectangle window_;
 	std::unique_ptr<SwStatsCpu> stats_;
 	eGL egl_;
 	GBM gbmSurface_;
 	uint32_t width_;
 	uint32_t height_;
-	bool ccmEnabled_;
 
 	GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = {
 		{ -1.0f, -1.0f },