libcamera: software_isp: GPU support for unpacked 10/12-bit formats

The GPU processing supports 8-bit sensor formats and 10/12-bit packed
formats.  Support for 10/12-bit unpacked formats is missing, let's add
it.

10/12-bit unpacked formats use two adjacent bytes to store the value.
This means the 8-bit shaders can be used if we can modify them for
additional support of 16-bit addressing.  This requires the following
modifications:

- Using GL_RG (two bytes per pixel) instead of GL_LUMINANCE (one byte
  per pixel) as the texture format for the given input formats.

- Setting the texture width to the number of pixels rather than the
  number of bytes.

- Making the definition of `fetch' macro variable, according to the
  pixel format.

- Using only `fetch' for accessing the texture.

Signed-off-by: Milan Zamazal <mzamazal@redhat.com>
Signed-off-by: Bryan O'Donoghue <bryan.odonoghue@linaro.org>

include/libcamera/internal/shaders/bayer_8.frag

@@ -32,9 +32,17 @@ uniform mat3		ccm;
 void main(void) {
     vec3 rgb;
 
+    #if defined(RAW10P)
+    #define pixel(p) p.r / 4.0 + p.g * 64.0
+    #define fetch(x, y) pixel(texture2D(tex_y, vec2(x, y)))
+    #elif defined(RAW12P)
+    #define pixel(p) p.r / 16.0 + p.g * 16.0
+    #define fetch(x, y) pixel(texture2D(tex_y, vec2(x, y)))
+    #else
     #define fetch(x, y) texture2D(tex_y, vec2(x, y)).r
+    #endif
 
-    float C = texture2D(tex_y, center.xy).r; // ( 0, 0)
+    float C = fetch(center.x, center.y); // ( 0, 0)
     const vec4 kC = vec4( 4.0,  6.0,  5.0,  5.0) / 8.0;
 
     // Determine which of four types of pixels we are on.

src/libcamera/software_isp/debayer_egl.cpp

@@ -151,6 +151,8 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm
 	}
 
 	// Pixel location parameters
+	glFormat_ = GL_LUMINANCE;
+	bytesPerPixel_ = 1;
 	switch (inputFormat) {
 	case libcamera::formats::SBGGR8:
 	case libcamera::formats::SBGGR10_CSI2P:
@@ -197,20 +199,38 @@ int DebayerEGL::initBayerShaders(PixelFormat inputFormat, PixelFormat outputForm
 	case libcamera::formats::SGRBG10_CSI2P:
 	case libcamera::formats::SRGGB10_CSI2P:
 		egl_.pushEnv(shaderEnv, "#define RAW10P");
+		if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) {
+			fragmentShaderData = bayer_8_frag;
+			fragmentShaderDataLen = bayer_8_frag_len;
+			vertexShaderData = bayer_8_vert;
+			vertexShaderDataLen = bayer_8_vert_len;
+			glFormat_ = GL_RG;
+			bytesPerPixel_ = 2;
+		} else {
 			fragmentShaderData = bayer_1x_packed_frag;
 			fragmentShaderDataLen = bayer_1x_packed_frag_len;
 			vertexShaderData = identity_vert;
 			vertexShaderDataLen = identity_vert_len;
+		}
 		break;
 	case libcamera::formats::SBGGR12_CSI2P:
 	case libcamera::formats::SGBRG12_CSI2P:
 	case libcamera::formats::SGRBG12_CSI2P:
 	case libcamera::formats::SRGGB12_CSI2P:
 		egl_.pushEnv(shaderEnv, "#define RAW12P");
+		if (BayerFormat::fromPixelFormat(inputFormat).packing == BayerFormat::Packing::None) {
+			fragmentShaderData = bayer_8_frag;
+			fragmentShaderDataLen = bayer_8_frag_len;
+			vertexShaderData = bayer_8_vert;
+			vertexShaderDataLen = bayer_8_vert_len;
+			glFormat_ = GL_RG;
+			bytesPerPixel_ = 2;
+		} else {
 			fragmentShaderData = bayer_1x_packed_frag;
 			fragmentShaderDataLen = bayer_1x_packed_frag_len;
 			vertexShaderData = identity_vert;
 			vertexShaderDataLen = identity_vert_len;
+		}
 		break;
 	default:
 		goto invalid_fmt;
@@ -430,7 +450,7 @@ void DebayerEGL::setShaderVariableValues(void)
 	GLfloat firstRed[] = { firstRed_x_, firstRed_y_ };
 	GLfloat imgSize[] = { (GLfloat)width_,
 			      (GLfloat)height_ };
-	GLfloat Step[] = { 1.0f / (inputConfig_.stride - 1),
+	GLfloat Step[] = { static_cast<float>(bytesPerPixel_) / (inputConfig_.stride - 1),
 			   1.0f / (height_ - 1) };
 	GLfloat Stride = 1.0f;
 	GLfloat projIdentityMatrix[] = {
@@ -507,7 +527,7 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay
 
 	// Greate a standard texture
 	// we will replace this with the DMA version at some point
-	egl_.createTexture2D(eglImageBayerIn_, inputConfig_.stride, height_, in.planes()[0].data());
+	egl_.createTexture2D(eglImageBayerIn_, glFormat_, inputConfig_.stride / bytesPerPixel_, height_, in.planes()[0].data());
 
 	// Populate bayer parameters
 	if (ccmEnabled_) {
@@ -518,9 +538,9 @@ void DebayerEGL::debayerGPU(MappedFrameBuffer &in, MappedFrameBuffer &out, Debay
 		};
 		glUniformMatrix3fv(ccmUniformDataIn_, 1, GL_FALSE, ccm);
 	} else {
-		egl_.createTexture2D(eglImageRedLookup_, DebayerParams::kRGBLookupSize, 1, &params.red);
+		egl_.createTexture2D(eglImageRedLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, &params.red);
-		egl_.createTexture2D(eglImageGreenLookup_, DebayerParams::kRGBLookupSize, 1, &params.green);
+		egl_.createTexture2D(eglImageGreenLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, &params.green);
-		egl_.createTexture2D(eglImageBlueLookup_, DebayerParams::kRGBLookupSize, 1, &params.blue);
+		egl_.createTexture2D(eglImageBlueLookup_, GL_LUMINANCE, DebayerParams::kRGBLookupSize, 1, &params.blue);
 	}
 
 	// Setup the scene

src/libcamera/software_isp/debayer_egl.h

@@ -150,6 +150,8 @@ private:
 	GBM gbmSurface_;
 	uint32_t width_;
 	uint32_t height_;
+	GLint glFormat_;
+	unsigned int bytesPerPixel_;
 
 	GLfloat vcoordinates[DEBAYER_OPENGL_COORDS][2] = {
 		{ -1.0f, -1.0f },