NekoCWD/libcamera
NekoCWD/libcamera
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 7 Wiki Activity Actions

ipa: raspberrypi: Generalise the autofocus algorithm

Browse source 
Remove any hard-coded assumptions about the target hardware platform
from the autofocus algorithm. Instead, use the "target" string provided
by the camera tuning config and generalised statistics structures to
determing parameters such as grid and region sizes.

Additionally, PDAF statistics are represented by a generalised region
statistics structure to be device agnostic.

These changes also require the autofocus algorithm to initialise
region weights on the first frame's prepare()/process() call rather
than during initialisation.

Signed-off-by: Nick Hollinghurst <nick.hollinghurst@raspberrypi.com>
Signed-off-by: Naushir Patuck <naush@raspberrypi.com>
Tested-by: Naushir Patuck <naush@raspberrypi.com>
Reviewed-by: Naushir Patuck <naush@raspberrypi.com>
Reviewed-by: Jacopo Mondi <jacopo.mondi@ideasonboard.com>
Signed-off-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
This commit is contained in:
Nick Hollinghurst 2023-03-27 13:20:29 +01:00 • committed by Kieran Bingham
parent e51a9f7b94
commit 43f57f26b4
4 changed files with 134 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

23
src/ipa/raspberrypi/cam_helper_imx708.cpp
View file

@ -69,11 +69,14 @@ private:
/* Largest long exposure scale factor given as a left shift on the frame length. */ /* Largest long exposure scale factor given as a left shift on the frame length. */
static constexpr int longExposureShiftMax = 7; static constexpr int longExposureShiftMax = 7;
static constexpr int pdafStatsRows = 12;
static constexpr int pdafStatsCols = 16;
void populateMetadata(const MdParser::RegisterMap &registers, void populateMetadata(const MdParser::RegisterMap &registers,
Metadata &metadata) const override; Metadata &metadata) const override;
static bool parsePdafData(const uint8_t *ptr, size_t len, unsigned bpp, static bool parsePdafData(const uint8_t *ptr, size_t len, unsigned bpp,
PdafData &pdaf); PdafRegions &pdaf);
bool parseAEHist(const uint8_t *ptr, size_t len, unsigned bpp); bool parseAEHist(const uint8_t *ptr, size_t len, unsigned bpp);
void putAGCStatistics(StatisticsPtr stats); void putAGCStatistics(StatisticsPtr stats);
@ -120,11 +123,11 @@ void CamHelperImx708::prepare(libcamera::Span<const uint8_t> buffer, Metadata &m
size_t bytesPerLine = (mode_.width * mode_.bitdepth) >> 3; size_t bytesPerLine = (mode_.width * mode_.bitdepth) >> 3;
if (buffer.size() > 2 * bytesPerLine) { if (buffer.size() > 2 * bytesPerLine) {
PdafData pdaf; PdafRegions pdaf;
if (parsePdafData(&buffer[2 * bytesPerLine], if (parsePdafData(&buffer[2 * bytesPerLine],
buffer.size() - 2 * bytesPerLine, buffer.size() - 2 * bytesPerLine,
mode_.bitdepth, pdaf)) mode_.bitdepth, pdaf))
metadata.set("pdaf.data", pdaf); metadata.set("pdaf.regions", pdaf);
} }
/* Parse AE-HIST data where present */ /* Parse AE-HIST data where present */
@ -239,7 +242,7 @@ void CamHelperImx708::populateMetadata(const MdParser::RegisterMap &registers,
} }
bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len, bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len,
unsigned bpp, PdafData &pdaf) unsigned bpp, PdafRegions &pdaf)
{ {
size_t step = bpp >> 1; /* bytes per PDAF grid entry */ size_t step = bpp >> 1; /* bytes per PDAF grid entry */
@ -248,13 +251,17 @@ bool CamHelperImx708::parsePdafData(const uint8_t *ptr, size_t len,
return false; return false;
} }
pdaf.init({ pdafStatsCols, pdafStatsRows });
ptr += 2 * step; ptr += 2 * step;
for (unsigned i = 0; i < PDAF_DATA_ROWS; ++i) { for (unsigned i = 0; i < pdafStatsRows; ++i) {
for (unsigned j = 0; j < PDAF_DATA_COLS; ++j) { for (unsigned j = 0; j < pdafStatsCols; ++j) {
unsigned c = (ptr[0] << 3) | (ptr[1] >> 5); unsigned c = (ptr[0] << 3) | (ptr[1] >> 5);
int p = (((ptr[1] & 0x0F) - (ptr[1] & 0x10)) << 6) | (ptr[2] >> 2); int p = (((ptr[1] & 0x0F) - (ptr[1] & 0x10)) << 6) | (ptr[2] >> 2);
pdaf.conf[i][j] = c; PdafData pdafData;
pdaf.phase[i][j] = c ? p : 0; pdafData.conf = c;
pdafData.phase = c ? p : 0;
pdaf.set(libcamera::Point(j, i), { pdafData, 1, 0 });
ptr += step; ptr += step;
} }
} }

21
src/ipa/raspberrypi/controller/pdaf_data.h
View file

@ -2,20 +2,23 @@
/* /*
* Copyright (C) 2022, Raspberry Pi Ltd * Copyright (C) 2022, Raspberry Pi Ltd
* *
* pdaf_data.h - PDAF Metadata; for now this is * pdaf_data.h - PDAF Metadata
* largely based on IMX708's PDAF "Type 1" output.
*/ */
#pragma once #pragma once
#include <stdint.h> #include <stdint.h>
#define PDAF_DATA_ROWS 12 #include "region_stats.h"
#define PDAF_DATA_COLS 16
namespace RPiController {
struct PdafData { struct PdafData {
/* Confidence values, in raster order, in arbitrary units */ /* Confidence, in arbitrary units */
uint16_t conf[PDAF_DATA_ROWS][PDAF_DATA_COLS]; uint16_t conf;
/* Phase error, in s16 Q4 format (S.11.4) */
/* Phase error, in raster order, in s11 Q4 format (S.6.4) */ int16_t phase;
int16_t phase[PDAF_DATA_ROWS][PDAF_DATA_COLS];
}; };
using PdafRegions = RegionStats<PdafData>;
} /* namespace RPiController */

168
src/ipa/raspberrypi/controller/rpi/af.cpp
View file

@ -174,9 +174,8 @@ Af::Af(Controller *controller)
statsRegion_(0, 0, 0, 0), statsRegion_(0, 0, 0, 0),
windows_(), windows_(),
useWindows_(false), useWindows_(false),
phaseWeights_{}, phaseWeights_(),
contrastWeights_{}, contrastWeights_(),
sumWeights_(0),
scanState_(ScanState::Idle), scanState_(ScanState::Idle),
initted_(false), initted_(false),
ftarget_(-1.0), ftarget_(-1.0),
@ -190,7 +189,15 @@ Af::Af(Controller *controller)
scanData_(), scanData_(),
reportState_(AfState::Idle) reportState_(AfState::Idle)
{ {
scanData_.reserve(24); /*
* Reserve space for data, to reduce memory fragmentation. It's too early
* to query the size of the PDAF (from camera) and Contrast (from ISP)
* statistics, but these are plausible upper bounds.
*/
phaseWeights_.w.reserve(16 * 12);
contrastWeights_.w.reserve(getHardwareConfig().focusRegions.width *
getHardwareConfig().focusRegions.height);
scanData_.reserve(32);
} }
Af::~Af() Af::~Af()
@ -226,7 +233,7 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met
<< statsRegion_.y << ',' << statsRegion_.y << ','
<< statsRegion_.width << ',' << statsRegion_.width << ','
<< statsRegion_.height; << statsRegion_.height;
computeWeights(); invalidateWeights();
if (scanState_ >= ScanState::Coarse && scanState_ < ScanState::Settle) { if (scanState_ >= ScanState::Coarse && scanState_ < ScanState::Settle) {
/* /*
@ -239,111 +246,99 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met
skipCount_ = cfg_.skipFrames; skipCount_ = cfg_.skipFrames;
} }
void Af::computeWeights() void Af::computeWeights(RegionWeights *wgts, unsigned rows, unsigned cols)
{ {
constexpr int MaxCellWeight = 240 / (int)MaxWindows; wgts->rows = rows;
wgts->cols = cols;
wgts->sum = 0;
wgts->w.resize(rows * cols);
std::fill(wgts->w.begin(), wgts->w.end(), 0);
sumWeights_ = 0; if (rows > 0 && cols > 0 && useWindows_ &&
for (int i = 0; i < PDAF_DATA_ROWS; ++i) statsRegion_.height >= rows && statsRegion_.width >= cols) {
std::fill(phaseWeights_[i], phaseWeights_[i] + PDAF_DATA_COLS, 0);
if (useWindows_ &&
statsRegion_.width >= PDAF_DATA_COLS && statsRegion_.height >= PDAF_DATA_ROWS) {
/* /*
* Here we just merge all of the given windows, weighted by area. * Here we just merge all of the given windows, weighted by area.
* \todo Perhaps a better approach might be to find the phase in each * \todo Perhaps a better approach might be to find the phase in each
* window and choose either the closest or the highest-confidence one? * window and choose either the closest or the highest-confidence one?
* * Ensure weights sum to less than (1<<16). 46080 is a "round number"
* Using mostly "int" arithmetic, because Rectangle has signed x, y * below 65536, for better rounding when window size is a simple
* fraction of image dimensions.
*/ */
int cellH = (int)(statsRegion_.height / PDAF_DATA_ROWS); const unsigned maxCellWeight = 46080u / (MaxWindows * rows * cols);
int cellW = (int)(statsRegion_.width / PDAF_DATA_COLS); const unsigned cellH = statsRegion_.height / rows;
int cellA = cellH * cellW; const unsigned cellW = statsRegion_.width / cols;
const unsigned cellA = cellH * cellW;
for (auto &w : windows_) { for (auto &w : windows_) {
for (int i = 0; i < PDAF_DATA_ROWS; ++i) { for (unsigned r = 0; r < rows; ++r) {
int y0 = std::max(statsRegion_.y + cellH * i, w.y); int y0 = std::max(statsRegion_.y + (int)(cellH * r), w.y);
int y1 = std::min(statsRegion_.y + cellH * (i + 1), w.y + (int)(w.height)); int y1 = std::min(statsRegion_.y + (int)(cellH * (r + 1)),
w.y + (int)(w.height));
if (y0 >= y1) if (y0 >= y1)
continue; continue;
y1 -= y0; y1 -= y0;
for (int j = 0; j < PDAF_DATA_COLS; ++j) { for (unsigned c = 0; c < cols; ++c) {
int x0 = std::max(statsRegion_.x + cellW * j, w.x); int x0 = std::max(statsRegion_.x + (int)(cellW * c), w.x);
int x1 = std::min(statsRegion_.x + cellW * (j + 1), w.x + (int)(w.width)); int x1 = std::min(statsRegion_.x + (int)(cellW * (c + 1)),
w.x + (int)(w.width));
if (x0 >= x1) if (x0 >= x1)
continue; continue;
int a = y1 * (x1 - x0); unsigned a = y1 * (x1 - x0);
a = (MaxCellWeight * a + cellA - 1) / cellA; a = (maxCellWeight * a + cellA - 1) / cellA;
phaseWeights_[i][j] += a; wgts->w[r * cols + c] += a;
sumWeights_ += a; wgts->sum += a;
} }
} }
} }
} }
if (sumWeights_ == 0) { if (wgts->sum == 0) {
/* /* Default AF window is the middle 1/2 width of the middle 1/3 height */
* Default AF window is the middle 1/2 width of the middle 1/3 height for (unsigned r = rows / 3; r < rows - rows / 3; ++r) {
* since this maps nicely to both PDAF (16x12) and Focus (4x3) grids. for (unsigned c = cols / 4; c < cols - cols / 4; ++c) {
*/ wgts->w[r * cols + c] = 1;
for (int i = PDAF_DATA_ROWS / 3; i < 2 * PDAF_DATA_ROWS / 3; ++i) { wgts->sum += 1;
for (int j = PDAF_DATA_COLS / 4; j < 3 * PDAF_DATA_COLS / 4; ++j) {
phaseWeights_[i][j] = MaxCellWeight;
sumWeights_ += MaxCellWeight;
} }
} }
} }
/* Scale from PDAF to Focus Statistics grid (which has fixed size 4x3) */
constexpr int FocusStatsRows = 3;
constexpr int FocusStatsCols = 4;
static_assert(FOCUS_REGIONS == FocusStatsRows * FocusStatsCols);
static_assert(PDAF_DATA_ROWS % FocusStatsRows == 0);
static_assert(PDAF_DATA_COLS % FocusStatsCols == 0);
constexpr int YFactor = PDAF_DATA_ROWS / FocusStatsRows;
constexpr int XFactor = PDAF_DATA_COLS / FocusStatsCols;
LOG(RPiAf, Debug) << "Recomputed weights:";
for (int i = 0; i < FocusStatsRows; ++i) {
for (int j = 0; j < FocusStatsCols; ++j) {
unsigned w = 0;
for (int y = 0; y < YFactor; ++y)
for (int x = 0; x < XFactor; ++x)
w += phaseWeights_[YFactor * i + y][XFactor * j + x];
contrastWeights_[FocusStatsCols * i + j] = w;
}
LOG(RPiAf, Debug) << " "
<< contrastWeights_[FocusStatsCols * i + 0] << " "
<< contrastWeights_[FocusStatsCols * i + 1] << " "
<< contrastWeights_[FocusStatsCols * i + 2] << " "
<< contrastWeights_[FocusStatsCols * i + 3];
}
} }
bool Af::getPhase(PdafData const &data, double &phase, double &conf) const void Af::invalidateWeights()
{ {
phaseWeights_.sum = 0;
contrastWeights_.sum = 0;
}
bool Af::getPhase(PdafRegions const &regions, double &phase, double &conf)
{
libcamera::Size size = regions.size();
if (size.height != phaseWeights_.rows || size.width != phaseWeights_.cols ||
phaseWeights_.sum == 0) {
LOG(RPiAf, Debug) << "Recompute Phase weights " << size.width << 'x' << size.height;
computeWeights(&phaseWeights_, size.height, size.width);
}
uint32_t sumWc = 0; uint32_t sumWc = 0;
int64_t sumWcp = 0; int64_t sumWcp = 0;
for (unsigned i = 0; i < regions.numRegions(); ++i) {
for (unsigned i = 0; i < PDAF_DATA_ROWS; ++i) { unsigned w = phaseWeights_.w[i];
for (unsigned j = 0; j < PDAF_DATA_COLS; ++j) { if (w) {
if (phaseWeights_[i][j]) { const PdafData &data = regions.get(i).val;
uint32_t c = data.conf[i][j]; unsigned c = data.conf;
if (c >= cfg_.confThresh) { if (c >= cfg_.confThresh) {
if (c > cfg_.confClip) if (c > cfg_.confClip)
c = cfg_.confClip; c = cfg_.confClip;
c -= (cfg_.confThresh >> 2); c -= (cfg_.confThresh >> 2);
sumWc += phaseWeights_[i][j] * c; sumWc += w * c;
c -= (cfg_.confThresh >> 2); c -= (cfg_.confThresh >> 2);
sumWcp += phaseWeights_[i][j] * data.phase[i][j] * (int64_t)c; sumWcp += (int64_t)(w * c) * (int64_t)data.phase;
}
} }
} }
} }
if (0 < sumWeights_ && sumWeights_ <= sumWc) { if (0 < phaseWeights_.sum && phaseWeights_.sum <= sumWc) {
phase = (double)sumWcp / (double)sumWc; phase = (double)sumWcp / (double)sumWc;
conf = (double)sumWc / (double)sumWeights_; conf = (double)sumWc / (double)phaseWeights_.sum;
return true; return true;
} else { } else {
phase = 0.0; phase = 0.0;
@ -352,14 +347,21 @@ bool Af::getPhase(PdafData const &data, double &phase, double &conf) const
} }
} }
double Af::getContrast(const FocusRegions &focusStats) const double Af::getContrast(const FocusRegions &focusStats)
{ {
uint32_t sumWc = 0; libcamera::Size size = focusStats.size();
if (size.height != contrastWeights_.rows ||
size.width != contrastWeights_.cols || contrastWeights_.sum == 0) {
LOG(RPiAf, Debug) << "Recompute Contrast weights "
<< size.width << 'x' << size.height;
computeWeights(&contrastWeights_, size.height, size.width);
}
uint64_t sumWc = 0;
for (unsigned i = 0; i < focusStats.numRegions(); ++i) for (unsigned i = 0; i < focusStats.numRegions(); ++i)
sumWc += contrastWeights_[i] * focusStats.get(i).val; sumWc += contrastWeights_.w[i] * focusStats.get(i).val;
return (sumWeights_ == 0) ? 0.0 : (double)sumWc / (double)sumWeights_; return (contrastWeights_.sum > 0) ? ((double)sumWc / (double)contrastWeights_.sum) : 0.0;
} }
void Af::doPDAF(double phase, double conf) void Af::doPDAF(double phase, double conf)
@ -623,14 +625,14 @@ void Af::prepare(Metadata *imageMetadata)
if (initted_) { if (initted_) {
/* Get PDAF from the embedded metadata, and run AF algorithm core */ /* Get PDAF from the embedded metadata, and run AF algorithm core */
PdafData data; PdafRegions regions;
double phase = 0.0, conf = 0.0; double phase = 0.0, conf = 0.0;
double oldFt = ftarget_; double oldFt = ftarget_;
double oldFs = fsmooth_; double oldFs = fsmooth_;
ScanState oldSs = scanState_; ScanState oldSs = scanState_;
uint32_t oldSt = stepCount_; uint32_t oldSt = stepCount_;
if (imageMetadata->get("pdaf.data", data) == 0) if (imageMetadata->get("pdaf.regions", regions) == 0)
getPhase(data, phase, conf); getPhase(regions, phase, conf);
doAF(prevContrast_, phase, conf); doAF(prevContrast_, phase, conf);
updateLensPosition(); updateLensPosition();
LOG(RPiAf, Debug) << std::fixed << std::setprecision(2) LOG(RPiAf, Debug) << std::fixed << std::setprecision(2)
@ -691,7 +693,7 @@ void Af::setMetering(bool mode)
{ {
if (useWindows_ != mode) { if (useWindows_ != mode) {
useWindows_ = mode; useWindows_ = mode;
computeWeights(); invalidateWeights();
} }
} }
@ -708,7 +710,9 @@ void Af::setWindows(libcamera::Span<libcamera::Rectangle const> const &wins)
if (windows_.size() >= MaxWindows) if (windows_.size() >= MaxWindows)
break; break;
} }
computeWeights();
if (useWindows_)
invalidateWeights();
} }
bool Af::setLensPosition(double dioptres, int *hwpos) bool Af::setLensPosition(double dioptres, int *hwpos)

28
src/ipa/raspberrypi/controller/rpi/af.h
View file

@ -11,12 +11,6 @@
#include "../pdaf_data.h" #include "../pdaf_data.h"
#include "../pwl.h" #include "../pwl.h"
/*
* \todo FOCUS_REGIONS is taken from bcm2835-isp.h, but should be made as a
* generic RegionStats structure.
*/
#define FOCUS_REGIONS 12
/* /*
* This algorithm implements a hybrid of CDAF and PDAF, favouring PDAF. * This algorithm implements a hybrid of CDAF and PDAF, favouring PDAF.
* *
@ -121,9 +115,20 @@ private:
double conf; double conf;
}; };
void computeWeights(); struct RegionWeights {
bool getPhase(PdafData const &data, double &phase, double &conf) const; unsigned rows;
double getContrast(const FocusRegions &focusStats) const; unsigned cols;
uint32_t sum;
std::vector<uint16_t> w;
RegionWeights()
: rows(0), cols(0), sum(0), w() {}
};
void computeWeights(RegionWeights *wgts, unsigned rows, unsigned cols);
void invalidateWeights();
bool getPhase(PdafRegions const &regions, double &phase, double &conf);
double getContrast(const FocusRegions &focusStats);
void doPDAF(double phase, double conf); void doPDAF(double phase, double conf);
bool earlyTerminationByPhase(double phase); bool earlyTerminationByPhase(double phase);
double findPeak(unsigned index) const; double findPeak(unsigned index) const;
@ -143,9 +148,8 @@ private:
libcamera::Rectangle statsRegion_; libcamera::Rectangle statsRegion_;
std::vector<libcamera::Rectangle> windows_; std::vector<libcamera::Rectangle> windows_;
bool useWindows_; bool useWindows_;
uint8_t phaseWeights_[PDAF_DATA_ROWS][PDAF_DATA_COLS]; RegionWeights phaseWeights_;
uint16_t contrastWeights_[FOCUS_REGIONS]; RegionWeights contrastWeights_;
uint32_t sumWeights_;
/* Working state. */ /* Working state. */
ScanState scanState_; ScanState scanState_;