Mirror/libcamera
1
0
Fork 0
mirror of https://git.libcamera.org/libcamera/libcamera.git synced 2025-07-25 17:45:06 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions

ipa: raspberrypi: Use an array of RPiController::Metadata objects

Browse source 
Allow the IPA to cycle through an array of RPiController::Metadata objects, one
per prepare()/process() invocation, when running the controller algorithms. This
allows historical metadata objects to be retained, and subsequently passed into
the controller algorithms on future frames. At present, only a single index into
this array is used.

This change provides a route to fixing a problem with the AGC algorithm, where
if a manual shutter/gain is requested, the algorithm does not currently retain
any context of the total exposure that it has calculated. As a result, the
wrong digital gain would be applied when the frame with the manual shutter/gain
is processed by the ISP.

Signed-off-by: Naushir Patuck <naush@raspberrypi.com>
Reviewed-by: David Plowman <david.plowman@raspberrypi.com>
Tested-by: David Plowman <david.plowman@raspberrypi.com>
Reviewed-by: Kieran Bingham <kieran.bingham@ideasonboard.com>
Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
This commit is contained in:
Naushir Patuck 2022-11-15 09:07:53 +00:00 • committed by Laurent Pinchart
parent 087f0c3c42
commit 44cffefd60
1 changed files with 35 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

63
src/ipa/raspberrypi/raspberrypi.cpp
View file

@ -57,6 +57,9 @@ namespace libcamera {
using namespace std::literals::chrono_literals; using namespace std::literals::chrono_literals;
using utils::Duration; using utils::Duration;
/* Number of metadata objects available in the context list. */
constexpr unsigned int numMetadataContexts = 16;
/* Configure the sensor with these values initially. */ /* Configure the sensor with these values initially. */
constexpr double defaultAnalogueGain = 1.0; constexpr double defaultAnalogueGain = 1.0;
constexpr Duration defaultExposureTime = 20.0ms; constexpr Duration defaultExposureTime = 20.0ms;
@ -163,7 +166,7 @@ private:
/* Raspberry Pi controller specific defines. */ /* Raspberry Pi controller specific defines. */
std::unique_ptr<RPiController::CamHelper> helper_; std::unique_ptr<RPiController::CamHelper> helper_;
RPiController::Controller controller_; RPiController::Controller controller_;
RPiController::Metadata rpiMetadata_; std::array<RPiController::Metadata, numMetadataContexts> rpiMetadata_;
/* /*
* We count frames to decide if the frame must be hidden (e.g. from * We count frames to decide if the frame must be hidden (e.g. from
@ -539,14 +542,15 @@ void IPARPi::signalIspPrepare(const ISPConfig &data)
void IPARPi::reportMetadata() void IPARPi::reportMetadata()
{ {
std::unique_lock<RPiController::Metadata> lock(rpiMetadata_); RPiController::Metadata &rpiMetadata = rpiMetadata_[0];
std::unique_lock<RPiController::Metadata> lock(rpiMetadata);
/* /*
* Certain information about the current frame and how it will be * Certain information about the current frame and how it will be
* processed can be extracted and placed into the libcamera metadata * processed can be extracted and placed into the libcamera metadata
* buffer, where an application could query it. * buffer, where an application could query it.
*/ */
DeviceStatus *deviceStatus = rpiMetadata_.getLocked<DeviceStatus>("device.status"); DeviceStatus *deviceStatus = rpiMetadata.getLocked<DeviceStatus>("device.status");
if (deviceStatus) { if (deviceStatus) {
libcameraMetadata_.set(controls::ExposureTime, libcameraMetadata_.set(controls::ExposureTime,
deviceStatus->shutterSpeed.get<std::micro>()); deviceStatus->shutterSpeed.get<std::micro>());
@ -557,24 +561,24 @@ void IPARPi::reportMetadata()
libcameraMetadata_.set(controls::SensorTemperature, *deviceStatus->sensorTemperature); libcameraMetadata_.set(controls::SensorTemperature, *deviceStatus->sensorTemperature);
} }
AgcStatus *agcStatus = rpiMetadata_.getLocked<AgcStatus>("agc.status"); AgcStatus *agcStatus = rpiMetadata.getLocked<AgcStatus>("agc.status");
if (agcStatus) { if (agcStatus) {
libcameraMetadata_.set(controls::AeLocked, agcStatus->locked); libcameraMetadata_.set(controls::AeLocked, agcStatus->locked);
libcameraMetadata_.set(controls::DigitalGain, agcStatus->digitalGain); libcameraMetadata_.set(controls::DigitalGain, agcStatus->digitalGain);
} }
LuxStatus *luxStatus = rpiMetadata_.getLocked<LuxStatus>("lux.status"); LuxStatus *luxStatus = rpiMetadata.getLocked<LuxStatus>("lux.status");
if (luxStatus) if (luxStatus)
libcameraMetadata_.set(controls::Lux, luxStatus->lux); libcameraMetadata_.set(controls::Lux, luxStatus->lux);
AwbStatus *awbStatus = rpiMetadata_.getLocked<AwbStatus>("awb.status"); AwbStatus *awbStatus = rpiMetadata.getLocked<AwbStatus>("awb.status");
if (awbStatus) { if (awbStatus) {
libcameraMetadata_.set(controls::ColourGains, { static_cast<float>(awbStatus->gainR), libcameraMetadata_.set(controls::ColourGains, { static_cast<float>(awbStatus->gainR),
static_cast<float>(awbStatus->gainB) }); static_cast<float>(awbStatus->gainB) });
libcameraMetadata_.set(controls::ColourTemperature, awbStatus->temperatureK); libcameraMetadata_.set(controls::ColourTemperature, awbStatus->temperatureK);
} }
BlackLevelStatus *blackLevelStatus = rpiMetadata_.getLocked<BlackLevelStatus>("black_level.status"); BlackLevelStatus *blackLevelStatus = rpiMetadata.getLocked<BlackLevelStatus>("black_level.status");
if (blackLevelStatus) if (blackLevelStatus)
libcameraMetadata_.set(controls::SensorBlackLevels, libcameraMetadata_.set(controls::SensorBlackLevels,
{ static_cast<int32_t>(blackLevelStatus->blackLevelR), { static_cast<int32_t>(blackLevelStatus->blackLevelR),
@ -582,7 +586,7 @@ void IPARPi::reportMetadata()
static_cast<int32_t>(blackLevelStatus->blackLevelG), static_cast<int32_t>(blackLevelStatus->blackLevelG),
static_cast<int32_t>(blackLevelStatus->blackLevelB) }); static_cast<int32_t>(blackLevelStatus->blackLevelB) });
FocusStatus *focusStatus = rpiMetadata_.getLocked<FocusStatus>("focus.status"); FocusStatus *focusStatus = rpiMetadata.getLocked<FocusStatus>("focus.status");
if (focusStatus && focusStatus->num == 12) { if (focusStatus && focusStatus->num == 12) {
/* /*
* We get a 4x3 grid of regions by default. Calculate the average * We get a 4x3 grid of regions by default. Calculate the average
@ -593,7 +597,7 @@ void IPARPi::reportMetadata()
libcameraMetadata_.set(controls::FocusFoM, focusFoM); libcameraMetadata_.set(controls::FocusFoM, focusFoM);
} }
CcmStatus *ccmStatus = rpiMetadata_.getLocked<CcmStatus>("ccm.status"); CcmStatus *ccmStatus = rpiMetadata.getLocked<CcmStatus>("ccm.status");
if (ccmStatus) { if (ccmStatus) {
float m[9]; float m[9];
for (unsigned int i = 0; i < 9; i++) for (unsigned int i = 0; i < 9; i++)
@ -1008,9 +1012,10 @@ void IPARPi::prepareISP(const ISPConfig &data)
{ {
int64_t frameTimestamp = data.controls.get(controls::SensorTimestamp).value_or(0); int64_t frameTimestamp = data.controls.get(controls::SensorTimestamp).value_or(0);
RPiController::Metadata lastMetadata; RPiController::Metadata lastMetadata;
RPiController::Metadata &rpiMetadata = rpiMetadata_[0];
Span<uint8_t> embeddedBuffer; Span<uint8_t> embeddedBuffer;
lastMetadata = std::move(rpiMetadata_); lastMetadata = std::move(rpiMetadata);
fillDeviceStatus(data.controls); fillDeviceStatus(data.controls);
if (data.embeddedBufferPresent) { if (data.embeddedBufferPresent) {
@ -1027,7 +1032,7 @@ void IPARPi::prepareISP(const ISPConfig &data)
* This may overwrite the DeviceStatus using values from the sensor * This may overwrite the DeviceStatus using values from the sensor
* metadata, and may also do additional custom processing. * metadata, and may also do additional custom processing.
*/ */
helper_->prepare(embeddedBuffer, rpiMetadata_); helper_->prepare(embeddedBuffer, rpiMetadata);
/* Done with embedded data now, return to pipeline handler asap. */ /* Done with embedded data now, return to pipeline handler asap. */
if (data.embeddedBufferPresent) if (data.embeddedBufferPresent)
@ -1043,7 +1048,7 @@ void IPARPi::prepareISP(const ISPConfig &data)
* current frame, or any other bits of metadata that were added * current frame, or any other bits of metadata that were added
* in helper_->Prepare(). * in helper_->Prepare().
*/ */
rpiMetadata_.merge(lastMetadata); rpiMetadata.merge(lastMetadata);
processPending_ = false; processPending_ = false;
return; return;
} }
@ -1053,48 +1058,48 @@ void IPARPi::prepareISP(const ISPConfig &data)
ControlList ctrls(ispCtrls_); ControlList ctrls(ispCtrls_);
controller_.prepare(&rpiMetadata_); controller_.prepare(&rpiMetadata);
/* Lock the metadata buffer to avoid constant locks/unlocks. */ /* Lock the metadata buffer to avoid constant locks/unlocks. */
std::unique_lock<RPiController::Metadata> lock(rpiMetadata_); std::unique_lock<RPiController::Metadata> lock(rpiMetadata);
AwbStatus *awbStatus = rpiMetadata_.getLocked<AwbStatus>("awb.status"); AwbStatus *awbStatus = rpiMetadata.getLocked<AwbStatus>("awb.status");
if (awbStatus) if (awbStatus)
applyAWB(awbStatus, ctrls); applyAWB(awbStatus, ctrls);
CcmStatus *ccmStatus = rpiMetadata_.getLocked<CcmStatus>("ccm.status"); CcmStatus *ccmStatus = rpiMetadata.getLocked<CcmStatus>("ccm.status");
if (ccmStatus) if (ccmStatus)
applyCCM(ccmStatus, ctrls); applyCCM(ccmStatus, ctrls);
AgcStatus *dgStatus = rpiMetadata_.getLocked<AgcStatus>("agc.status"); AgcStatus *dgStatus = rpiMetadata.getLocked<AgcStatus>("agc.status");
if (dgStatus) if (dgStatus)
applyDG(dgStatus, ctrls); applyDG(dgStatus, ctrls);
AlscStatus *lsStatus = rpiMetadata_.getLocked<AlscStatus>("alsc.status"); AlscStatus *lsStatus = rpiMetadata.getLocked<AlscStatus>("alsc.status");
if (lsStatus) if (lsStatus)
applyLS(lsStatus, ctrls); applyLS(lsStatus, ctrls);
ContrastStatus *contrastStatus = rpiMetadata_.getLocked<ContrastStatus>("contrast.status"); ContrastStatus *contrastStatus = rpiMetadata.getLocked<ContrastStatus>("contrast.status");
if (contrastStatus) if (contrastStatus)
applyGamma(contrastStatus, ctrls); applyGamma(contrastStatus, ctrls);
BlackLevelStatus *blackLevelStatus = rpiMetadata_.getLocked<BlackLevelStatus>("black_level.status"); BlackLevelStatus *blackLevelStatus = rpiMetadata.getLocked<BlackLevelStatus>("black_level.status");
if (blackLevelStatus) if (blackLevelStatus)
applyBlackLevel(blackLevelStatus, ctrls); applyBlackLevel(blackLevelStatus, ctrls);
GeqStatus *geqStatus = rpiMetadata_.getLocked<GeqStatus>("geq.status"); GeqStatus *geqStatus = rpiMetadata.getLocked<GeqStatus>("geq.status");
if (geqStatus) if (geqStatus)
applyGEQ(geqStatus, ctrls); applyGEQ(geqStatus, ctrls);
DenoiseStatus *denoiseStatus = rpiMetadata_.getLocked<DenoiseStatus>("denoise.status"); DenoiseStatus *denoiseStatus = rpiMetadata.getLocked<DenoiseStatus>("denoise.status");
if (denoiseStatus) if (denoiseStatus)
applyDenoise(denoiseStatus, ctrls); applyDenoise(denoiseStatus, ctrls);
SharpenStatus *sharpenStatus = rpiMetadata_.getLocked<SharpenStatus>("sharpen.status"); SharpenStatus *sharpenStatus = rpiMetadata.getLocked<SharpenStatus>("sharpen.status");
if (sharpenStatus) if (sharpenStatus)
applySharpen(sharpenStatus, ctrls); applySharpen(sharpenStatus, ctrls);
DpcStatus *dpcStatus = rpiMetadata_.getLocked<DpcStatus>("dpc.status"); DpcStatus *dpcStatus = rpiMetadata.getLocked<DpcStatus>("dpc.status");
if (dpcStatus) if (dpcStatus)
applyDPC(dpcStatus, ctrls); applyDPC(dpcStatus, ctrls);
@ -1118,11 +1123,13 @@ void IPARPi::fillDeviceStatus(const ControlList &sensorControls)
LOG(IPARPI, Debug) << "Metadata - " << deviceStatus; LOG(IPARPI, Debug) << "Metadata - " << deviceStatus;
rpiMetadata_.set("device.status", deviceStatus); rpiMetadata_[0].set("device.status", deviceStatus);
} }
void IPARPi::processStats(unsigned int bufferId) void IPARPi::processStats(unsigned int bufferId)
{ {
RPiController::Metadata &rpiMetadata = rpiMetadata_[0];
auto it = buffers_.find(bufferId); auto it = buffers_.find(bufferId);
if (it == buffers_.end()) { if (it == buffers_.end()) {
LOG(IPARPI, Error) << "Could not find stats buffer!"; LOG(IPARPI, Error) << "Could not find stats buffer!";
@ -1132,11 +1139,11 @@ void IPARPi::processStats(unsigned int bufferId)
Span<uint8_t> mem = it->second.planes()[0]; Span<uint8_t> mem = it->second.planes()[0];
bcm2835_isp_stats *stats = reinterpret_cast<bcm2835_isp_stats *>(mem.data()); bcm2835_isp_stats *stats = reinterpret_cast<bcm2835_isp_stats *>(mem.data());
RPiController::StatisticsPtr statistics = std::make_shared<bcm2835_isp_stats>(*stats); RPiController::StatisticsPtr statistics = std::make_shared<bcm2835_isp_stats>(*stats);
helper_->process(statistics, rpiMetadata_); helper_->process(statistics, rpiMetadata);
controller_.process(statistics, &rpiMetadata_); controller_.process(statistics, &rpiMetadata);
struct AgcStatus agcStatus; struct AgcStatus agcStatus;
if (rpiMetadata_.get("agc.status", agcStatus) == 0) { if (rpiMetadata.get("agc.status", agcStatus) == 0) {
ControlList ctrls(sensorCtrls_); ControlList ctrls(sensorCtrls_);
applyAGC(&agcStatus, ctrls); applyAGC(&agcStatus, ctrls);