diff --git a/.forgejo/workflows/build-alpine.yaml b/.forgejo/workflows/build-alpine.yaml new file mode 100644 index 000000000..9fc5ff5ea --- /dev/null +++ b/.forgejo/workflows/build-alpine.yaml @@ -0,0 +1,58 @@ +name: PostmarketOS Build +run-name: PostmarketOS Build +on: + push: + workflow_dispatch: + +jobs: + prepare: + name: Prepare + runs-on: Pmbootstrap + outputs: + time: ${{ steps.time.outputs.time }} + steps: + - name: Set start Time + id: time + shell: sh + run: echo time=$(date +"%Y%m%d%H%M%S") >> $GITHUB_OUTPUT + - name: Update pmbootstrap + uses: actions/pmbootstrap-update@master + - name: Remove libcamera aport + run: rm -rf ${{env.PMB_PMAPORTS}}/temp/libcamera + + build: + name: Build for ${{ matrix.info.arch }} + runs-on: Pmbootstrap + strategy: + matrix: + info: + - arch: x86_64 + - arch: aarch64 + needs: prepare + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Build packages + id: build + uses: actions/pmbootstrap-build@main + with: + name: libcamera + aports: ${{github.workspace}}/package/alpine + arch: ${{ matrix.info.arch }} + src: ${{github.workspace}} + time: ${{ needs.prepare.outputs.time }} + - name: "Upload packages" + uses: actions/upload-alpine-package@main + with: + files: ${{steps.build.outputs.packages}} + secret: ${{secrets.PACKAGE_TOKEN}} + + clean: + name: "Clean" + runs-on: Pmbootstrap + needs: build + if: always() + continue-on-error: true + steps: + - name: Update pmbootstrap + uses: actions/pmbootstrap-update@master diff --git a/.forgejo/workflows/sync-with-upstream.yaml b/.forgejo/workflows/sync-with-upstream.yaml new file mode 100644 index 000000000..808cbaee8 --- /dev/null +++ b/.forgejo/workflows/sync-with-upstream.yaml @@ -0,0 +1,18 @@ +name: Sync fork with upstream +run-name: Sync fork with upstream +on: + schedule: + - cron: "@daily" + workflow_dispatch: + +jobs: + sync: + name: Sync + runs-on: Misc + steps: + - name: Sync repository with upstream + uses: actions/sync-with-mirror@main + with: + secret: ${{ secrets.PUSH_TOKEN }} + name: libcamera + branch: master diff --git a/.gitignore b/.gitignore index 51d314408..656f622f0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ *.pyc __pycache__/ venv/ +.vscode/ +.cache/ +compile_commands.json diff --git a/include/libcamera/framebuffer.h b/include/libcamera/framebuffer.h index ff8392430..e83825b46 100644 --- a/include/libcamera/framebuffer.h +++ b/include/libcamera/framebuffer.h @@ -26,6 +26,7 @@ struct FrameMetadata { FrameSuccess, FrameError, FrameCancelled, + FrameStartup, }; struct Plane { diff --git a/include/libcamera/internal/clock_recovery.h b/include/libcamera/internal/clock_recovery.h new file mode 100644 index 000000000..43e46b7dc --- /dev/null +++ b/include/libcamera/internal/clock_recovery.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Raspberry Pi Ltd + * + * Camera recovery algorithm + */ +#pragma once + +#include + +namespace libcamera { + +class ClockRecovery +{ +public: + ClockRecovery(); + + void configure(unsigned int numSamples = 100, unsigned int maxJitter = 2000, + unsigned int minSamples = 10, unsigned int errorThreshold = 50000); + void reset(); + + void addSample(); + void addSample(uint64_t input, uint64_t output); + + uint64_t getOutput(uint64_t input); + +private: + /* Approximate number of samples over which the model state persists. */ + unsigned int numSamples_; + /* Remove any output jitter larger than this immediately. */ + unsigned int maxJitter_; + /* Number of samples required before we start to use model estimates. */ + unsigned int minSamples_; + /* Threshold above which we assume the wallclock has been reset. */ + unsigned int errorThreshold_; + + /* How many samples seen (up to numSamples_). */ + unsigned int count_; + /* This gets subtracted from all input values, just to make the numbers easier. */ + uint64_t inputBase_; + /* As above, for the output. */ + uint64_t outputBase_; + /* The previous input sample. */ + uint64_t lastInput_; + /* The previous output sample. */ + uint64_t lastOutput_; + + /* Average x value seen so far. */ + double xAve_; + /* Average y value seen so far */ + double yAve_; + /* Average x^2 value seen so far. */ + double x2Ave_; + /* Average x*y value seen so far. */ + double xyAve_; + + /* + * The latest estimate of linear parameters to derive the output clock + * from the input. + */ + double slope_; + double offset_; + + /* Use this cumulative error to monitor for spontaneous clock updates. */ + double error_; +}; + +} /* namespace libcamera */ diff --git a/include/libcamera/internal/delayed_controls.h b/include/libcamera/internal/delayed_controls.h index e8d3014d9..b64d8bba7 100644 --- a/include/libcamera/internal/delayed_controls.h +++ b/include/libcamera/internal/delayed_controls.h @@ -10,13 +10,15 @@ #include #include +#include + #include namespace libcamera { class V4L2Device; -class DelayedControls +class DelayedControls : public Object { public: struct ControlParams { diff --git a/include/libcamera/internal/meson.build b/include/libcamera/internal/meson.build index 33f318b2b..5c80a28c4 100644 --- a/include/libcamera/internal/meson.build +++ b/include/libcamera/internal/meson.build @@ -11,6 +11,7 @@ libcamera_internal_headers = files([ 'camera_manager.h', 'camera_sensor.h', 'camera_sensor_properties.h', + 'clock_recovery.h', 'control_serializer.h', 'control_validator.h', 'converter.h', diff --git a/include/libcamera/internal/process.h b/include/libcamera/internal/process.h index b1d07a5a5..6c34aef2f 100644 --- a/include/libcamera/internal/process.h +++ b/include/libcamera/internal/process.h @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -42,6 +43,8 @@ public: Signal finished; private: + LIBCAMERA_DISABLE_COPY_AND_MOVE(Process) + void closeAllFdsExcept(const std::vector &fds); int isolate(); void died(int wstatus); diff --git a/include/libcamera/internal/software_isp/debayer_params.h b/include/libcamera/internal/software_isp/debayer_params.h index 217cd5d92..6c36defc9 100644 --- a/include/libcamera/internal/software_isp/debayer_params.h +++ b/include/libcamera/internal/software_isp/debayer_params.h @@ -49,6 +49,15 @@ struct DebayerParams { CcmLookupTable greenCcm; CcmLookupTable blueCcm; LookupTable gammaLut; + + /* + * Statistic controls + * + * Statistic collecting are very slow. We can disable it for some actions like + * video capture or streaming. + * TODO: Add statistic window control + */ + bool collect_stats; }; } /* namespace libcamera */ diff --git a/include/libcamera/internal/software_isp/software_isp.h b/include/libcamera/internal/software_isp/software_isp.h index 786246592..8f3a6b1b1 100644 --- a/include/libcamera/internal/software_isp/software_isp.h +++ b/include/libcamera/internal/software_isp/software_isp.h @@ -86,11 +86,11 @@ public: Signal outputBufferReady; Signal ispStatsReady; Signal metadataReady; - Signal setSensorControls; + Signal setSensorControls; private: void saveIspParams(); - void setSensorCtrls(const ControlList &sensorControls); + void setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls); void statsReady(uint32_t frame, uint32_t bufferId); void inputReady(FrameBuffer *input); void outputReady(FrameBuffer *output); diff --git a/include/libcamera/internal/software_isp/swisp_stats.h b/include/libcamera/internal/software_isp/swisp_stats.h index ae11f112e..3377dd825 100644 --- a/include/libcamera/internal/software_isp/swisp_stats.h +++ b/include/libcamera/internal/software_isp/swisp_stats.h @@ -44,6 +44,10 @@ struct SwIspStats { * \brief A histogram of luminance values */ Histogram yHistogram; + /** + * \brief Holds the sharpness of an image + */ + uint64_t sharpness; }; } /* namespace libcamera */ diff --git a/include/libcamera/ipa/raspberrypi.mojom b/include/libcamera/ipa/raspberrypi.mojom index e30c70bde..12b083e9d 100644 --- a/include/libcamera/ipa/raspberrypi.mojom +++ b/include/libcamera/ipa/raspberrypi.mojom @@ -52,7 +52,8 @@ struct ConfigResult { struct StartResult { libcamera.ControlList controls; - int32 dropFrameCount; + int32 startupFrameCount; + int32 invalidFrameCount; }; struct PrepareParams { diff --git a/include/libcamera/ipa/soft.mojom b/include/libcamera/ipa/soft.mojom index 77328c5fd..e5767532c 100644 --- a/include/libcamera/ipa/soft.mojom +++ b/include/libcamera/ipa/soft.mojom @@ -10,6 +10,7 @@ import "include/libcamera/ipa/core.mojom"; struct IPAConfigInfo { libcamera.ControlInfoMap sensorControls; + libcamera.ControlInfoMap lensControls; }; interface IPASoftInterface { @@ -32,7 +33,7 @@ interface IPASoftInterface { }; interface IPASoftEventInterface { - setSensorControls(libcamera.ControlList sensorControls); + setSensorControls(libcamera.ControlList sensorControls, libcamera.ControlList lensControls); setIspParams(); metadataReady(uint32 frame, libcamera.ControlList metadata); }; diff --git a/package/alpine/APKBUILD b/package/alpine/APKBUILD new file mode 100644 index 000000000..ddfc72950 --- /dev/null +++ b/package/alpine/APKBUILD @@ -0,0 +1,132 @@ +pkgname=libcamera +pkgver=9999999 +pkgrel=0 +pkgdesc="Linux camera framework" +url="https://libcamera.org/" +arch="all" +license="LGPL-2.1-or-later AND GPL-2.0-or-later" +depends_dev=" + eudev-dev + glib-dev + gnutls-dev + gst-plugins-bad-dev + qt6-qtbase-dev + " + +makedepends="$depends_dev + coreutils + doxygen + graphviz + gtest-dev + libevent-dev + libpisp-dev + libunwind-dev + libyuv-dev + linux-headers + meson + py3-jinja2 + py3-ply + py3-sphinx + py3-yaml + qt6-qttools-dev + yaml-dev + " +subpackages=" + $pkgname-dbg + $pkgname-dev + $pkgname-doc + qcam + $pkgname-gstreamer + $pkgname-v4l2 + $pkgname-tools + " +source="" + +builddir="$srcdir/$pkgname-v$_pkgver" +# gstreamer tests fail +# manual strip because ipa .sign files depend on the file contents- have to re-sign after strip +options="!strip !check" + +case "$CARCH" in +arm*|aarch64) + subpackages="$subpackages $pkgname-raspberrypi" + ;; +esac + +case "$CARCH" in +ppc64le|s390x|riscv64|loongarch64) + # doesn't install any ipa + ;; +*) + # WIP: HACK? Don't depend on this this shit + # depends="$pkgname-ipa=$pkgver-r$pkgrel" + subpackages="$subpackages $pkgname-ipa" + ;; +esac + +build() { + abuild-meson \ + -Dtest=false \ + -Dv4l2=true \ + -Dwerror=false \ + . output + meson compile -C output +} + +package() { + DESTDIR="$pkgdir" meson install --no-rebuild -C output + + # manual strip first.. + scanelf --recursive \ + --nobanner \ + --etype "ET_DYN,ET_EXEC" \ + --format "%F" \ + "$pkgdir" \ + | while read -r file; do + strip "$file" + done +} + +ipa() { + depends="" + amove usr/lib/libcamera + # then sign ipa's + local ipa + for ipa in "$subpkgdir"/usr/lib/libcamera/ipa/ipa*.so; do + msg "signing $ipa" + "$builddir"/src/ipa/ipa-sign.sh \ + "$(find "$builddir"/output -type f -iname "*ipa-priv-key.pem")" \ + "$ipa" \ + "$ipa".sign + done +} + +qcam() { + depends="" + amove usr/bin/qcam +} + +gstreamer() { + depends="" + amove usr/lib/gstreamer-1.0 +} + +v4l2() { + depends="" + amove usr/libexec/libcamera/v4l2-compat.so +} + +raspberrypi() { + depends="" + amove usr/share/libcamera/ipa/rpi + amove usr/libexec/libcamera/raspberrypi_ipa_proxy + amove usr/share/libcamera/pipeline/rpi/vc4 +} + +tools() { + depends="" + amove usr/bin/cam + amove usr/bin/lc-compliance +} + +sha512sums="" diff --git a/src/android/camera_device.cpp b/src/android/camera_device.cpp index a038131ae..80ff248c2 100644 --- a/src/android/camera_device.cpp +++ b/src/android/camera_device.cpp @@ -1079,7 +1079,7 @@ int CameraDevice::processCaptureRequest(camera3_capture_request_t *camera3Reques buffer.internalBuffer = frameBuffer; descriptor->request_->addBuffer(sourceStream->stream(), - frameBuffer, nullptr); + frameBuffer); requestedStreams.insert(sourceStream); } diff --git a/src/apps/common/image.cpp b/src/apps/common/image.cpp index a2a0f58f3..9a67238aa 100644 --- a/src/apps/common/image.cpp +++ b/src/apps/common/image.cpp @@ -98,12 +98,12 @@ unsigned int Image::numPlanes() const Span Image::data(unsigned int plane) { - assert(plane <= planes_.size()); + assert(plane < planes_.size()); return planes_[plane]; } Span Image::data(unsigned int plane) const { - assert(plane <= planes_.size()); + assert(plane < planes_.size()); return planes_[plane]; } diff --git a/src/gstreamer/gstlibcamera-controls.cpp.in b/src/gstreamer/gstlibcamera-controls.cpp.in index 89c530da0..2a16b39a9 100644 --- a/src/gstreamer/gstlibcamera-controls.cpp.in +++ b/src/gstreamer/gstlibcamera-controls.cpp.in @@ -68,7 +68,7 @@ static const GEnumValue {{ ctrl.name|snake_case }}_types[] = { "{{ enum.gst_name }}" }, {%- endfor %} - {0, NULL, NULL} + {0, nullptr, nullptr} }; #define TYPE_{{ ctrl.name|snake_case|upper }} \ diff --git a/src/gstreamer/gstlibcamerapad.cpp b/src/gstreamer/gstlibcamerapad.cpp index 3bc2bc87e..22b967198 100644 --- a/src/gstreamer/gstlibcamerapad.cpp +++ b/src/gstreamer/gstlibcamerapad.cpp @@ -72,6 +72,10 @@ gst_libcamera_pad_query(GstPad *pad, GstObject *parent, GstQuery *query) if (query->type != GST_QUERY_LATENCY) return gst_pad_query_default(pad, parent, query); + GLibLocker lock(GST_OBJECT(self)); + if (self->latency == GST_CLOCK_TIME_NONE) + return FALSE; + /* TRUE here means live, we assumes that max latency is the same as min * as we have no idea that duration of frames. */ gst_query_set_latency(query, TRUE, self->latency, self->latency); @@ -81,6 +85,7 @@ gst_libcamera_pad_query(GstPad *pad, GstObject *parent, GstQuery *query) static void gst_libcamera_pad_init(GstLibcameraPad *self) { + self->latency = GST_CLOCK_TIME_NONE; GST_PAD_QUERYFUNC(self) = gst_libcamera_pad_query; } @@ -102,7 +107,7 @@ gst_libcamera_stream_role_get_type() "libcamera::Viewfinder", "view-finder", }, - { 0, NULL, NULL } + { 0, nullptr, nullptr } }; if (!type) diff --git a/src/gstreamer/gstlibcameraprovider.cpp b/src/gstreamer/gstlibcameraprovider.cpp index 5da96ea3f..08862363c 100644 --- a/src/gstreamer/gstlibcameraprovider.cpp +++ b/src/gstreamer/gstlibcameraprovider.cpp @@ -32,7 +32,7 @@ GST_DEBUG_CATEGORY_STATIC(provider_debug); */ enum { - PROP_DEVICE_NAME = 1, + PROP_DEVICE_ = 1, }; #define GST_TYPE_LIBCAMERA_DEVICE gst_libcamera_device_get_type() @@ -76,14 +76,11 @@ gst_libcamera_device_reconfigure_element(GstDevice *device, static void gst_libcamera_device_set_property(GObject *object, guint prop_id, - const GValue *value, GParamSpec *pspec) + [[maybe_unused]]const GValue *value, GParamSpec *pspec) { - GstLibcameraDevice *device = GST_LIBCAMERA_DEVICE(object); + // GstLibcameraDevice *device = GST_LIBCAMERA_DEVICE(object); switch (prop_id) { - case PROP_DEVICE_NAME: - device->name = g_value_dup_string(value); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -117,12 +114,6 @@ gst_libcamera_device_class_init(GstLibcameraDeviceClass *klass) object_class->set_property = gst_libcamera_device_set_property; object_class->finalize = gst_libcamera_device_finalize; - - GParamSpec *pspec = g_param_spec_string("name", "Name", - "The name of the camera device", "", - (GParamFlags)(G_PARAM_STATIC_STRINGS | G_PARAM_WRITABLE | - G_PARAM_CONSTRUCT_ONLY)); - g_object_class_install_property(object_class, PROP_DEVICE_NAME, pspec); } static GstDevice * diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp index b34f08977..da8eb4e5c 100644 --- a/src/gstreamer/gstlibcamerasrc.cpp +++ b/src/gstreamer/gstlibcamerasrc.cpp @@ -29,6 +29,8 @@ #include #include +#include +#include #include #include @@ -234,6 +236,8 @@ GstLibcameraSrcState::requestCompleted(Request *request) GLibLocker locker(&lock_); controls_.readMetadata(request); + if(queuedRequests_.empty()) + return; wrap = std::move(queuedRequests_.front()); queuedRequests_.pop(); @@ -285,10 +289,19 @@ gst_libcamera_extrapolate_info(GstVideoInfo *info, guint32 stride) } static GstFlowReturn -gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, const GstVideoInfo *dest_info, guint32 stride) +gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, + const GstVideoInfo *dest_info, guint32 stride) { - GstVideoInfo src_info = *dest_info; + /* + * When dropping support for versions earlier than v1.22.0, use + * + * g_auto (GstVideoFrame) src_frame = GST_VIDEO_FRAME_INIT; + * g_auto (GstVideoFrame) dest_frame = GST_VIDEO_FRAME_INIT; + * + * and drop the gst_video_frame_unmap() calls. + */ GstVideoFrame src_frame, dest_frame; + GstVideoInfo src_info = *dest_info; gst_libcamera_extrapolate_info(&src_info, stride); src_info.size = gst_buffer_get_size(src); @@ -298,7 +311,12 @@ gst_libcamera_video_frame_copy(GstBuffer *src, GstBuffer *dest, const GstVideoIn return GST_FLOW_ERROR; } - if (!gst_video_frame_map(&dest_frame, const_cast(dest_info), dest, GST_MAP_WRITE)) { + /* + * When dropping support for versions earlier than 1.20.0, drop the + * const_cast<>(). + */ + if (!gst_video_frame_map(&dest_frame, const_cast(dest_info), + dest, GST_MAP_WRITE)) { GST_ERROR("Could not map dest buffer"); gst_video_frame_unmap(&src_frame); return GST_FLOW_ERROR; @@ -352,10 +370,10 @@ int GstLibcameraSrcState::processRequest() if (video_pool) { /* Only set video pool when a copy is needed. */ - GstBuffer *copy = NULL; + GstBuffer *copy = nullptr; const GstVideoInfo info = gst_libcamera_pad_get_video_info(srcpad); - ret = gst_buffer_pool_acquire_buffer(video_pool, ©, NULL); + ret = gst_buffer_pool_acquire_buffer(video_pool, ©, nullptr); if (ret != GST_FLOW_OK) { gst_buffer_unref(buffer); GST_ELEMENT_ERROR(src_, RESOURCE, SETTINGS, @@ -507,6 +525,73 @@ gst_libcamera_src_open(GstLibcameraSrc *self) return true; } +/** + * \brief Create a video pool for a pad + * \param[in] self The libcamerasrc instance + * \param[in] srcpad The pad + * \param[in] caps The pad caps + * \param[in] info The video info for the pad + * + * This function creates and returns a video buffer pool for the given pad if + * needed to accommodate stride mismatch. If the peer element supports stride + * negotiation through the meta API, no pool is needed and the function will + * return a null pool. + * + * \return A tuple containing the video buffers pool pointer and an error code + */ +static std::tuple +gst_libcamera_create_video_pool(GstLibcameraSrc *self, GstPad *srcpad, + GstCaps *caps, const GstVideoInfo *info) +{ + g_autoptr(GstQuery) query = nullptr; + g_autoptr(GstBufferPool) pool = nullptr; + const gboolean need_pool = true; + + /* + * Get the peer allocation hints to check if it supports the meta API. + * If so, the stride will be negotiated, and there's no need to create a + * video pool. + */ + query = gst_query_new_allocation(caps, need_pool); + + if (!gst_pad_peer_query(srcpad, query)) + GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); + else if (gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, nullptr)) + return { nullptr, 0 }; + + GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); + + /* + * If the allocation query has pools, use the first one. Otherwise, + * create a new pool. + */ + if (gst_query_get_n_allocation_pools(query) > 0) + gst_query_parse_nth_allocation_pool(query, 0, &pool, nullptr, + nullptr, nullptr); + + if (!pool) { + GstStructure *config; + guint min_buffers = 3; + + pool = gst_video_buffer_pool_new(); + config = gst_buffer_pool_get_config(pool); + gst_buffer_pool_config_set_params(config, caps, info->size, min_buffers, 0); + + GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); + + gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(pool), config); + } + + if (!gst_buffer_pool_set_active(pool, true)) { + GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, + ("Failed to active buffer pool"), + ("gst_libcamera_src_negotiate() failed.")); + return { nullptr, -EINVAL }; + } + + return { std::exchange(pool, nullptr), 0 }; +} + /* Must be called with stream_lock held. */ static bool gst_libcamera_src_negotiate(GstLibcameraSrc *self) @@ -578,7 +663,7 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self) for (gsize i = 0; i < state->srcpads_.size(); i++) { GstPad *srcpad = state->srcpads_[i]; const StreamConfiguration &stream_cfg = state->config_->at(i); - GstBufferPool *video_pool = NULL; + GstBufferPool *video_pool = nullptr; GstVideoInfo info; g_autoptr(GstCaps) caps = gst_libcamera_stream_configuration_to_caps(stream_cfg, transfer[i]); @@ -589,50 +674,13 @@ gst_libcamera_src_negotiate(GstLibcameraSrc *self) /* Stride mismatch between camera stride and that calculated by video-info. */ if (static_cast(info.stride[0]) != stream_cfg.stride && GST_VIDEO_INFO_FORMAT(&info) != GST_VIDEO_FORMAT_ENCODED) { - GstQuery *query = NULL; - const gboolean need_pool = true; - gboolean has_video_meta = false; - gst_libcamera_extrapolate_info(&info, stream_cfg.stride); - query = gst_query_new_allocation(caps, need_pool); - if (!gst_pad_peer_query(srcpad, query)) - GST_DEBUG_OBJECT(self, "Didn't get downstream ALLOCATION hints"); - else - has_video_meta = gst_query_find_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); - - if (!has_video_meta) { - GstBufferPool *pool = NULL; - - if (gst_query_get_n_allocation_pools(query) > 0) - gst_query_parse_nth_allocation_pool(query, 0, &pool, NULL, NULL, NULL); - - if (pool) - video_pool = pool; - else { - GstStructure *config; - guint min_buffers = 3; - video_pool = gst_video_buffer_pool_new(); - - config = gst_buffer_pool_get_config(video_pool); - gst_buffer_pool_config_set_params(config, caps, info.size, min_buffers, 0); - - GST_DEBUG_OBJECT(self, "Own pool config is %" GST_PTR_FORMAT, config); - - gst_buffer_pool_set_config(GST_BUFFER_POOL_CAST(video_pool), config); - } - - GST_WARNING_OBJECT(self, "Downstream doesn't support video meta, need to copy frame."); - - if (!gst_buffer_pool_set_active(video_pool, true)) { - gst_caps_unref(caps); - GST_ELEMENT_ERROR(self, RESOURCE, SETTINGS, - ("Failed to active buffer pool"), - ("gst_libcamera_src_negotiate() failed.")); - return false; - } - } - gst_query_unref(query); + std::tie(video_pool, ret) = + gst_libcamera_create_video_pool(self, srcpad, + caps, &info); + if (ret) + return false; } GstLibcameraPool *pool = gst_libcamera_pool_new(self->allocator, @@ -835,8 +883,10 @@ gst_libcamera_src_task_leave([[maybe_unused]] GstTask *task, { GLibRecLocker locker(&self->stream_lock); - for (GstPad *srcpad : state->srcpads_) + for (GstPad *srcpad : state->srcpads_) { + gst_libcamera_pad_set_latency(srcpad, GST_CLOCK_TIME_NONE); gst_libcamera_pad_set_pool(srcpad, nullptr); + } } g_clear_object(&self->allocator); @@ -1020,7 +1070,7 @@ gst_libcamera_src_request_new_pad(GstElement *element, GstPadTemplate *templ, const gchar *name, [[maybe_unused]] const GstCaps *caps) { GstLibcameraSrc *self = GST_LIBCAMERA_SRC(element); - g_autoptr(GstPad) pad = NULL; + g_autoptr(GstPad) pad = nullptr; GST_DEBUG_OBJECT(self, "new request pad created"); @@ -1034,12 +1084,12 @@ gst_libcamera_src_request_new_pad(GstElement *element, GstPadTemplate *templ, GST_ELEMENT_ERROR(element, STREAM, FAILED, ("Internal data stream error."), ("Could not add pad to element")); - return NULL; + return nullptr; } gst_child_proxy_child_added(GST_CHILD_PROXY(self), G_OBJECT(pad), GST_OBJECT_NAME(pad)); - return reinterpret_cast(g_steal_pointer(&pad)); + return std::exchange(pad, nullptr); } static void diff --git a/src/ipa/rkisp1/algorithms/filter.cpp b/src/ipa/rkisp1/algorithms/filter.cpp index 7598ef8a9..8ad798017 100644 --- a/src/ipa/rkisp1/algorithms/filter.cpp +++ b/src/ipa/rkisp1/algorithms/filter.cpp @@ -39,6 +39,17 @@ LOG_DEFINE_CATEGORY(RkISP1Filter) static constexpr uint32_t kFiltLumWeightDefault = 0x00022040; static constexpr uint32_t kFiltModeDefault = 0x000004f2; +/** + * \copydoc libcamera::ipa::Algorithm::init + */ +int Filter::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + auto &cmap = context.ctrlMap; + cmap[&controls::Sharpness] = ControlInfo(0.0f, 10.0f, 1.0f); + + return 0; +} /** * \copydoc libcamera::ipa::Algorithm::queueRequest */ diff --git a/src/ipa/rkisp1/algorithms/filter.h b/src/ipa/rkisp1/algorithms/filter.h index 8f858e574..37d8938d3 100644 --- a/src/ipa/rkisp1/algorithms/filter.h +++ b/src/ipa/rkisp1/algorithms/filter.h @@ -21,6 +21,7 @@ public: Filter() = default; ~Filter() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; void queueRequest(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const ControlList &controls) override; diff --git a/src/ipa/rkisp1/rkisp1.cpp b/src/ipa/rkisp1/rkisp1.cpp index 1ed7d7d92..cf66d5553 100644 --- a/src/ipa/rkisp1/rkisp1.cpp +++ b/src/ipa/rkisp1/rkisp1.cpp @@ -116,7 +116,6 @@ const IPAHwSettings ipaHwSettingsV12{ /* List of controls handled by the RkISP1 IPA */ const ControlInfoMap::Map rkisp1Controls{ { &controls::DebugMetadataEnable, ControlInfo(false, true, false) }, - { &controls::Sharpness, ControlInfo(0.0f, 10.0f, 1.0f) }, { &controls::draft::NoiseReductionMode, ControlInfo(controls::draft::NoiseReductionModeValues) }, }; diff --git a/src/ipa/rpi/common/ipa_base.cpp b/src/ipa/rpi/common/ipa_base.cpp index e0f8b7e78..a5bdcbb58 100644 --- a/src/ipa/rpi/common/ipa_base.cpp +++ b/src/ipa/rpi/common/ipa_base.cpp @@ -58,23 +58,24 @@ const ControlInfoMap::Map ipaControls{ /* \todo Move this to the Camera class */ { &controls::AeEnable, ControlInfo(false, true, true) }, { &controls::ExposureTimeMode, - ControlInfo(static_cast(controls::ExposureTimeModeAuto), - static_cast(controls::ExposureTimeModeManual), - static_cast(controls::ExposureTimeModeAuto)) }, + ControlInfo({ { ControlValue(controls::ExposureTimeModeAuto), + ControlValue(controls::ExposureTimeModeManual) } }, + ControlValue(controls::ExposureTimeModeAuto)) }, { &controls::ExposureTime, ControlInfo(1, 66666, static_cast(defaultExposureTime.get())) }, { &controls::AnalogueGainMode, - ControlInfo(static_cast(controls::AnalogueGainModeAuto), - static_cast(controls::AnalogueGainModeManual), - static_cast(controls::AnalogueGainModeAuto)) }, + ControlInfo({ { ControlValue(controls::AnalogueGainModeAuto), + ControlValue(controls::AnalogueGainModeManual) } }, + ControlValue(controls::AnalogueGainModeAuto)) }, { &controls::AnalogueGain, ControlInfo(1.0f, 16.0f, 1.0f) }, { &controls::AeMeteringMode, ControlInfo(controls::AeMeteringModeValues) }, { &controls::AeConstraintMode, ControlInfo(controls::AeConstraintModeValues) }, { &controls::AeExposureMode, ControlInfo(controls::AeExposureModeValues) }, { &controls::ExposureValue, ControlInfo(-8.0f, 8.0f, 0.0f) }, - { &controls::AeFlickerMode, ControlInfo(static_cast(controls::FlickerOff), - static_cast(controls::FlickerManual), - static_cast(controls::FlickerOff)) }, + { &controls::AeFlickerMode, + ControlInfo({ { ControlValue(controls::FlickerOff), + ControlValue(controls::FlickerManual) } }, + ControlValue(controls::FlickerOff)) }, { &controls::AeFlickerPeriod, ControlInfo(100, 1000000) }, { &controls::Brightness, ControlInfo(-1.0f, 1.0f, 0.0f) }, { &controls::Contrast, ControlInfo(0.0f, 32.0f, 1.0f) }, @@ -232,25 +233,6 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa agcStatus.analogueGain = defaultAnalogueGain; applyAGC(&agcStatus, ctrls); - /* - * Set the lens to the default (typically hyperfocal) position - * on first start. - */ - if (lensPresent_) { - RPiController::AfAlgorithm *af = - dynamic_cast(controller_.getAlgorithm("af")); - - if (af) { - float defaultPos = - ipaAfControls.at(&controls::LensPosition).def().get(); - ControlList lensCtrl(lensCtrls_); - int32_t hwpos; - - af->setLensPosition(defaultPos, &hwpos); - lensCtrl.set(V4L2_CID_FOCUS_ABSOLUTE, hwpos); - result->lensControls = std::move(lensCtrl); - } - } } result->sensorControls = std::move(ctrls); @@ -280,8 +262,20 @@ int32_t IpaBase::configure(const IPACameraSensorInfo &sensorInfo, const ConfigPa ctrlMap.merge(ControlInfoMap::Map(ipaColourControls)); /* Declare Autofocus controls, only if we have a controllable lens */ - if (lensPresent_) + if (lensPresent_) { ctrlMap.merge(ControlInfoMap::Map(ipaAfControls)); + RPiController::AfAlgorithm *af = + dynamic_cast(controller_.getAlgorithm("af")); + if (af) { + double min, max, dflt; + af->getLensLimits(min, max); + dflt = af->getDefaultLensPosition(); + ctrlMap[&controls::LensPosition] = + ControlInfo(static_cast(min), + static_cast(max), + static_cast(dflt)); + } + } result->controlInfo = ControlInfoMap(std::move(ctrlMap), controls::controls); @@ -319,14 +313,35 @@ void IpaBase::start(const ControlList &controls, StartResult *result) /* Make a note of this as it tells us the HDR status of the first few frames. */ hdrStatus_ = agcStatus.hdr; + /* + * AF: If no lens position was specified, drive lens to a default position. + * This had to be deferred (not initialised by a constructor) until here + * to ensure that exactly ONE starting position is sent to the lens driver. + * It should be the static API default, not dependent on AF range or mode. + */ + if (firstStart_ && lensPresent_) { + RPiController::AfAlgorithm *af = dynamic_cast( + controller_.getAlgorithm("af")); + if (af && !af->getLensPosition()) { + int32_t hwpos; + double pos = af->getDefaultLensPosition(); + if (af->setLensPosition(pos, &hwpos, true)) { + ControlList lensCtrls(lensCtrls_); + lensCtrls.set(V4L2_CID_FOCUS_ABSOLUTE, hwpos); + setLensControls.emit(lensCtrls); + } + } + } + /* * Initialise frame counts, and decide how many frames must be hidden or * "mistrusted", which depends on whether this is a startup from cold, * or merely a mode switch in a running system. */ + unsigned int agcConvergenceFrames = 0, awbConvergenceFrames = 0; frameCount_ = 0; if (firstStart_) { - dropFrameCount_ = helper_->hideFramesStartup(); + invalidCount_ = helper_->hideFramesStartup(); mistrustCount_ = helper_->mistrustFramesStartup(); /* @@ -336,7 +351,6 @@ void IpaBase::start(const ControlList &controls, StartResult *result) * (mistrustCount_) that they won't see. But if zero (i.e. * no convergence necessary), no frames need to be dropped. */ - unsigned int agcConvergenceFrames = 0; RPiController::AgcAlgorithm *agc = dynamic_cast( controller_.getAlgorithm("agc")); if (agc) { @@ -345,7 +359,6 @@ void IpaBase::start(const ControlList &controls, StartResult *result) agcConvergenceFrames += mistrustCount_; } - unsigned int awbConvergenceFrames = 0; RPiController::AwbAlgorithm *awb = dynamic_cast( controller_.getAlgorithm("awb")); if (awb) { @@ -353,15 +366,18 @@ void IpaBase::start(const ControlList &controls, StartResult *result) if (awbConvergenceFrames) awbConvergenceFrames += mistrustCount_; } - - dropFrameCount_ = std::max({ dropFrameCount_, agcConvergenceFrames, awbConvergenceFrames }); - LOG(IPARPI, Debug) << "Drop " << dropFrameCount_ << " frames on startup"; } else { - dropFrameCount_ = helper_->hideFramesModeSwitch(); + invalidCount_ = helper_->hideFramesModeSwitch(); mistrustCount_ = helper_->mistrustFramesModeSwitch(); } - result->dropFrameCount = dropFrameCount_; + result->startupFrameCount = std::max({ agcConvergenceFrames, awbConvergenceFrames }); + result->invalidFrameCount = invalidCount_; + + invalidCount_ = std::max({ invalidCount_, agcConvergenceFrames, awbConvergenceFrames }); + + LOG(IPARPI, Debug) << "Startup frames: " << result->startupFrameCount + << " Invalid frames: " << result->invalidFrameCount; firstStart_ = false; lastRunTimestamp_ = 0; @@ -441,7 +457,7 @@ void IpaBase::prepareIsp(const PrepareParams ¶ms) /* Allow a 10% margin on the comparison below. */ Duration delta = (frameTimestamp - lastRunTimestamp_) * 1.0ns; - if (lastRunTimestamp_ && frameCount_ > dropFrameCount_ && + if (lastRunTimestamp_ && frameCount_ > invalidCount_ && delta < controllerMinFrameDuration * 0.9 && !hdrChange) { /* * Ensure we merge the previous frame's metadata with the current diff --git a/src/ipa/rpi/common/ipa_base.h b/src/ipa/rpi/common/ipa_base.h index 1a811beb3..e818104ba 100644 --- a/src/ipa/rpi/common/ipa_base.h +++ b/src/ipa/rpi/common/ipa_base.h @@ -115,8 +115,8 @@ private: /* How many frames we should avoid running control algos on. */ unsigned int mistrustCount_; - /* Number of frames that need to be dropped on startup. */ - unsigned int dropFrameCount_; + /* Number of frames that need to be marked as dropped on startup. */ + unsigned int invalidCount_; /* Frame timestamp for the last run of the controller. */ uint64_t lastRunTimestamp_; diff --git a/src/ipa/rpi/controller/af_algorithm.h b/src/ipa/rpi/controller/af_algorithm.h index ad9b57545..382609f9b 100644 --- a/src/ipa/rpi/controller/af_algorithm.h +++ b/src/ipa/rpi/controller/af_algorithm.h @@ -33,6 +33,10 @@ public: * * getMode() is provided mainly for validating controls. * getLensPosition() is provided for populating DeviceStatus. + * + * getDefaultlensPosition() and getLensLimits() were added for + * populating ControlInfoMap. They return the static API limits + * which should be independent of the current range or mode. */ enum AfRange { AfRangeNormal = 0, @@ -66,7 +70,9 @@ public: } virtual void setMode(AfMode mode) = 0; virtual AfMode getMode() const = 0; - virtual bool setLensPosition(double dioptres, int32_t *hwpos) = 0; + virtual double getDefaultLensPosition() const = 0; + virtual void getLensLimits(double &min, double &max) const = 0; + virtual bool setLensPosition(double dioptres, int32_t *hwpos, bool force = false) = 0; virtual std::optional getLensPosition() const = 0; virtual void triggerScan() = 0; virtual void cancelScan() = 0; diff --git a/src/ipa/rpi/controller/rpi/af.cpp b/src/ipa/rpi/controller/rpi/af.cpp index 2157eb94f..26e599303 100644 --- a/src/ipa/rpi/controller/rpi/af.cpp +++ b/src/ipa/rpi/controller/rpi/af.cpp @@ -46,6 +46,8 @@ Af::SpeedDependentParams::SpeedDependentParams() : stepCoarse(1.0), stepFine(0.25), contrastRatio(0.75), + retriggerRatio(0.75), + retriggerDelay(10), pdafGain(-0.02), pdafSquelch(0.125), maxSlew(2.0), @@ -60,6 +62,7 @@ Af::CfgParams::CfgParams() confThresh(16), confClip(512), skipFrames(5), + checkForIR(false), map() { } @@ -87,6 +90,8 @@ void Af::SpeedDependentParams::read(const libcamera::YamlObject ¶ms) readNumber(stepCoarse, params, "step_coarse"); readNumber(stepFine, params, "step_fine"); readNumber(contrastRatio, params, "contrast_ratio"); + readNumber(retriggerRatio, params, "retrigger_ratio"); + readNumber(retriggerDelay, params, "retrigger_delay"); readNumber(pdafGain, params, "pdaf_gain"); readNumber(pdafSquelch, params, "pdaf_squelch"); readNumber(maxSlew, params, "max_slew"); @@ -137,6 +142,7 @@ int Af::CfgParams::read(const libcamera::YamlObject ¶ms) readNumber(confThresh, params, "conf_thresh"); readNumber(confClip, params, "conf_clip"); readNumber(skipFrames, params, "skip_frames"); + readNumber(checkForIR, params, "check_for_ir"); if (params.contains("map")) map = params["map"].get(ipa::Pwl{}); @@ -176,27 +182,38 @@ Af::Af(Controller *controller) useWindows_(false), phaseWeights_(), contrastWeights_(), + awbWeights_(), scanState_(ScanState::Idle), initted_(false), + irFlag_(false), ftarget_(-1.0), fsmooth_(-1.0), prevContrast_(0.0), + oldSceneContrast_(0.0), + prevAverage_{ 0.0, 0.0, 0.0 }, + oldSceneAverage_{ 0.0, 0.0, 0.0 }, + prevPhase_(0.0), skipCount_(0), stepCount_(0), dropCount_(0), + sameSignCount_(0), + sceneChangeCount_(0), scanMaxContrast_(0.0), scanMinContrast_(1.0e9), + scanStep_(0.0), scanData_(), reportState_(AfState::Idle) { /* - * Reserve space for data, to reduce memory fragmentation. It's too early - * to query the size of the PDAF (from camera) and Contrast (from ISP) - * statistics, but these are plausible upper bounds. + * Reserve space for data structures, to reduce memory fragmentation. + * It's too early to query the size of the PDAF sensor data, so guess. */ + windows_.reserve(1); phaseWeights_.w.reserve(16 * 12); contrastWeights_.w.reserve(getHardwareConfig().focusRegions.width * getHardwareConfig().focusRegions.height); + contrastWeights_.w.reserve(getHardwareConfig().awbRegions.width * + getHardwareConfig().awbRegions.height); scanData_.reserve(32); } @@ -235,13 +252,14 @@ void Af::switchMode(CameraMode const &cameraMode, [[maybe_unused]] Metadata *met << statsRegion_.height; invalidateWeights(); - if (scanState_ >= ScanState::Coarse && scanState_ < ScanState::Settle) { + if (scanState_ >= ScanState::Coarse1 && scanState_ < ScanState::Settle) { /* * If a scan was in progress, re-start it, as CDAF statistics * may have changed. Though if the application is just about * to take a still picture, this will not help... */ startProgrammedScan(); + updateLensPosition(); } skipCount_ = cfg_.skipFrames; } @@ -307,6 +325,7 @@ void Af::invalidateWeights() { phaseWeights_.sum = 0; contrastWeights_.sum = 0; + awbWeights_.sum = 0; } bool Af::getPhase(PdafRegions const ®ions, double &phase, double &conf) @@ -328,9 +347,8 @@ bool Af::getPhase(PdafRegions const ®ions, double &phase, double &conf) if (c >= cfg_.confThresh) { if (c > cfg_.confClip) c = cfg_.confClip; - c -= (cfg_.confThresh >> 2); + c -= (cfg_.confThresh >> 1); sumWc += w * c; - c -= (cfg_.confThresh >> 2); sumWcp += (int64_t)(w * c) * (int64_t)data.phase; } } @@ -364,6 +382,54 @@ double Af::getContrast(const FocusRegions &focusStats) return (contrastWeights_.sum > 0) ? ((double)sumWc / (double)contrastWeights_.sum) : 0.0; } +/* + * Get the average R, G, B values in AF window[s] (from AWB statistics). + * Optionally, check if all of {R,G,B} are within 4:5 of each other + * across more than 50% of the counted area and within the AF window: + * for an RGB sensor this strongly suggests that IR lighting is in use. + */ + +bool Af::getAverageAndTestIr(const RgbyRegions &awbStats, double rgb[3]) +{ + libcamera::Size size = awbStats.size(); + if (size.height != awbWeights_.rows || + size.width != awbWeights_.cols || awbWeights_.sum == 0) { + LOG(RPiAf, Debug) << "Recompute RGB weights " << size.width << 'x' << size.height; + computeWeights(&awbWeights_, size.height, size.width); + } + + uint64_t sr = 0, sg = 0, sb = 0, sw = 1; + uint64_t greyCount = 0, allCount = 0; + for (unsigned i = 0; i < awbStats.numRegions(); ++i) { + uint64_t r = awbStats.get(i).val.rSum; + uint64_t g = awbStats.get(i).val.gSum; + uint64_t b = awbStats.get(i).val.bSum; + uint64_t w = awbWeights_.w[i]; + if (w) { + sw += w; + sr += w * r; + sg += w * g; + sb += w * b; + } + if (cfg_.checkForIR) { + if (4 * r < 5 * b && 4 * b < 5 * r && + 4 * r < 5 * g && 4 * g < 5 * r && + 4 * b < 5 * g && 4 * g < 5 * b) + greyCount += awbStats.get(i).counted; + allCount += awbStats.get(i).counted; + } + } + + rgb[0] = sr / (double)sw; + rgb[1] = sg / (double)sw; + rgb[2] = sb / (double)sw; + + return (cfg_.checkForIR && 2 * greyCount > allCount && + 4 * sr < 5 * sb && 4 * sb < 5 * sr && + 4 * sr < 5 * sg && 4 * sg < 5 * sr && + 4 * sb < 5 * sg && 4 * sg < 5 * sb); +} + void Af::doPDAF(double phase, double conf) { /* Apply loop gain */ @@ -410,7 +476,7 @@ void Af::doPDAF(double phase, double conf) bool Af::earlyTerminationByPhase(double phase) { if (scanData_.size() > 0 && - scanData_[scanData_.size() - 1].conf >= cfg_.confEpsilon) { + scanData_[scanData_.size() - 1].conf >= cfg_.confThresh) { double oldFocus = scanData_[scanData_.size() - 1].focus; double oldPhase = scanData_[scanData_.size() - 1].phase; @@ -419,11 +485,12 @@ bool Af::earlyTerminationByPhase(double phase) * Interpolate/extrapolate the lens position for zero phase. * Check that the extrapolation is well-conditioned. */ - if ((ftarget_ - oldFocus) * (phase - oldPhase) > 0.0) { + if ((ftarget_ - oldFocus) * (phase - oldPhase) * cfg_.speeds[speed_].pdafGain < 0.0) { double param = phase / (phase - oldPhase); - if (-3.0 <= param && param <= 3.5) { - ftarget_ += param * (oldFocus - ftarget_); + if ((-2.5 <= param || mode_ == AfModeContinuous) && param <= 3.0) { LOG(RPiAf, Debug) << "ETBP: param=" << param; + param = std::max(param, -2.5); + ftarget_ += param * (oldFocus - ftarget_); return true; } } @@ -436,15 +503,28 @@ double Af::findPeak(unsigned i) const { double f = scanData_[i].focus; - if (i > 0 && i + 1 < scanData_.size()) { - double dropLo = scanData_[i].contrast - scanData_[i - 1].contrast; - double dropHi = scanData_[i].contrast - scanData_[i + 1].contrast; - if (0.0 <= dropLo && dropLo < dropHi) { - double param = 0.3125 * (1.0 - dropLo / dropHi) * (1.6 - dropLo / dropHi); - f += param * (scanData_[i - 1].focus - f); - } else if (0.0 <= dropHi && dropHi < dropLo) { - double param = 0.3125 * (1.0 - dropHi / dropLo) * (1.6 - dropHi / dropLo); - f += param * (scanData_[i + 1].focus - f); + if (scanData_.size() >= 3) { + /* + * Given the sample with the highest contrast score and its two + * neighbours either side (or same side if at the end of a scan), + * solve for the best lens position by fitting a parabola. + * Adapted from awb.cpp: interpolateQaudaratic() + */ + + if (i == 0) + i++; + else if (i + 1 >= scanData_.size()) + i--; + + double abx = scanData_[i - 1].focus - scanData_[i].focus; + double aby = scanData_[i - 1].contrast - scanData_[i].contrast; + double cbx = scanData_[i + 1].focus - scanData_[i].focus; + double cby = scanData_[i + 1].contrast - scanData_[i].contrast; + double denom = 2.0 * (aby * cbx - cby * abx); + if (std::abs(denom) >= (1.0 / 64.0) && denom * abx > 0.0) { + f = (aby * cbx * cbx - cby * abx * abx) / denom; + f = std::clamp(f, std::min(abx, cbx), std::max(abx, cbx)); + f += scanData_[i].focus; } } @@ -458,36 +538,49 @@ void Af::doScan(double contrast, double phase, double conf) if (scanData_.empty() || contrast > scanMaxContrast_) { scanMaxContrast_ = contrast; scanMaxIndex_ = scanData_.size(); + if (scanState_ != ScanState::Fine) + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); } if (contrast < scanMinContrast_) scanMinContrast_ = contrast; scanData_.emplace_back(ScanRecord{ ftarget_, contrast, phase, conf }); - if (scanState_ == ScanState::Coarse) { - if (ftarget_ >= cfg_.ranges[range_].focusMax || - contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { - /* - * Finished course scan, or termination based on contrast. - * Jump to just after max contrast and start fine scan. - */ - ftarget_ = std::min(ftarget_, findPeak(scanMaxIndex_) + - 2.0 * cfg_.speeds[speed_].stepFine); - scanState_ = ScanState::Fine; - scanData_.clear(); - } else - ftarget_ += cfg_.speeds[speed_].stepCoarse; - } else { /* ScanState::Fine */ - if (ftarget_ <= cfg_.ranges[range_].focusMin || scanData_.size() >= 5 || - contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { - /* - * Finished fine scan, or termination based on contrast. - * Use quadratic peak-finding to find best contrast position. - */ - ftarget_ = findPeak(scanMaxIndex_); + if ((scanStep_ >= 0.0 && ftarget_ >= cfg_.ranges[range_].focusMax) || + (scanStep_ <= 0.0 && ftarget_ <= cfg_.ranges[range_].focusMin) || + (scanState_ == ScanState::Fine && scanData_.size() >= 3) || + contrast < cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + double pk = findPeak(scanMaxIndex_); + /* + * Finished a scan, by hitting a limit or due to constrast dropping off. + * If this is a first coarse scan and we didn't bracket the peak, reverse! + * If this is a fine scan, or no fine step was defined, we've finished. + * Otherwise, start fine scan in opposite direction. + */ + if (scanState_ == ScanState::Coarse1 && + scanData_[0].contrast >= cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + scanStep_ = -scanStep_; + scanState_ = ScanState::Coarse2; + } else if (scanState_ == ScanState::Fine || cfg_.speeds[speed_].stepFine <= 0.0) { + ftarget_ = pk; scanState_ = ScanState::Settle; - } else - ftarget_ -= cfg_.speeds[speed_].stepFine; - } + } else if (scanState_ == ScanState::Coarse1 && + scanData_[0].contrast >= cfg_.speeds[speed_].contrastRatio * scanMaxContrast_) { + scanStep_ = -scanStep_; + scanState_ = ScanState::Coarse2; + } else if (scanStep_ >= 0.0) { + ftarget_ = std::min(pk + cfg_.speeds[speed_].stepFine, + cfg_.ranges[range_].focusMax); + scanStep_ = -cfg_.speeds[speed_].stepFine; + scanState_ = ScanState::Fine; + } else { + ftarget_ = std::max(pk - cfg_.speeds[speed_].stepFine, + cfg_.ranges[range_].focusMin); + scanStep_ = cfg_.speeds[speed_].stepFine; + scanState_ = ScanState::Fine; + } + scanData_.clear(); + } else + ftarget_ += scanStep_; stepCount_ = (ftarget_ == fsmooth_) ? 0 : cfg_.speeds[speed_].stepFrames; } @@ -501,26 +594,70 @@ void Af::doAF(double contrast, double phase, double conf) return; } + /* Count frames for which PDAF phase has had same sign */ + if (phase * prevPhase_ <= 0.0) + sameSignCount_ = 0; + else + sameSignCount_++; + prevPhase_ = phase; + + if (mode_ == AfModeManual) + return; /* nothing to do */ + if (scanState_ == ScanState::Pdaf) { /* * Use PDAF closed-loop control whenever available, in both CAF * mode and (for a limited number of iterations) when triggered. - * If PDAF fails (due to poor contrast, noise or large defocus), - * fall back to a CDAF-based scan. To avoid "nuisance" scans, - * scan only after a number of frames with low PDAF confidence. + * If PDAF fails (due to poor contrast, noise or large defocus) + * for at least dropoutFrames, fall back to a CDAF-based scan + * immediately (in triggered-auto) or on scene change (in CAF). */ - if (conf > (dropCount_ ? 1.0 : 0.25) * cfg_.confEpsilon) { - doPDAF(phase, conf); + if (conf >= cfg_.confEpsilon) { + if (mode_ == AfModeAuto || sameSignCount_ >= 3) + doPDAF(phase, conf); if (stepCount_ > 0) stepCount_--; else if (mode_ != AfModeContinuous) scanState_ = ScanState::Idle; + oldSceneContrast_ = contrast; + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); + sceneChangeCount_ = 0; dropCount_ = 0; - } else if (++dropCount_ == cfg_.speeds[speed_].dropoutFrames) - startProgrammedScan(); - } else if (scanState_ >= ScanState::Coarse && fsmooth_ == ftarget_) { + return; + } else { + dropCount_++; + if (dropCount_ < cfg_.speeds[speed_].dropoutFrames) + return; + if (mode_ != AfModeContinuous) { + startProgrammedScan(); + return; + } + /* else fall through to waiting for a scene change */ + } + } + if (scanState_ < ScanState::Coarse1 && mode_ == AfModeContinuous) { /* - * Scanning sequence. This means PDAF has become unavailable. + * In CAF mode, not in a scan, and PDAF is unavailable. + * Wait for a scene change, followed by stability. + */ + if (contrast + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneContrast_ || + oldSceneContrast_ + 1.0 < cfg_.speeds[speed_].retriggerRatio * contrast || + prevAverage_[0] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[0] || + oldSceneAverage_[0] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[0] || + prevAverage_[1] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[1] || + oldSceneAverage_[1] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[1] || + prevAverage_[2] + 1.0 < cfg_.speeds[speed_].retriggerRatio * oldSceneAverage_[2] || + oldSceneAverage_[2] + 1.0 < cfg_.speeds[speed_].retriggerRatio * prevAverage_[2]) { + oldSceneContrast_ = contrast; + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); + sceneChangeCount_ = 1; + } else if (sceneChangeCount_) + sceneChangeCount_++; + if (sceneChangeCount_ >= cfg_.speeds[speed_].retriggerDelay) + startProgrammedScan(); + } else if (scanState_ >= ScanState::Coarse1 && fsmooth_ == ftarget_) { + /* + * CDAF-based scanning sequence. * Allow a delay between steps for CDAF FoM statistics to be * updated, and a "settling time" at the end of the sequence. * [A coarse or fine scan can be abandoned if two PDAF samples @@ -539,11 +676,14 @@ void Af::doAF(double contrast, double phase, double conf) scanState_ = ScanState::Pdaf; else scanState_ = ScanState::Idle; + dropCount_ = 0; + sceneChangeCount_ = 0; + oldSceneContrast_ = std::max(scanMaxContrast_, prevContrast_); scanData_.clear(); - } else if (conf >= cfg_.confEpsilon && earlyTerminationByPhase(phase)) { + } else if (conf >= cfg_.confThresh && earlyTerminationByPhase(phase)) { + std::copy(prevAverage_, prevAverage_ + 3, oldSceneAverage_); scanState_ = ScanState::Settle; - stepCount_ = (mode_ == AfModeContinuous) ? 0 - : cfg_.speeds[speed_].stepFrames; + stepCount_ = (mode_ == AfModeContinuous) ? 0 : cfg_.speeds[speed_].stepFrames; } else doScan(contrast, phase, conf); } @@ -573,7 +713,8 @@ void Af::updateLensPosition() void Af::startAF() { /* Use PDAF if the tuning file allows it; else CDAF. */ - if (cfg_.speeds[speed_].dropoutFrames > 0 && + if (cfg_.speeds[speed_].pdafGain != 0.0 && + cfg_.speeds[speed_].dropoutFrames > 0 && (mode_ == AfModeContinuous || cfg_.speeds[speed_].pdafFrames > 0)) { if (!initted_) { ftarget_ = cfg_.ranges[range_].focusDefault; @@ -583,16 +724,30 @@ void Af::startAF() scanState_ = ScanState::Pdaf; scanData_.clear(); dropCount_ = 0; + oldSceneContrast_ = 0.0; + sceneChangeCount_ = 0; reportState_ = AfState::Scanning; - } else + } else { startProgrammedScan(); + updateLensPosition(); + } } void Af::startProgrammedScan() { - ftarget_ = cfg_.ranges[range_].focusMin; - updateLensPosition(); - scanState_ = ScanState::Coarse; + if (!initted_ || mode_ != AfModeContinuous || + fsmooth_ <= cfg_.ranges[range_].focusMin + 2.0 * cfg_.speeds[speed_].stepCoarse) { + ftarget_ = cfg_.ranges[range_].focusMin; + scanStep_ = cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse2; + } else if (fsmooth_ >= cfg_.ranges[range_].focusMax - 2.0 * cfg_.speeds[speed_].stepCoarse) { + ftarget_ = cfg_.ranges[range_].focusMax; + scanStep_ = -cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse2; + } else { + scanStep_ = -cfg_.speeds[speed_].stepCoarse; + scanState_ = ScanState::Coarse1; + } scanMaxContrast_ = 0.0; scanMinContrast_ = 1.0e9; scanMaxIndex_ = 0; @@ -633,7 +788,7 @@ void Af::prepare(Metadata *imageMetadata) uint32_t oldSt = stepCount_; if (imageMetadata->get("pdaf.regions", regions) == 0) getPhase(regions, phase, conf); - doAF(prevContrast_, phase, conf); + doAF(prevContrast_, phase, irFlag_ ? 0 : conf); updateLensPosition(); LOG(RPiAf, Debug) << std::fixed << std::setprecision(2) << static_cast(reportState_) @@ -643,7 +798,8 @@ void Af::prepare(Metadata *imageMetadata) << " ft" << oldFt << "->" << ftarget_ << " fs" << oldFs << "->" << fsmooth_ << " cont=" << (int)prevContrast_ - << " phase=" << (int)phase << " conf=" << (int)conf; + << " phase=" << (int)phase << " conf=" << (int)conf + << (irFlag_ ? " IR" : ""); } /* Report status and produce new lens setting */ @@ -656,6 +812,8 @@ void Af::prepare(Metadata *imageMetadata) if (mode_ == AfModeAuto && scanState_ != ScanState::Idle) status.state = AfState::Scanning; + else if (mode_ == AfModeManual) + status.state = AfState::Idle; else status.state = reportState_; status.lensSetting = initted_ ? std::optional(cfg_.map.eval(fsmooth_)) @@ -667,6 +825,7 @@ void Af::process(StatisticsPtr &stats, [[maybe_unused]] Metadata *imageMetadata) { (void)imageMetadata; prevContrast_ = getContrast(stats->focusRegions); + irFlag_ = getAverageAndTestIr(stats->awbRegions, prevAverage_); } /* Controls */ @@ -715,11 +874,23 @@ void Af::setWindows(libcamera::Span const &wins) invalidateWeights(); } -bool Af::setLensPosition(double dioptres, int *hwpos) +double Af::getDefaultLensPosition() const +{ + return cfg_.ranges[AfRangeNormal].focusDefault; +} + +void Af::getLensLimits(double &min, double &max) const +{ + /* Limits for manual focus are set by map, not by ranges */ + min = cfg_.map.domain().start; + max = cfg_.map.domain().end; +} + +bool Af::setLensPosition(double dioptres, int *hwpos, bool force) { bool changed = false; - if (mode_ == AfModeManual) { + if (mode_ == AfModeManual || force) { LOG(RPiAf, Debug) << "setLensPosition: " << dioptres; ftarget_ = cfg_.map.domain().clamp(dioptres); changed = !(initted_ && fsmooth_ == ftarget_); @@ -763,7 +934,7 @@ void Af::setMode(AfAlgorithm::AfMode mode) pauseFlag_ = false; if (mode == AfModeContinuous) scanState_ = ScanState::Trigger; - else if (mode != AfModeAuto || scanState_ < ScanState::Coarse) + else if (mode != AfModeAuto || scanState_ < ScanState::Coarse1) goIdle(); } } @@ -779,12 +950,14 @@ void Af::pause(AfAlgorithm::AfPause pause) if (mode_ == AfModeContinuous) { if (pause == AfPauseResume && pauseFlag_) { pauseFlag_ = false; - if (scanState_ < ScanState::Coarse) + if (scanState_ < ScanState::Coarse1) scanState_ = ScanState::Trigger; } else if (pause != AfPauseResume && !pauseFlag_) { pauseFlag_ = true; - if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse) - goIdle(); + if (pause == AfPauseImmediate || scanState_ < ScanState::Coarse1) { + scanState_ = ScanState::Idle; + scanData_.clear(); + } } } } diff --git a/src/ipa/rpi/controller/rpi/af.h b/src/ipa/rpi/controller/rpi/af.h index 317a51f3e..d35a39d12 100644 --- a/src/ipa/rpi/controller/rpi/af.h +++ b/src/ipa/rpi/controller/rpi/af.h @@ -15,20 +15,28 @@ /* * This algorithm implements a hybrid of CDAF and PDAF, favouring PDAF. * - * Whenever PDAF is available, it is used in a continuous feedback loop. - * When triggered in auto mode, we simply enable AF for a limited number - * of frames (it may terminate early if the delta becomes small enough). + * Whenever PDAF is available (and reports sufficiently high confidence), + * it is used for continuous feedback control of the lens position. When + * triggered in Auto mode, we enable the loop for a limited number of frames + * (it may terminate sooner if the phase becomes small). In CAF mode, the + * PDAF loop runs continuously. Very small lens movements are suppressed. * * When PDAF confidence is low (due e.g. to low contrast or extreme defocus) * or PDAF data are absent, fall back to CDAF with a programmed scan pattern. - * A coarse and fine scan are performed, using ISP's CDAF focus FoM to - * estimate the lens position with peak contrast. This is slower due to - * extra latency in the ISP, and requires a settling time between steps. + * A coarse and fine scan are performed, using the ISP's CDAF contrast FoM + * to estimate the lens position with peak contrast. (This is slower due to + * extra latency in the ISP, and requires a settling time between steps.) + * The scan may terminate early if PDAF recovers and allows the zero-phase + * lens position to be interpolated. * - * Some hysteresis is applied to the switch between PDAF and CDAF, to avoid - * "nuisance" scans. During each interval where PDAF is not working, only - * ONE scan will be performed; CAF cannot track objects using CDAF alone. + * In CAF mode, the fallback to a CDAF scan is triggered when PDAF fails to + * report high confidence and a configurable number of frames have elapsed + * since the last image change since either PDAF was working or a previous + * scan found peak contrast. Image changes are detected using both contrast + * and AWB statistics (within the AF window[s]). * + * IR lighting can interfere with the correct operation of PDAF, so we + * optionally try to detect it (from AWB statistics). */ namespace RPiController { @@ -54,7 +62,9 @@ public: void setWindows(libcamera::Span const &wins) override; void setMode(AfMode mode) override; AfMode getMode() const override; - bool setLensPosition(double dioptres, int32_t *hwpos) override; + double getDefaultLensPosition() const override; + void getLensLimits(double &min, double &max) const override; + bool setLensPosition(double dioptres, int32_t *hwpos, bool force) override; std::optional getLensPosition() const override; void triggerScan() override; void cancelScan() override; @@ -65,7 +75,8 @@ private: Idle = 0, Trigger, Pdaf, - Coarse, + Coarse1, + Coarse2, Fine, Settle }; @@ -80,9 +91,11 @@ private: }; struct SpeedDependentParams { - double stepCoarse; /* used for scans */ - double stepFine; /* used for scans */ + double stepCoarse; /* in dioptres; used for scans */ + double stepFine; /* in dioptres; used for scans */ double contrastRatio; /* used for scan termination and reporting */ + double retriggerRatio; /* contrast and RGB ratio for re-triggering */ + uint32_t retriggerDelay; /* frames of stability before re-triggering */ double pdafGain; /* coefficient for PDAF feedback loop */ double pdafSquelch; /* PDAF stability parameter (device-specific) */ double maxSlew; /* limit for lens movement per frame */ @@ -101,6 +114,7 @@ private: uint32_t confThresh; /* PDAF confidence cell min (sensor-specific) */ uint32_t confClip; /* PDAF confidence cell max (sensor-specific) */ uint32_t skipFrames; /* frames to skip at start or modeswitch */ + bool checkForIR; /* Set this if PDAF is unreliable in IR light */ libcamera::ipa::Pwl map; /* converts dioptres -> lens driver position */ CfgParams(); @@ -129,6 +143,7 @@ private: void invalidateWeights(); bool getPhase(PdafRegions const ®ions, double &phase, double &conf); double getContrast(const FocusRegions &focusStats); + bool getAverageAndTestIr(const RgbyRegions &awbStats, double rgb[3]); void doPDAF(double phase, double conf); bool earlyTerminationByPhase(double phase); double findPeak(unsigned index) const; @@ -150,15 +165,20 @@ private: bool useWindows_; RegionWeights phaseWeights_; RegionWeights contrastWeights_; + RegionWeights awbWeights_; /* Working state. */ ScanState scanState_; - bool initted_; + bool initted_, irFlag_; double ftarget_, fsmooth_; - double prevContrast_; + double prevContrast_, oldSceneContrast_; + double prevAverage_[3], oldSceneAverage_[3]; + double prevPhase_; unsigned skipCount_, stepCount_, dropCount_; + unsigned sameSignCount_; + unsigned sceneChangeCount_; unsigned scanMaxIndex_; - double scanMaxContrast_, scanMinContrast_; + double scanMaxContrast_, scanMinContrast_, scanStep_; std::vector scanData_; AfState reportState_; }; diff --git a/src/ipa/rpi/pisp/data/imx708.json b/src/ipa/rpi/pisp/data/imx708.json index e8d25c216..7f2e78655 100644 --- a/src/ipa/rpi/pisp/data/imx708.json +++ b/src/ipa/rpi/pisp/data/imx708.json @@ -1139,11 +1139,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1151,6 +1167,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -1267,4 +1284,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_noir.json b/src/ipa/rpi/pisp/data/imx708_noir.json index e69afb0c6..c5e6a2652 100644 --- a/src/ipa/rpi/pisp/data/imx708_noir.json +++ b/src/ipa/rpi/pisp/data/imx708_noir.json @@ -1156,11 +1156,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1168,6 +1184,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -1230,4 +1247,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_wide.json b/src/ipa/rpi/pisp/data/imx708_wide.json index 9fff05d93..8550cdfc1 100644 --- a/src/ipa/rpi/pisp/data/imx708_wide.json +++ b/src/ipa/rpi/pisp/data/imx708_wide.json @@ -1148,23 +1148,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1172,6 +1176,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -1290,4 +1295,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/pisp/data/imx708_wide_noir.json b/src/ipa/rpi/pisp/data/imx708_wide_noir.json index 75d1149b6..069a06180 100644 --- a/src/ipa/rpi/pisp/data/imx708_wide_noir.json +++ b/src/ipa/rpi/pisp/data/imx708_wide_noir.json @@ -1057,23 +1057,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio" : 0.8, + "retrigger_delay" : 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -1081,6 +1085,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -1145,4 +1150,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708.json b/src/ipa/rpi/vc4/data/imx708.json index 4de6f0796..e54ceff49 100644 --- a/src/ipa/rpi/vc4/data/imx708.json +++ b/src/ipa/rpi/vc4/data/imx708.json @@ -638,11 +638,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -650,6 +666,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -668,4 +685,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_noir.json b/src/ipa/rpi/vc4/data/imx708_noir.json index 7b7ee874f..f351a1800 100644 --- a/src/ipa/rpi/vc4/data/imx708_noir.json +++ b/src/ipa/rpi/vc4/data/imx708_noir.json @@ -737,11 +737,27 @@ "step_coarse": 1.0, "step_fine": 0.25, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, + "pdaf_gain": -0.016, + "pdaf_squelch": 0.125, + "max_slew": 1.5, + "pdaf_frames": 20, + "dropout_frames": 6, + "step_frames": 5 + }, + "fast": + { + "step_coarse": 1.25, + "step_fine": 0.0, + "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.02, "pdaf_squelch": 0.125, "max_slew": 2.0, - "pdaf_frames": 20, - "dropout_frames": 6, + "pdaf_frames": 16, + "dropout_frames": 4, "step_frames": 4 } }, @@ -749,6 +765,7 @@ "conf_thresh": 16, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 445, 15.0, 925 ] } }, @@ -767,4 +784,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_wide.json b/src/ipa/rpi/vc4/data/imx708_wide.json index 6f45aafc0..bf1b122cd 100644 --- a/src/ipa/rpi/vc4/data/imx708_wide.json +++ b/src/ipa/rpi/vc4/data/imx708_wide.json @@ -637,23 +637,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -661,6 +665,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": false, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -679,4 +684,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/rpi/vc4/data/imx708_wide_noir.json b/src/ipa/rpi/vc4/data/imx708_wide_noir.json index b9a5227e1..ea1c8c690 100644 --- a/src/ipa/rpi/vc4/data/imx708_wide_noir.json +++ b/src/ipa/rpi/vc4/data/imx708_wide_noir.json @@ -628,23 +628,27 @@ "step_coarse": 2.0, "step_fine": 0.5, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 10, "pdaf_gain": -0.03, "pdaf_squelch": 0.2, - "max_slew": 4.0, + "max_slew": 3.0, "pdaf_frames": 20, "dropout_frames": 6, - "step_frames": 4 + "step_frames": 5 }, "fast": { - "step_coarse": 2.0, - "step_fine": 0.5, + "step_coarse": 2.5, + "step_fine": 0.0, "contrast_ratio": 0.75, + "retrigger_ratio": 0.8, + "retrigger_delay": 8, "pdaf_gain": -0.05, "pdaf_squelch": 0.2, - "max_slew": 5.0, + "max_slew": 4.0, "pdaf_frames": 16, - "dropout_frames": 6, + "dropout_frames": 4, "step_frames": 4 } }, @@ -652,6 +656,7 @@ "conf_thresh": 12, "conf_clip": 512, "skip_frames": 5, + "check_for_ir": true, "map": [ 0.0, 420, 35.0, 920 ] } }, @@ -670,4 +675,4 @@ } } ] -} \ No newline at end of file +} diff --git a/src/ipa/simple/algorithms/af.cpp b/src/ipa/simple/algorithms/af.cpp new file mode 100644 index 000000000..52ddf7f1a --- /dev/null +++ b/src/ipa/simple/algorithms/af.cpp @@ -0,0 +1,131 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#include "af.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftAutoFocus) + +namespace ipa::soft::algorithms { + +Af::Af() +{ +} + +int Af::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::LensPosition] = ControlInfo(0.0f, 100.0f, 50.0f); + context.ctrlMap[&controls::AfTrigger] = ControlInfo(0, 1, 0); + return 0; +} + +int Af::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.focus_sweep = std::optional(); + context.activeState.knobs.focus_pos = std::optional(); + context.activeState.knobs.focus_sweep = false; + context.activeState.knobs.focus_pos = 0; + context.configuration.focus.skip = 10; + return 0; +} + +void Af::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &focus_pos = controls.get(controls::LensPosition); + const auto &af_trigger = controls.get(controls::AfTrigger); + if (focus_pos.has_value()) { + context.activeState.knobs.focus_pos = focus_pos; + LOG(IPASoftAutoFocus, Debug) << "Setting focus position to " << focus_pos.value(); + } + if (af_trigger.has_value()) { + context.activeState.knobs.focus_sweep = af_trigger.value() == 1; + if(context.activeState.knobs.focus_sweep){ + context.activeState.knobs.focus_pos = 0; + context.configuration.focus.focus_max_pos = 0; + context.configuration.focus.sharpness_max = 0; + context.configuration.focus.start = 0; + context.configuration.focus.stop = 100; + context.configuration.focus.step = 25; + LOG(IPASoftAutoFocus, Info) << "Starting focus sweep"; + } + } + +} + +void Af::updateFocus([[maybe_unused]] IPAContext &context, [[maybe_unused]] IPAFrameContext &frameContext, [[maybe_unused]] double exposureMSV) +{ + frameContext.lens.focus_pos = context.activeState.knobs.focus_pos.value_or(50.0) / 100.0 * (context.configuration.focus.focus_max - context.configuration.focus.focus_min); +} + +void Af::step(uint32_t& skip, double& start, double& stop, double& step, double& focus_pos, double& max_pos, uint64_t& max_sharp, uint64_t sharp, bool& sweep){ + if(!sweep) + return; + if(skip != 0){ + skip --; + return; + } + skip = 2; + if(focus_pos < start) { + focus_pos = start; + return; + } + if(sharp > max_sharp) { + max_sharp = sharp; + max_pos = focus_pos; + } + if(focus_pos >= stop) { + LOG(IPASoftAutoFocus, Info) << "Best focus on step " <sharpness, + context.activeState.knobs.focus_sweep.value()); + updateFocus(context, frameContext, 0); +} + +REGISTER_IPA_ALGORITHM(Af, "Af") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/af.h b/src/ipa/simple/algorithms/af.h new file mode 100644 index 000000000..901393717 --- /dev/null +++ b/src/ipa/simple/algorithms/af.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Auto focus + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Af : public Algorithm +{ +public: + Af(); + ~Af() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void process(IPAContext &context, const uint32_t frame, + IPAFrameContext &frameContext, + const SwIspStats *stats, + ControlList &metadata) override; + +private: + void updateFocus(IPAContext &context, IPAFrameContext &frameContext, double focus); + void step(uint32_t& skip, double& start, double& stop, double& step, double& focus_pos, double& max_pos, uint64_t& max_sharp, uint64_t sharp, bool& sweep); +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/agc.cpp b/src/ipa/simple/algorithms/agc.cpp index c46bb0ebe..35f4798cf 100644 --- a/src/ipa/simple/algorithms/agc.cpp +++ b/src/ipa/simple/algorithms/agc.cpp @@ -41,6 +41,47 @@ Agc::Agc() { } +int Agc::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::AeEnable] = ControlInfo(false, true, true); + context.ctrlMap[&controls::Brightness] = ControlInfo(0.0f, 2.0f, 1.0f); + context.ctrlMap[&controls::ExposureValue] = ControlInfo(0.0f, 0.5f, 1.0f); + + return 0; +} + +int Agc::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.brightness = std::optional(); + context.activeState.knobs.ae_enabled = std::optional(); + + return 0; +} + +void Agc::queueRequest(typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &brightness = controls.get(controls::Brightness); + const auto &ae_enabled = controls.get(controls::AeEnable); + const auto &exposure_value = controls.get(controls::ExposureValue); + if (brightness.has_value()) { + context.activeState.knobs.brightness = brightness; + LOG(IPASoftExposure, Debug) << "Setting brightness to " << brightness.value(); + } + if (ae_enabled.has_value()) { + context.activeState.knobs.ae_enabled = ae_enabled; + LOG(IPASoftExposure, Debug) << "Setting ae_enable to " << ae_enabled.value(); + } + if (exposure_value.has_value()) { + context.activeState.knobs.exposure_value = exposure_value.value(); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << exposure_value.value(); + } +} + void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, double exposureMSV) { /* @@ -54,6 +95,8 @@ void Agc::updateExposure(IPAContext &context, IPAFrameContext &frameContext, dou double next; int32_t &exposure = frameContext.sensor.exposure; double &again = frameContext.sensor.gain; + const auto brightness = context.activeState.knobs.brightness.value_or(1.0); + exposureMSV /= brightness; if (exposureMSV < kExposureOptimal - kExposureSatisfactory) { next = exposure * kExpNumeratorUp / kExpDenominator; @@ -103,10 +146,17 @@ void Agc::process(IPAContext &context, const SwIspStats *stats, ControlList &metadata) { + const auto ae_enable = context.activeState.knobs.ae_enabled.value_or(true); + + if (!ae_enable) + frameContext.sensor.exposure = (int32_t)( context.activeState.knobs.exposure_value.value_or(0.5) * (context.configuration.agc.exposureMax - context.configuration.agc.exposureMin)); utils::Duration exposureTime = context.configuration.agc.lineDuration * frameContext.sensor.exposure; metadata.set(controls::ExposureTime, exposureTime.get()); metadata.set(controls::AnalogueGain, frameContext.sensor.gain); + LOG(IPASoftExposure, Debug) << "Setting exposure value to " << frameContext.sensor.exposure; + if (!ae_enable) + return; /* * Calculate Mean Sample Value (MSV) according to formula from: diff --git a/src/ipa/simple/algorithms/agc.h b/src/ipa/simple/algorithms/agc.h index 112d9f5a1..00bc101e7 100644 --- a/src/ipa/simple/algorithms/agc.h +++ b/src/ipa/simple/algorithms/agc.h @@ -19,6 +19,13 @@ public: Agc(); ~Agc() = default; + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; void process(IPAContext &context, const uint32_t frame, IPAFrameContext &frameContext, const SwIspStats *stats, diff --git a/src/ipa/simple/algorithms/meson.build b/src/ipa/simple/algorithms/meson.build index 2d0adb059..e5666b262 100644 --- a/src/ipa/simple/algorithms/meson.build +++ b/src/ipa/simple/algorithms/meson.build @@ -6,4 +6,6 @@ soft_simple_ipa_algorithms = files([ 'blc.cpp', 'ccm.cpp', 'lut.cpp', + 'af.cpp', + 'stat.cpp', ]) diff --git a/src/ipa/simple/algorithms/stat.cpp b/src/ipa/simple/algorithms/stat.cpp new file mode 100644 index 000000000..181a5d818 --- /dev/null +++ b/src/ipa/simple/algorithms/stat.cpp @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Debayer statistic controls + */ + +#include "stat.h" + +#include + +#include + +#include "control_ids.h" + +namespace libcamera { + +LOG_DEFINE_CATEGORY(IPASoftStatistic) + +namespace ipa::soft::algorithms { + +Stat::Stat() +{ +} + +int Stat::init(IPAContext &context, + [[maybe_unused]] const YamlObject &tuningData) +{ + context.ctrlMap[&controls::DebugMetadataEnable] = ControlInfo(false, true, true); + return 0; +} + +int Stat::configure(IPAContext &context, + [[maybe_unused]] const IPAConfigInfo &configInfo) +{ + context.activeState.knobs.stats_enabled = std::optional(); + + return 0; +} + +void Stat::queueRequest([[maybe_unused]] typename Module::Context &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]] typename Module::FrameContext &frameContext, + const ControlList &controls) +{ + const auto &stats_enabled = controls.get(controls::DebugMetadataEnable); + if (stats_enabled.has_value()) { + context.activeState.knobs.stats_enabled = stats_enabled; + LOG(IPASoftStatistic, Debug) << "Setting debayer enabled to " << stats_enabled.value(); + } +} + +void Stat::prepare([[maybe_unused]]IPAContext &context, + [[maybe_unused]] const uint32_t frame, + [[maybe_unused]]IPAFrameContext &frameContext, + [[maybe_unused]] DebayerParams *params) +{ + params->collect_stats = context.activeState.knobs.stats_enabled.value_or(true); +} + +REGISTER_IPA_ALGORITHM(Stat, "Stat") + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/algorithms/stat.h b/src/ipa/simple/algorithms/stat.h new file mode 100644 index 000000000..dc0051cb0 --- /dev/null +++ b/src/ipa/simple/algorithms/stat.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2025 Vasiliy Doylov + * + * Debayer statistic controls + */ + +#pragma once + +#include "algorithm.h" + +namespace libcamera { + +namespace ipa::soft::algorithms { + +class Stat : public Algorithm +{ +public: + Stat(); + ~Stat() = default; + + int init(IPAContext &context, const YamlObject &tuningData) override; + int configure(IPAContext &context, const IPAConfigInfo &configInfo) override; + void queueRequest(typename Module::Context &context, + const uint32_t frame, + typename Module::FrameContext &frameContext, + const ControlList &controls) + override; + void prepare(IPAContext &context, + const uint32_t frame, + IPAFrameContext &frameContext, + DebayerParams *params) override; + +}; + +} /* namespace ipa::soft::algorithms */ + +} /* namespace libcamera */ diff --git a/src/ipa/simple/data/uncalibrated.yaml b/src/ipa/simple/data/uncalibrated.yaml index 5508e6686..47eaf9c61 100644 --- a/src/ipa/simple/data/uncalibrated.yaml +++ b/src/ipa/simple/data/uncalibrated.yaml @@ -16,4 +16,6 @@ algorithms: # 0, 0, 1] - Lut: - Agc: + - Af: + - Stat: ... diff --git a/src/ipa/simple/ipa_context.h b/src/ipa/simple/ipa_context.h index a471b80ae..f59c4006f 100644 --- a/src/ipa/simple/ipa_context.h +++ b/src/ipa/simple/ipa_context.h @@ -34,6 +34,13 @@ struct IPASessionConfiguration { struct { std::optional level; } black; + struct { + int32_t focus_min, focus_max; + double focus_max_pos; + uint64_t sharpness_max; + double start, stop, step; + uint32_t skip; + } focus; }; struct IPAActiveState { @@ -64,6 +71,18 @@ struct IPAActiveState { /* 0..2 range, 1.0 = normal */ std::optional contrast; std::optional saturation; + /* 0..2 range, 1.0 = normal */ + std::optional brightness; + /* 0..1 range, 1 = normal */ + std::optional ae_enabled; + /* 0..1 range, 0.5 = normal */ + std::optional exposure_value; + /* 0..100 range, 50.0 = normal */ + std::optional focus_pos; + /* 0..1 range, 1 = normal */ + std::optional stats_enabled; + /* 0..1 range, 0 = normal */ + std::optional focus_sweep; } knobs; }; @@ -77,6 +96,10 @@ struct IPAFrameContext : public FrameContext { double gain; } sensor; + struct { + int32_t focus_pos; + } lens; + struct { double red; double blue; diff --git a/src/ipa/simple/soft_simple.cpp b/src/ipa/simple/soft_simple.cpp index c94c4cd55..53fa74c03 100644 --- a/src/ipa/simple/soft_simple.cpp +++ b/src/ipa/simple/soft_simple.cpp @@ -77,6 +77,7 @@ private: SwIspStats *stats_; std::unique_ptr camHelper_; ControlInfoMap sensorInfoMap_; + ControlInfoMap lensInfoMap_; /* Local parameter storage */ struct IPAContext context_; @@ -196,6 +197,7 @@ int IPASoftSimple::init(const IPASettings &settings, int IPASoftSimple::configure(const IPAConfigInfo &configInfo) { sensorInfoMap_ = configInfo.sensorControls; + lensInfoMap_ = configInfo.lensControls; const ControlInfo &exposureInfo = sensorInfoMap_.find(V4L2_CID_EXPOSURE)->second; const ControlInfo &gainInfo = sensorInfoMap_.find(V4L2_CID_ANALOGUE_GAIN)->second; @@ -205,6 +207,17 @@ int IPASoftSimple::configure(const IPAConfigInfo &configInfo) context_.activeState = {}; context_.frameContexts.clear(); + if (lensInfoMap_.empty()) { + LOG(IPASoft, Warning) << "No camera leans found! Focus control disabled."; + context_.configuration.focus.focus_min = 0; + context_.configuration.focus.focus_max = 0; + } else { + const ControlInfo &lensInfo = lensInfoMap_.find(V4L2_CID_FOCUS_ABSOLUTE)->second; + context_.configuration.focus.focus_min = lensInfo.min().get(); + context_.configuration.focus.focus_max = lensInfo.max().get(); + LOG(IPASoft, Warning) << "Camera leans found! Focus: " << context_.configuration.focus.focus_min << "-" << context_.configuration.focus.focus_max; + } + context_.configuration.agc.lineDuration = context_.sensorInfo.minLineLength * 1.0s / context_.sensorInfo.pixelRate; context_.configuration.agc.exposureMin = exposureInfo.min().get(); @@ -327,7 +340,10 @@ void IPASoftSimple::processStats(const uint32_t frame, ctrls.set(V4L2_CID_ANALOGUE_GAIN, static_cast(camHelper_ ? camHelper_->gainCode(againNew) : againNew)); - setSensorControls.emit(ctrls); + ControlList lens_ctrls(lensInfoMap_); + lens_ctrls.set(V4L2_CID_FOCUS_ABSOLUTE, frameContext.lens.focus_pos); + + setSensorControls.emit(ctrls, lens_ctrls); } std::string IPASoftSimple::logPrefix() const diff --git a/src/libcamera/base/log.cpp b/src/libcamera/base/log.cpp index 8bf3e1daa..6a8e2a3eb 100644 --- a/src/libcamera/base/log.cpp +++ b/src/libcamera/base/log.cpp @@ -690,8 +690,9 @@ LogSeverity Logger::parseLogLevel(std::string_view level) unsigned int severity = LogInvalid; if (std::isdigit(level[0])) { - auto [end, ec] = std::from_chars(level.data(), level.data() + level.size(), severity); - if (ec != std::errc() || *end != '\0' || severity > LogFatal) + const char *levelEnd = level.data() + level.size(); + auto [end, ec] = std::from_chars(level.data(), levelEnd, severity); + if (ec != std::errc() || end != levelEnd || severity > LogFatal) severity = LogInvalid; } else { for (unsigned int i = 0; i < std::size(names); ++i) { diff --git a/src/libcamera/camera.cpp b/src/libcamera/camera.cpp index c180a5fdd..b4f9d2433 100644 --- a/src/libcamera/camera.cpp +++ b/src/libcamera/camera.cpp @@ -488,7 +488,7 @@ std::size_t CameraConfiguration::size() const * * \return A CameraConfiguration::Status value that describes the validation * status. - * \retval CameraConfigutation::Adjusted The configuration has been adjusted + * \retval CameraConfiguration::Adjusted The configuration has been adjusted * and is now valid. The color space of some or all of the streams may have * been changed. The caller shall check the color spaces carefully. * \retval CameraConfiguration::Valid The configuration was already valid and diff --git a/src/libcamera/clock_recovery.cpp b/src/libcamera/clock_recovery.cpp new file mode 100644 index 000000000..abacf444f --- /dev/null +++ b/src/libcamera/clock_recovery.cpp @@ -0,0 +1,230 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +/* + * Copyright (C) 2024, Raspberry Pi Ltd + * + * Clock recovery algorithm + */ + +#include "libcamera/internal/clock_recovery.h" + +#include + +#include + +/** + * \file clock_recovery.h + * \brief Clock recovery - deriving one clock from another independent clock + */ + +namespace libcamera { + +LOG_DEFINE_CATEGORY(ClockRec) + +/** + * \class ClockRecovery + * \brief Recover an output clock from an input clock + * + * The ClockRecovery class derives an output clock from an input clock, + * modelling the output clock as being linearly related to the input clock. + * For example, we may use it to derive wall clock timestamps from timestamps + * measured by the internal system clock which counts local time since boot. + * + * When pairs of corresponding input and output timestamps are available, + * they should be submitted to the model with addSample(). The model will + * update, and output clock values for known input clock values can be + * obtained using getOutput(). + * + * As a convenience, if the input clock is indeed the time since boot, and the + * output clock represents a real wallclock time, then addSample() can be + * called with no arguments, and a pair of timestamps will be captured at + * that moment. + * + * The configure() function accepts some configuration parameters to control + * the linear fitting process. + */ + +/** + * \brief Construct a ClockRecovery + */ +ClockRecovery::ClockRecovery() +{ + configure(); + reset(); +} + +/** + * \brief Set configuration parameters + * \param[in] numSamples The approximate duration for which the state of the model + * is persistent + * \param[in] maxJitter New output samples are clamped to no more than this + * amount of jitter, to prevent sudden swings from having a large effect + * \param[in] minSamples The fitted clock model is not used to generate outputs + * until this many samples have been received + * \param[in] errorThreshold If the accumulated differences between input and + * output clocks reaches this amount over a few frames, the model is reset + */ +void ClockRecovery::configure(unsigned int numSamples, unsigned int maxJitter, + unsigned int minSamples, unsigned int errorThreshold) +{ + LOG(ClockRec, Debug) + << "configure " << numSamples << " " << maxJitter << " " << minSamples << " " << errorThreshold; + + numSamples_ = numSamples; + maxJitter_ = maxJitter; + minSamples_ = minSamples; + errorThreshold_ = errorThreshold; +} + +/** + * \brief Reset the clock recovery model and start again from scratch + */ +void ClockRecovery::reset() +{ + LOG(ClockRec, Debug) << "reset"; + + lastInput_ = 0; + lastOutput_ = 0; + xAve_ = 0; + yAve_ = 0; + x2Ave_ = 0; + xyAve_ = 0; + count_ = 0; + error_ = 0.0; + /* + * Setting slope_ and offset_ to zero initially means that the clocks + * advance at exactly the same rate. + */ + slope_ = 0.0; + offset_ = 0.0; +} + +/** + * \brief Add a sample point to the clock recovery model, for recovering a wall + * clock value from the internal system time since boot + * + * This is a convenience function to make it easy to derive a wall clock value + * (using the Linux CLOCK_REALTIME) from the time since the system started + * (measured by CLOCK_BOOTTIME). + */ +void ClockRecovery::addSample() +{ + LOG(ClockRec, Debug) << "addSample"; + + struct timespec bootTime1; + struct timespec bootTime2; + struct timespec wallTime; + + /* Get boot and wall clocks in microseconds. */ + clock_gettime(CLOCK_BOOTTIME, &bootTime1); + clock_gettime(CLOCK_REALTIME, &wallTime); + clock_gettime(CLOCK_BOOTTIME, &bootTime2); + uint64_t boot1 = bootTime1.tv_sec * 1000000ULL + bootTime1.tv_nsec / 1000; + uint64_t boot2 = bootTime2.tv_sec * 1000000ULL + bootTime2.tv_nsec / 1000; + uint64_t boot = (boot1 + boot2) / 2; + uint64_t wall = wallTime.tv_sec * 1000000ULL + wallTime.tv_nsec / 1000; + + addSample(boot, wall); +} + +/** + * \brief Add a sample point to the clock recovery model, specifying the exact + * input and output clock values + * \param[in] input The input clock value + * \param[in] output The value of the output clock at the same moment, as far + * as possible, that the input clock was sampled + * + * This function should be used for corresponding clocks other than the Linux + * BOOTTIME and REALTIME clocks. + */ +void ClockRecovery::addSample(uint64_t input, uint64_t output) +{ + LOG(ClockRec, Debug) << "addSample " << input << " " << output; + + if (count_ == 0) { + inputBase_ = input; + outputBase_ = output; + } + + /* + * We keep an eye on cumulative drift over the last several frames. If this exceeds a + * threshold, then probably the system clock has been updated and we're going to have to + * reset everything and start over. + */ + if (lastOutput_) { + int64_t inputDiff = getOutput(input) - getOutput(lastInput_); + int64_t outputDiff = output - lastOutput_; + error_ = error_ * 0.95 + (outputDiff - inputDiff); + if (std::abs(error_) > errorThreshold_) { + reset(); + inputBase_ = input; + outputBase_ = output; + } + } + lastInput_ = input; + lastOutput_ = output; + + /* + * Never let the new output value be more than maxJitter_ away from what + * we would have expected. This is just to reduce the effect of sudden + * large delays in the measured output. + */ + uint64_t expectedOutput = getOutput(input); + output = std::clamp(output, expectedOutput - maxJitter_, expectedOutput + maxJitter_); + + /* + * We use x, y, x^2 and x*y sums to calculate the best fit line. Here we + * update them by pretending we have count_ samples at the previous fit, + * and now one new one. Gradually the effect of the older values gets + * lost. This is a very simple way of updating the fit (there are much + * more complicated ones!), but it works well enough. Using averages + * instead of sums makes the relative effect of old values and the new + * sample clearer. + */ + double x = static_cast(input - inputBase_); + double y = static_cast(output - outputBase_) - x; + unsigned int count1 = count_ + 1; + xAve_ = (count_ * xAve_ + x) / count1; + yAve_ = (count_ * yAve_ + y) / count1; + x2Ave_ = (count_ * x2Ave_ + x * x) / count1; + xyAve_ = (count_ * xyAve_ + x * y) / count1; + + /* + * Don't update slope and offset until we've seen "enough" sample + * points. Note that the initial settings for slope_ and offset_ + * ensures that the wallclock advances at the same rate as the realtime + * clock (but with their respective initial offsets). + */ + if (count_ > minSamples_) { + /* These are the standard equations for least squares linear regression. */ + slope_ = (count1 * count1 * xyAve_ - count1 * xAve_ * count1 * yAve_) / + (count1 * count1 * x2Ave_ - count1 * xAve_ * count1 * xAve_); + offset_ = yAve_ - slope_ * xAve_; + } + + /* + * Don't increase count_ above numSamples_, as this controls the long-term + * amount of the residual fit. + */ + if (count1 < numSamples_) + count_++; +} + +/** + * \brief Calculate the output clock value according to the model from an input + * clock value + * \param[in] input The input clock value + * + * \return Output clock value + */ +uint64_t ClockRecovery::getOutput(uint64_t input) +{ + double x = static_cast(input - inputBase_); + double y = slope_ * x + offset_; + uint64_t output = y + x + outputBase_; + + LOG(ClockRec, Debug) << "getOutput " << input << " " << output; + + return output; +} + +} /* namespace libcamera */ diff --git a/src/libcamera/control_ids_core.yaml b/src/libcamera/control_ids_core.yaml index aa7448645..eec4b4f93 100644 --- a/src/libcamera/control_ids_core.yaml +++ b/src/libcamera/control_ids_core.yaml @@ -212,7 +212,7 @@ controls: description: | Exposure time for the frame applied in the sensor device. - This value is specified in micro-seconds. + This value is specified in microseconds. This control will only take effect if ExposureTimeMode is Manual. If this control is set when ExposureTimeMode is Auto, the value will be @@ -1268,4 +1268,20 @@ controls: description: | Enable or disable the debug metadata. + - FrameWallClock: + type: int64_t + direction: out + description: | + This timestamp corresponds to the same moment in time as the + SensorTimestamp, but is represented as a wall clock time as measured by + the CLOCK_REALTIME clock. Like SensorTimestamp, the timestamp value is + expressed in nanoseconds. + + Being a wall clock measurement, it can be used to synchronise timing + across different devices. + + \sa SensorTimestamp + + The FrameWallClock control can only be returned in metadata. + ... diff --git a/src/libcamera/control_ids_rpi.yaml b/src/libcamera/control_ids_rpi.yaml index 8d1e8b47c..a86151123 100644 --- a/src/libcamera/control_ids_rpi.yaml +++ b/src/libcamera/control_ids_rpi.yaml @@ -71,4 +71,116 @@ controls: \sa StatsOutputEnable + - SyncMode: + type: int32_t + direction: in + description: | + Enable or disable camera synchronisation ("sync") mode. + + When sync mode is enabled, a camera will synchronise frames temporally + with other cameras, either attached to the same device or a different + one. There should be one "server" device, which broadcasts timing + information to one or more "clients". Communication is one-way, from + server to clients only, and it is only clients that adjust their frame + timings to match the server. + + Sync mode requires all cameras to be running at (as far as possible) the + same fixed framerate. Clients may continue to make adjustments to keep + their cameras synchronised with the server for the duration of the + session, though any updates after the initial ones should remain small. + + \sa SyncReady + \sa SyncTimer + \sa SyncFrames + + enum: + - name: SyncModeOff + value: 0 + description: Disable sync mode. + - name: SyncModeServer + value: 1 + description: | + Enable sync mode, act as server. The server broadcasts timing + messages to any clients that are listening, so that the clients can + synchronise their camera frames with the server's. + - name: SyncModeClient + value: 2 + description: | + Enable sync mode, act as client. A client listens for any server + messages, and arranges for its camera frames to synchronise as + closely as possible with the server's. Many clients can listen out + for the same server. Clients can also be started ahead of any + servers, causing them merely to wait for the server to start. + + - SyncReady: + type: bool + direction: out + description: | + When using the camera synchronisation algorithm, the server broadcasts + timing information to the clients. This also includes the time (some + number of frames in the future, called the "ready time") at which the + server will signal its controlling application, using this control, to + start using the image frames. + + The client receives the "ready time" from the server, and will signal + its application to start using the frames at this same moment. + + While this control value is false, applications (on both client and + server) should continue to wait, and not use the frames. + + Once this value becomes true, it means that this is the first frame + where the server and its clients have agreed that they will both be + synchronised and that applications should begin consuming frames. + Thereafter, this control will continue to signal the value true for + the rest of the session. + + \sa SyncMode + \sa SyncTimer + \sa SyncFrames + + - SyncTimer: + type: int64_t + direction: out + description: | + This reports the amount of time, in microseconds, until the "ready + time", at which the server and client will signal their controlling + applications that the frames are now synchronised and should be + used. The value may be refined slightly over time, becoming more precise + as the "ready time" approaches. + + Servers always report this value, whereas clients will omit this control + until they have received a message from the server that enables them to + calculate it. + + Normally the value will start positive (the "ready time" is in the + future), and decrease towards zero, before becoming negative (the "ready + time" has elapsed). So there should be just one frame where the timer + value is, or is very close to, zero - the one for which the SyncReady + control becomes true. At this moment, the value indicates how closely + synchronised the client believes it is with the server. + + But note that if frames are being dropped, then the "near zero" valued + frame, or indeed any other, could be skipped. In these cases the timer + value allows an application to deduce that this has happened. + + \sa SyncMode + \sa SyncReady + \sa SyncFrames + + - SyncFrames: + type: int32_t + direction: in + description: | + The number of frames the server should wait, after enabling + SyncModeServer, before signalling (via the SyncReady control) that + frames should be used. This therefore determines the "ready time" for + all synchronised cameras. + + This control value should be set only for the device that is to act as + the server, before or at the same moment at which SyncModeServer is + enabled. + + \sa SyncMode + \sa SyncReady + \sa SyncTimer ... diff --git a/src/libcamera/framebuffer.cpp b/src/libcamera/framebuffer.cpp index 826848f75..219db50d6 100644 --- a/src/libcamera/framebuffer.cpp +++ b/src/libcamera/framebuffer.cpp @@ -43,12 +43,19 @@ LOG_DEFINE_CATEGORY(Buffer) * The frame has been captured with success and contains valid data. All fields * of the FrameMetadata structure are valid. * \var FrameMetadata::FrameError - * An error occurred during capture of the frame. The frame data may be partly - * or fully invalid. The sequence and timestamp fields of the FrameMetadata - * structure is valid, the other fields may be invalid. + * The frame data is partly or fully corrupted, missing or otherwise invalid. + * This can for instance indicate a hardware transmission error, or invalid data + * produced by the sensor during its startup phase. The sequence and timestamp + * fields of the FrameMetadata structure is valid, all the other fields may be + * invalid. * \var FrameMetadata::FrameCancelled * Capture stopped before the frame completed. The frame data is not valid. All * fields of the FrameMetadata structure but the status field are invalid. + * \var FrameMetadata::FrameStartup + * The frame has been successfully captured. However, the IPA is in a + * cold-start or reset phase and will result in image quality parameters + * producing unusable images. Applications are recommended to not consume these + * frames. All other fields of the FrameMetadata structure are valid. */ /** diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build index 202db1efe..de1eb99b2 100644 --- a/src/libcamera/meson.build +++ b/src/libcamera/meson.build @@ -21,6 +21,7 @@ libcamera_internal_sources = files([ 'byte_stream_buffer.cpp', 'camera_controls.cpp', 'camera_lens.cpp', + 'clock_recovery.cpp', 'control_serializer.cpp', 'control_validator.cpp', 'converter.cpp', @@ -83,7 +84,10 @@ if not cc.has_function('dlopen') libdl = cc.find_library('dl') endif libudev = dependency('libudev', required : get_option('udev')) -libyaml = dependency('yaml-0.1', required : false) +libyaml = dependency('yaml-0.1', default_options : [ + 'default_library=static', + 'werror=false', +]) # Use one of gnutls or libcrypto (provided by OpenSSL), trying gnutls first. libcrypto = dependency('gnutls', required : false) @@ -119,17 +123,6 @@ if libudev.found() ]) endif -# Fallback to a subproject if libyaml isn't found, as it's not packaged in AOSP. -if not libyaml.found() - cmake = import('cmake') - - libyaml_vars = cmake.subproject_options() - libyaml_vars.add_cmake_defines({'CMAKE_POSITION_INDEPENDENT_CODE': 'ON'}) - libyaml_vars.append_compile_args('c', '-Wno-unused-value') - libyaml_wrap = cmake.subproject('libyaml', options : libyaml_vars) - libyaml = libyaml_wrap.dependency('yaml') -endif - control_sources = [] controls_mode_files = { diff --git a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp index ecda426a6..f4014b95d 100644 --- a/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp +++ b/src/libcamera/pipeline/imx8-isi/imx8-isi.cpp @@ -761,30 +761,28 @@ PipelineHandlerISI::generateConfiguration(Camera *camera, */ StreamConfiguration cfg; - switch (role) { - case StreamRole::StillCapture: - case StreamRole::Viewfinder: - case StreamRole::VideoRecording: { - Size size = role == StreamRole::StillCapture - ? data->sensor_->resolution() - : PipelineHandlerISI::kPreviewSize; - cfg = generateYUVConfiguration(camera, size); - if (cfg.pixelFormat.isValid()) - break; + switch (role) { + case StreamRole::StillCapture: + case StreamRole::Viewfinder: + case StreamRole::VideoRecording: { + Size size = role == StreamRole::StillCapture + ? data->sensor_->resolution() + : PipelineHandlerISI::kPreviewSize; + cfg = generateYUVConfiguration(camera, size); + if (cfg.pixelFormat.isValid()) + break; + /* + * Fallback to use a Bayer format if that's what the + * sensor supports. + */ + [[fallthrough]]; + } - /* - * Fallback to use a Bayer format if that's what the - * sensor supports. - */ - [[fallthrough]]; - - } - - case StreamRole::Raw: { - cfg = generateRawConfiguration(camera); - break; - } + case StreamRole::Raw: { + cfg = generateRawConfiguration(camera); + break; + } default: LOG(ISI, Error) << "Requested stream role not supported: " << role; @@ -822,7 +820,7 @@ int PipelineHandlerISI::configure(Camera *camera, CameraConfiguration *c) * routing table instead of resetting it. */ V4L2Subdevice::Routing routing = {}; - unsigned int xbarFirstSource = crossbar_->entity()->pads().size() / 2 + 1; + unsigned int xbarFirstSource = crossbar_->entity()->pads().size() - pipes_.size(); for (const auto &[idx, config] : utils::enumerate(*c)) { uint32_t sourcePad = xbarFirstSource + idx; @@ -1005,7 +1003,7 @@ bool PipelineHandlerISI::match(DeviceEnumerator *enumerator) ret = capture->open(); if (ret) - return ret; + return false; pipes_.push_back({ std::move(isi), std::move(capture) }); } diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 1f13e5230..eafe94427 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -659,9 +659,9 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) if (!result.controls.empty()) data->setSensorControls(result.controls); - /* Configure the number of dropped frames required on startup. */ - data->dropFrameCount_ = data->config_.disableStartupFrameDrops - ? 0 : result.dropFrameCount; + /* Configure the number of startup and invalid frames reported by the IPA. */ + data->startupFrameCount_ = result.startupFrameCount; + data->invalidFrameCount_ = result.invalidFrameCount; for (auto const stream : data->streams_) stream->resetBuffers(); @@ -678,7 +678,6 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) data->buffersAllocated_ = true; } - /* We need to set the dropFrameCount_ before queueing buffers. */ ret = queueAllBuffers(camera); if (ret) { LOG(RPI, Error) << "Failed to queue buffers"; @@ -686,6 +685,9 @@ int PipelineHandlerBase::start(Camera *camera, const ControlList *controls) return ret; } + /* A good moment to add an initial clock sample. */ + data->wallClockRecovery_.addSample(); + /* * Reset the delayed controls with the gain and exposure values set by * the IPA. @@ -804,7 +806,8 @@ int PipelineHandlerBase::registerCamera(std::unique_ptr &camera * chain. There may be a cascade of devices in this chain! */ MediaLink *link = sensorEntity->getPadByIndex(0)->links()[0]; - data->enumerateVideoDevices(link, frontendName); + if (!data->enumerateVideoDevices(link, frontendName)) + return -EINVAL; ipa::RPi::InitResult result; if (data->loadIPA(&result)) { @@ -894,28 +897,12 @@ int PipelineHandlerBase::queueAllBuffers(Camera *camera) int ret; for (auto const stream : data->streams_) { - if (!(stream->getFlags() & StreamFlag::External)) { - ret = stream->queueAllBuffers(); - if (ret < 0) - return ret; - } else { - /* - * For external streams, we must queue up a set of internal - * buffers to handle the number of drop frames requested by - * the IPA. This is done by passing nullptr in queueBuffer(). - * - * The below queueBuffer() call will do nothing if there - * are not enough internal buffers allocated, but this will - * be handled by queuing the request for buffers in the - * RPiStream object. - */ - unsigned int i; - for (i = 0; i < data->dropFrameCount_; i++) { - ret = stream->queueBuffer(nullptr); - if (ret) - return ret; - } - } + if (stream->getFlags() & StreamFlag::External) + continue; + + ret = stream->queueAllBuffers(); + if (ret < 0) + return ret; } return 0; @@ -1032,16 +1019,20 @@ void CameraData::freeBuffers() * | Sensor2 | | Sensor3 | * +---------+ +---------+ */ -void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &frontend) +bool CameraData::enumerateVideoDevices(MediaLink *link, const std::string &frontend) { const MediaPad *sinkPad = link->sink(); const MediaEntity *entity = sinkPad->entity(); bool frontendFound = false; + /* Once we reach the Frontend entity, we are done. */ + if (link->sink()->entity()->name() == frontend) + return true; + /* We only deal with Video Mux and Bridge devices in cascade. */ if (entity->function() != MEDIA_ENT_F_VID_MUX && entity->function() != MEDIA_ENT_F_VID_IF_BRIDGE) - return; + return false; /* Find the source pad for this Video Mux or Bridge device. */ const MediaPad *sourcePad = nullptr; @@ -1053,7 +1044,7 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front * and this branch in the cascade. */ if (sourcePad) - return; + return false; sourcePad = pad; } @@ -1070,12 +1061,9 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front * other Video Mux and Bridge devices. */ for (MediaLink *l : sourcePad->links()) { - enumerateVideoDevices(l, frontend); - /* Once we reach the Frontend entity, we are done. */ - if (l->sink()->entity()->name() == frontend) { - frontendFound = true; + frontendFound = enumerateVideoDevices(l, frontend); + if (frontendFound) break; - } } /* This identifies the end of our entity enumeration recursion. */ @@ -1090,12 +1078,13 @@ void CameraData::enumerateVideoDevices(MediaLink *link, const std::string &front bridgeDevices_.clear(); } } + + return frontendFound; } int CameraData::loadPipelineConfiguration() { config_ = { - .disableStartupFrameDrops = false, .cameraTimeoutValue = 0, }; @@ -1132,8 +1121,10 @@ int CameraData::loadPipelineConfiguration() const YamlObject &phConfig = (*root)["pipeline_handler"]; - config_.disableStartupFrameDrops = - phConfig["disable_startup_frame_drops"].get(config_.disableStartupFrameDrops); + if (phConfig.contains("disable_startup_frame_drops")) + LOG(RPI, Warning) + << "The disable_startup_frame_drops key is now deprecated, " + << "startup frames are now identified by the FrameMetadata::Status::FrameStartup flag"; config_.cameraTimeoutValue = phConfig["camera_timeout_value_ms"].get(config_.cameraTimeoutValue); @@ -1412,7 +1403,15 @@ void CameraData::handleStreamBuffer(FrameBuffer *buffer, RPi::Stream *stream) * buffer back to the stream. */ Request *request = requestQueue_.empty() ? nullptr : requestQueue_.front(); - if (!dropFrameCount_ && request && request->findBuffer(stream) == buffer) { + if (request && request->findBuffer(stream) == buffer) { + FrameMetadata &md = buffer->_d()->metadata(); + + /* Mark the non-converged and invalid frames in the metadata. */ + if (invalidFrameCount_) + md.status = FrameMetadata::Status::FrameError; + else if (startupFrameCount_) + md.status = FrameMetadata::Status::FrameStartup; + /* * Tag the buffer as completed, returning it to the * application. @@ -1458,42 +1457,31 @@ void CameraData::handleState() void CameraData::checkRequestCompleted() { - bool requestCompleted = false; - /* - * If we are dropping this frame, do not touch the request, simply - * change the state to IDLE when ready. - */ - if (!dropFrameCount_) { - Request *request = requestQueue_.front(); - if (request->hasPendingBuffers()) - return; + Request *request = requestQueue_.front(); + if (request->hasPendingBuffers()) + return; - /* Must wait for metadata to be filled in before completing. */ - if (state_ != State::IpaComplete) - return; + /* Must wait for metadata to be filled in before completing. */ + if (state_ != State::IpaComplete) + return; - LOG(RPI, Debug) << "Completing request sequence: " - << request->sequence(); + LOG(RPI, Debug) << "Completing request sequence: " + << request->sequence(); - pipe()->completeRequest(request); - requestQueue_.pop(); - requestCompleted = true; - } + pipe()->completeRequest(request); + requestQueue_.pop(); - /* - * Make sure we have three outputs completed in the case of a dropped - * frame. - */ - if (state_ == State::IpaComplete && - ((ispOutputCount_ == ispOutputTotal_ && dropFrameCount_) || - requestCompleted)) { - LOG(RPI, Debug) << "Going into Idle state"; - state_ = State::Idle; - if (dropFrameCount_) { - dropFrameCount_--; - LOG(RPI, Debug) << "Dropping frame at the request of the IPA (" - << dropFrameCount_ << " left)"; - } + LOG(RPI, Debug) << "Going into Idle state"; + state_ = State::Idle; + + if (invalidFrameCount_) { + invalidFrameCount_--; + LOG(RPI, Debug) << "Decrementing invalid frames to " + << invalidFrameCount_; + } else if (startupFrameCount_) { + startupFrameCount_--; + LOG(RPI, Debug) << "Decrementing startup frames to " + << startupFrameCount_; } } @@ -1501,6 +1489,8 @@ void CameraData::fillRequestMetadata(const ControlList &bufferControls, Request { request->metadata().set(controls::SensorTimestamp, bufferControls.get(controls::SensorTimestamp).value_or(0)); + request->metadata().set(controls::FrameWallClock, + bufferControls.get(controls::FrameWallClock).value_or(0)); if (cropParams_.size()) { std::vector crops; diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index aae0c2f35..4bce4ec4f 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -20,6 +20,7 @@ #include "libcamera/internal/bayer_format.h" #include "libcamera/internal/camera.h" #include "libcamera/internal/camera_sensor.h" +#include "libcamera/internal/clock_recovery.h" #include "libcamera/internal/framebuffer.h" #include "libcamera/internal/media_device.h" #include "libcamera/internal/media_object.h" @@ -48,8 +49,7 @@ class CameraData : public Camera::Private public: CameraData(PipelineHandler *pipe) : Camera::Private(pipe), state_(State::Stopped), - dropFrameCount_(0), buffersAllocated_(false), - ispOutputCount_(0), ispOutputTotal_(0) + startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false) { } @@ -68,7 +68,7 @@ public: void freeBuffers(); virtual void platformFreeBuffers() = 0; - void enumerateVideoDevices(MediaLink *link, const std::string &frontend); + bool enumerateVideoDevices(MediaLink *link, const std::string &frontend); int loadPipelineConfiguration(); int loadIPA(ipa::RPi::InitResult *result); @@ -151,7 +151,8 @@ public: /* Mapping of CropParams keyed by the output stream order in CameraConfiguration */ std::map cropParams_; - unsigned int dropFrameCount_; + unsigned int startupFrameCount_; + unsigned int invalidFrameCount_; /* * If set, this stores the value that represets a gain of one for @@ -163,11 +164,6 @@ public: bool buffersAllocated_; struct Config { - /* - * Override any request from the IPA to drop a number of startup - * frames. - */ - bool disableStartupFrameDrops; /* * Override the camera timeout value calculated by the IPA based * on frame durations. @@ -177,15 +173,14 @@ public: Config config_; + ClockRecovery wallClockRecovery_; + protected: void fillRequestMetadata(const ControlList &bufferControls, Request *request); virtual void tryRunPipeline() = 0; - unsigned int ispOutputCount_; - unsigned int ispOutputTotal_; - private: void checkRequestCompleted(); }; diff --git a/src/libcamera/pipeline/rpi/pisp/data/example.yaml b/src/libcamera/pipeline/rpi/pisp/data/example.yaml index d67e654a8..baf03be79 100644 --- a/src/libcamera/pipeline/rpi/pisp/data/example.yaml +++ b/src/libcamera/pipeline/rpi/pisp/data/example.yaml @@ -16,11 +16,6 @@ # # "num_cfe_config_queue": 2, - # Override any request from the IPA to drop a number of startup - # frames. - # - # "disable_startup_frame_drops": false, - # Custom timeout value (in ms) for camera to use. This overrides # the value computed by the pipeline handler based on frame # durations. diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index 91e7f4c94..92b9070c1 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -1755,9 +1755,15 @@ void PiSPCameraData::cfeBufferDequeue(FrameBuffer *buffer) auto [ctrl, delayContext] = delayedCtrls_->get(buffer->metadata().sequence); /* * Add the frame timestamp to the ControlList for the IPA to use - * as it does not receive the FrameBuffer object. + * as it does not receive the FrameBuffer object. Also derive a + * corresponding wallclock value. */ - ctrl.set(controls::SensorTimestamp, buffer->metadata().timestamp); + wallClockRecovery_.addSample(); + uint64_t sensorTimestamp = buffer->metadata().timestamp; + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp); + + ctrl.set(controls::SensorTimestamp, sensorTimestamp); + ctrl.set(controls::FrameWallClock, wallClockTimestamp); job.sensorControls = std::move(ctrl); job.delayContext = delayContext; } else if (stream == &cfe_[Cfe::Config]) { @@ -1834,12 +1840,6 @@ void PiSPCameraData::beOutputDequeue(FrameBuffer *buffer) dmabufSyncEnd(buffer->planes()[0].fd); handleStreamBuffer(buffer, stream); - - /* - * Increment the number of ISP outputs generated. - * This is needed to track dropped frames. - */ - ispOutputCount_++; handleState(); } @@ -1885,7 +1885,6 @@ void PiSPCameraData::prepareIspComplete(const ipa::RPi::BufferIds &buffers, bool * If there is no need to run the Backend, just signal that the * input buffer is completed and all Backend outputs are ready. */ - ispOutputCount_ = ispOutputTotal_; buffer = cfe_[Cfe::Output0].getBuffers().at(bayerId).buffer; handleStreamBuffer(buffer, &cfe_[Cfe::Output0]); } else @@ -1994,7 +1993,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) global.bayer_enables |= PISP_BE_BAYER_ENABLE_INPUT; global.bayer_order = toPiSPBayerOrder(cfeFormat.fourcc); - ispOutputTotal_ = 1; /* Config buffer */ if (PISP_IMAGE_FORMAT_COMPRESSED(inputFormat.format)) { pisp_decompress_config decompress; decompress.offset = DefaultCompressionOffset; @@ -2025,7 +2023,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) setupOutputClipping(ispFormat0, outputFormat0); be_->SetOutputFormat(0, outputFormat0); - ispOutputTotal_++; } if (global.rgb_enables & PISP_BE_RGB_ENABLE_OUTPUT1) { @@ -2049,7 +2046,6 @@ int PiSPCameraData::configureBe(const std::optional &yuvColorSpace) setupOutputClipping(ispFormat1, outputFormat1); be_->SetOutputFormat(1, outputFormat1); - ispOutputTotal_++; } /* Setup the TDN I/O blocks in case TDN gets turned on later. */ @@ -2256,8 +2252,6 @@ void PiSPCameraData::prepareCfe() void PiSPCameraData::prepareBe(uint32_t bufferId, bool stitchSwapBuffers) { - ispOutputCount_ = 0; - FrameBuffer *buffer = cfe_[Cfe::Output0].getBuffers().at(bufferId).buffer; LOG(RPI, Debug) << "Input re-queue to ISP, buffer id " << bufferId diff --git a/src/libcamera/pipeline/rpi/vc4/data/example.yaml b/src/libcamera/pipeline/rpi/vc4/data/example.yaml index b8e01adea..27e543488 100644 --- a/src/libcamera/pipeline/rpi/vc4/data/example.yaml +++ b/src/libcamera/pipeline/rpi/vc4/data/example.yaml @@ -29,11 +29,6 @@ # # "min_total_unicam_buffers": 4, - # Override any request from the IPA to drop a number of startup - # frames. - # - # "disable_startup_frame_drops": false, - # Custom timeout value (in ms) for camera to use. This overrides # the value computed by the pipeline handler based on frame # durations. diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index fe910bdf2..5cadef527 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -597,8 +597,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi stream->setFlags(StreamFlag::External); } - ispOutputTotal_ = outStreams.size(); - /* * If ISP::Output0 stream has not been configured by the application, * we must allow the hardware to generate an output so that the data @@ -625,8 +623,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi return -EINVAL; } - ispOutputTotal_++; - LOG(RPI, Debug) << "Defaulting ISP Output0 format to " << format; } @@ -662,8 +658,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi << ret; return -EINVAL; } - - ispOutputTotal_++; } /* ISP statistics output format. */ @@ -676,8 +670,6 @@ int Vc4CameraData::platformConfigure(const RPi::RPiCameraConfiguration *rpiConfi return ret; } - ispOutputTotal_++; - /* * Configure the Unicam embedded data output format only if the sensor * supports it. @@ -781,9 +773,15 @@ void Vc4CameraData::unicamBufferDequeue(FrameBuffer *buffer) auto [ctrl, delayContext] = delayedCtrls_->get(buffer->metadata().sequence); /* * Add the frame timestamp to the ControlList for the IPA to use - * as it does not receive the FrameBuffer object. + * as it does not receive the FrameBuffer object. Also derive a + * corresponding wallclock value. */ - ctrl.set(controls::SensorTimestamp, buffer->metadata().timestamp); + wallClockRecovery_.addSample(); + uint64_t sensorTimestamp = buffer->metadata().timestamp; + uint64_t wallClockTimestamp = wallClockRecovery_.getOutput(sensorTimestamp); + + ctrl.set(controls::SensorTimestamp, sensorTimestamp); + ctrl.set(controls::FrameWallClock, wallClockTimestamp); bayerQueue_.push({ buffer, std::move(ctrl), delayContext }); } else { embeddedQueue_.push(buffer); @@ -843,12 +841,6 @@ void Vc4CameraData::ispOutputDequeue(FrameBuffer *buffer) handleStreamBuffer(buffer, stream); } - /* - * Increment the number of ISP outputs generated. - * This is needed to track dropped frames. - */ - ispOutputCount_++; - handleState(); } @@ -880,7 +872,6 @@ void Vc4CameraData::prepareIspComplete(const ipa::RPi::BufferIds &buffers, << ", timestamp: " << buffer->metadata().timestamp; isp_[Isp::Input].queueBuffer(buffer); - ispOutputCount_ = 0; if (sensorMetadata_ && embeddedId) { buffer = unicam_[Unicam::Embedded].getBuffers().at(embeddedId & RPi::MaskID).buffer; diff --git a/src/libcamera/pipeline/simple/simple.cpp b/src/libcamera/pipeline/simple/simple.cpp index efb07051b..4c002ca68 100644 --- a/src/libcamera/pipeline/simple/simple.cpp +++ b/src/libcamera/pipeline/simple/simple.cpp @@ -30,6 +30,7 @@ #include #include "libcamera/internal/camera.h" +#include "libcamera/internal/camera_lens.h" #include "libcamera/internal/camera_sensor.h" #include "libcamera/internal/camera_sensor_properties.h" #include "libcamera/internal/converter.h" @@ -41,6 +42,8 @@ #include "libcamera/internal/v4l2_subdevice.h" #include "libcamera/internal/v4l2_videodevice.h" +#include "libcamera/controls.h" + namespace libcamera { LOG_DEFINE_CATEGORY(SimplePipeline) @@ -356,7 +359,7 @@ private: void ispStatsReady(uint32_t frame, uint32_t bufferId); void metadataReady(uint32_t frame, const ControlList &metadata); - void setSensorControls(const ControlList &sensorControls); + void setSensorControls(const ControlList &sensorControls, const ControlList &lensControls); }; class SimpleCameraConfiguration : public CameraConfiguration @@ -1002,7 +1005,7 @@ void SimpleCameraData::metadataReady(uint32_t frame, const ControlList &metadata tryCompleteRequest(info->request); } -void SimpleCameraData::setSensorControls(const ControlList &sensorControls) +void SimpleCameraData::setSensorControls(const ControlList &sensorControls, const ControlList &lensControls) { delayedCtrls_->push(sensorControls); /* @@ -1013,10 +1016,21 @@ void SimpleCameraData::setSensorControls(const ControlList &sensorControls) * but it also bypasses delayedCtrls_, creating AGC regulation issues. * Both problems should be fixed. */ - if (!frameStartEmitter_) { - ControlList ctrls(sensorControls); - sensor_->setControls(&ctrls); - } + if (frameStartEmitter_) + return; + + ControlList ctrls(sensorControls); + sensor_->setControls(&ctrls); + + CameraLens *focusLens = sensor_->focusLens(); + if (!focusLens) + return; + + if (!lensControls.contains(V4L2_CID_FOCUS_ABSOLUTE)) + return; + + const ControlValue &focusValue = lensControls.get(V4L2_CID_FOCUS_ABSOLUTE); + focusLens->setFocusPosition(focusValue.get()); } /* Retrieve all source pads connected to a sink pad through active routes. */ @@ -1406,6 +1420,10 @@ int SimplePipelineHandler::configure(Camera *camera, CameraConfiguration *c) } else { ipa::soft::IPAConfigInfo configInfo; configInfo.sensorControls = data->sensor_->controls(); + if (data->sensor_->focusLens() != nullptr) + configInfo.lensControls = data->sensor_->focusLens()->controls(); + else + configInfo.lensControls = ControlInfoMap(); return data->swIsp_->configure(inputCfg, outputCfgs, configInfo); } } diff --git a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp index 58aa0eb4c..4b5816dfd 100644 --- a/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp +++ b/src/libcamera/pipeline/uvcvideo/uvcvideo.cpp @@ -100,7 +100,7 @@ public: private: int processControl(const UVCCameraData *data, ControlList *controls, unsigned int id, const ControlValue &value); - int processControls(UVCCameraData *data, Request *request); + int processControls(UVCCameraData *data, const ControlList &reqControls); bool acquireDevice(Camera *camera) override; void releaseDevice(Camera *camera) override; @@ -287,7 +287,7 @@ int PipelineHandlerUVC::exportFrameBuffers(Camera *camera, Stream *stream, return data->video_->exportBuffers(count, buffers); } -int PipelineHandlerUVC::start(Camera *camera, [[maybe_unused]] const ControlList *controls) +int PipelineHandlerUVC::start(Camera *camera, const ControlList *controls) { UVCCameraData *data = cameraData(camera); unsigned int count = data->stream_.configuration().bufferCount; @@ -296,13 +296,22 @@ int PipelineHandlerUVC::start(Camera *camera, [[maybe_unused]] const ControlList if (ret < 0) return ret; - ret = data->video_->streamOn(); - if (ret < 0) { - data->video_->releaseBuffers(); - return ret; + if (controls) { + ret = processControls(data, *controls); + if (ret < 0) + goto err_release_buffers; } + ret = data->video_->streamOn(); + if (ret < 0) + goto err_release_buffers; + return 0; + +err_release_buffers: + data->video_->releaseBuffers(); + + return ret; } void PipelineHandlerUVC::stopDevice(Camera *camera) @@ -331,6 +340,8 @@ int PipelineHandlerUVC::processControl(const UVCCameraData *data, ControlList *c cid = V4L2_CID_GAIN; else if (id == controls::Gamma) cid = V4L2_CID_GAMMA; + else if (id == controls::AeEnable) + return 0; /* Handled in `Camera::queueRequest()`. */ else return -EINVAL; @@ -410,11 +421,11 @@ int PipelineHandlerUVC::processControl(const UVCCameraData *data, ControlList *c return 0; } -int PipelineHandlerUVC::processControls(UVCCameraData *data, Request *request) +int PipelineHandlerUVC::processControls(UVCCameraData *data, const ControlList &reqControls) { ControlList controls(data->video_->controls()); - for (const auto &[id, value] : request->controls()) + for (const auto &[id, value] : reqControls) processControl(data, &controls, id, value); for (const auto &ctrl : controls) @@ -442,7 +453,7 @@ int PipelineHandlerUVC::queueRequestDevice(Camera *camera, Request *request) return -ENOENT; } - int ret = processControls(data, request); + int ret = processControls(data, request->controls()); if (ret < 0) return ret; diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index d84dff3c9..31d501a13 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -372,6 +372,8 @@ void PipelineHandler::stop(Camera *camera) /* Make sure no requests are pending. */ Camera::Private *data = camera->_d(); + // WIP: Just clean for now, idk maybe something wrong with thread sync? + data->queuedRequests_.clear(); ASSERT(data->queuedRequests_.empty()); data->requestSequence_ = 0; diff --git a/src/libcamera/process.cpp b/src/libcamera/process.cpp index d836fb07a..0eae68072 100644 --- a/src/libcamera/process.cpp +++ b/src/libcamera/process.cpp @@ -241,7 +241,12 @@ int Process::start(const std::string &path, int ret; if (running_) - return 0; + return -EBUSY; + + for (int fd : fds) { + if (fd < 0) + return -EINVAL; + } int childPid = fork(); if (childPid == -1) { @@ -279,14 +284,15 @@ int Process::start(const std::string &path, if (file && strcmp(file, "syslog")) unsetenv("LIBCAMERA_LOG_FILE"); - const char **argv = new const char *[args.size() + 2]; - unsigned int len = args.size(); + const size_t len = args.size(); + auto argv = std::make_unique(len + 2); + argv[0] = path.c_str(); - for (unsigned int i = 0; i < len; i++) + for (size_t i = 0; i < len; i++) argv[i + 1] = args[i].c_str(); argv[len + 1] = nullptr; - execv(path.c_str(), (char **)argv); + execv(path.c_str(), const_cast(argv.get())); _exit(EXIT_FAILURE); } @@ -297,6 +303,8 @@ void Process::closeAllFdsExcept(const std::vector &fds) std::vector v(fds); sort(v.begin(), v.end()); + ASSERT(v.empty() || v.front() >= 0); + DIR *dir = opendir("/proc/self/fd"); if (!dir) return; diff --git a/src/libcamera/sensor/camera_sensor.cpp b/src/libcamera/sensor/camera_sensor.cpp index d19b5e2e7..4f2fd2690 100644 --- a/src/libcamera/sensor/camera_sensor.cpp +++ b/src/libcamera/sensor/camera_sensor.cpp @@ -302,8 +302,9 @@ int CameraSensor::setEmbeddedDataEnabled(bool enable) * camera sensor, likely at configure() time. * * If the requested \a orientation cannot be obtained, the \a orientation - * parameter is adjusted to report the current image orientation and - * Transform::Identity is returned. + * parameter is adjusted to report the native image orientation (i.e. resulting + * from the physical mounting rotation of the camera sensor, without any + * transformation) and Transform::Identity is returned. * * If the requested \a orientation can be obtained, the function computes a * Transform and does not adjust \a orientation. diff --git a/src/libcamera/software_isp/debayer_cpu.cpp b/src/libcamera/software_isp/debayer_cpu.cpp index 66f6038c1..c6f070225 100644 --- a/src/libcamera/software_isp/debayer_cpu.cpp +++ b/src/libcamera/software_isp/debayer_cpu.cpp @@ -668,7 +668,7 @@ void DebayerCpu::process2(const uint8_t *src, uint8_t *dst) for (unsigned int y = window_.y; y < yEnd; y += 2) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(y, linePointers); + if (this->enable_statistic) stats_->processLine0(y, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -683,7 +683,7 @@ void DebayerCpu::process2(const uint8_t *src, uint8_t *dst) if (window_.y == 0) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(yEnd, linePointers); + if (this->enable_statistic) stats_->processLine0(yEnd, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -720,7 +720,7 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) for (unsigned int y = window_.y; y < yEnd; y += 4) { shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine0(y, linePointers); + if (this->enable_statistic) stats_->processLine0(y, linePointers); (this->*debayer0_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -733,7 +733,7 @@ void DebayerCpu::process4(const uint8_t *src, uint8_t *dst) shiftLinePointers(linePointers, src); memcpyNextLine(linePointers); - stats_->processLine2(y, linePointers); + if (this->enable_statistic) stats_->processLine2(y, linePointers); (this->*debayer2_)(dst, linePointers); src += inputConfig_.stride; dst += outputConfig_.stride; @@ -771,7 +771,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output for (const FrameBuffer::Plane &plane : output->planes()) dmaSyncers.emplace_back(plane.fd, DmaSyncer::SyncType::Write); - + enable_statistic = params.collect_stats; green_ = params.green; greenCcm_ = params.greenCcm; if (swapRedBlueGains_) { @@ -805,7 +805,7 @@ void DebayerCpu::process(uint32_t frame, FrameBuffer *input, FrameBuffer *output return; } - stats_->startFrame(); + if(this->enable_statistic) stats_->startFrame(); if (inputConfig_.patternSize.height == 2) process2(in.planes()[0].data(), out.planes()[0].data()); diff --git a/src/libcamera/software_isp/debayer_cpu.h b/src/libcamera/software_isp/debayer_cpu.h index 926195e98..54304053f 100644 --- a/src/libcamera/software_isp/debayer_cpu.h +++ b/src/libcamera/software_isp/debayer_cpu.h @@ -165,6 +165,7 @@ private: /* Skip 30 frames for things to stabilize then measure 30 frames */ static constexpr unsigned int kFramesToSkip = 30; static constexpr unsigned int kLastFrameToMeasure = 60; + bool enable_statistic = true; }; } /* namespace libcamera */ diff --git a/src/libcamera/software_isp/software_isp.cpp b/src/libcamera/software_isp/software_isp.cpp index 28e2a360e..8f41591c8 100644 --- a/src/libcamera/software_isp/software_isp.cpp +++ b/src/libcamera/software_isp/software_isp.cpp @@ -395,9 +395,9 @@ void SoftwareIsp::saveIspParams() debayerParams_ = *sharedParams_; } -void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls) +void SoftwareIsp::setSensorCtrls(const ControlList &sensorControls, const ControlList &lensControls) { - setSensorControls.emit(sensorControls); + setSensorControls.emit(sensorControls, lensControls); } void SoftwareIsp::statsReady(uint32_t frame, uint32_t bufferId) diff --git a/src/libcamera/software_isp/swstats_cpu.cpp b/src/libcamera/software_isp/swstats_cpu.cpp index c520c806e..a6a73f483 100644 --- a/src/libcamera/software_isp/swstats_cpu.cpp +++ b/src/libcamera/software_isp/swstats_cpu.cpp @@ -147,7 +147,10 @@ static constexpr unsigned int kBlueYMul = 29; /* 0.114 * 256 */ \ uint64_t sumR = 0; \ uint64_t sumG = 0; \ - uint64_t sumB = 0; + uint64_t sumB = 0; \ + pixel_t r0 = 0, r1 = 0, b0 = 0, \ + b1 = 0, g0 = 0, g1 = 0; \ + uint64_t sharpness = 0; #define SWSTATS_ACCUMULATE_LINE_STATS(div) \ sumR += r; \ @@ -157,12 +160,18 @@ static constexpr unsigned int kBlueYMul = 29; /* 0.114 * 256 */ yVal = r * kRedYMul; \ yVal += g * kGreenYMul; \ yVal += b * kBlueYMul; \ - stats_.yHistogram[yVal * SwIspStats::kYHistogramSize / (256 * 256 * (div))]++; + stats_.yHistogram[yVal * SwIspStats::kYHistogramSize / (256 * 256 * (div))]++; \ + if (r0 != 0) \ + sharpness += abs(r - 2*r1 + r0) * kRedYMul + abs(g - 2*g1 + g0) * kGreenYMul + abs(b - 2*b1 + b0) * kBlueYMul; \ + r0 = r1; g0 = g1; b0 = b1; \ + r1 = r; g1 = g; b1 = b; \ + #define SWSTATS_FINISH_LINE_STATS() \ stats_.sumR_ += sumR; \ stats_.sumG_ += sumG; \ - stats_.sumB_ += sumB; + stats_.sumB_ += sumB; \ + stats_.sharpness += sharpness; void SwStatsCpu::statsBGGR8Line0(const uint8_t *src[]) { @@ -306,6 +315,7 @@ void SwStatsCpu::startFrame(void) stats_.sumR_ = 0; stats_.sumB_ = 0; stats_.sumG_ = 0; + stats_.sharpness = 0; stats_.yHistogram.fill(0); } diff --git a/src/libcamera/v4l2_videodevice.cpp b/src/libcamera/v4l2_videodevice.cpp index d53aa2d3c..7822bf1ef 100644 --- a/src/libcamera/v4l2_videodevice.cpp +++ b/src/libcamera/v4l2_videodevice.cpp @@ -2031,10 +2031,9 @@ int V4L2VideoDevice::streamOff() /* Send back all queued buffers. */ for (auto it : queuedBuffers_) { FrameBuffer *buffer = it.second; - FrameMetadata &metadata = buffer->_d()->metadata(); cache_->put(it.first); - metadata.status = FrameMetadata::FrameCancelled; + buffer->_d()->cancel(); bufferReady.emit(buffer); } diff --git a/subprojects/libpisp.wrap b/subprojects/libpisp.wrap index 8b62c036f..0e0c7baaf 100644 --- a/subprojects/libpisp.wrap +++ b/subprojects/libpisp.wrap @@ -2,5 +2,5 @@ [wrap-git] url = https://github.com/raspberrypi/libpisp.git -revision = v1.2.0 +revision = v1.2.1 depth = 1 diff --git a/subprojects/libyaml.wrap b/subprojects/libyaml.wrap index 392416c61..44ac0ff8e 100644 --- a/subprojects/libyaml.wrap +++ b/subprojects/libyaml.wrap @@ -1,7 +1,13 @@ # SPDX-License-Identifier: CC0-1.0 -[wrap-git] -directory = libyaml -url = https://github.com/yaml/libyaml -# tags/0.2.5 -revision = 2c891fc7a770e8ba2fec34fc6b545c672beb37e6 +[wrap-file] +directory = yaml-0.2.5 +source_url = https://pyyaml.org/download/libyaml/yaml-0.2.5.tar.gz +source_filename = yaml-0.2.5.tar.gz +source_hash = c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4 +patch_filename = libyaml_0.2.5-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/libyaml_0.2.5-1/get_patch +patch_hash = bf2e9b922be00b6b00c5fce29d9fb8dc83f0431c77239f3b73e8b254d3f3f5b5 + +[provide] +yaml-0.1 = yaml_dep diff --git a/test/log/log_api.cpp b/test/log/log_api.cpp index 0b999738d..8d19cf0ce 100644 --- a/test/log/log_api.cpp +++ b/test/log/log_api.cpp @@ -26,6 +26,11 @@ using namespace std; using namespace libcamera; LOG_DEFINE_CATEGORY(LogAPITest) +LOG_DEFINE_CATEGORY(Cat0) +LOG_DEFINE_CATEGORY(Cat1) +LOG_DEFINE_CATEGORY(Cat2) +LOG_DEFINE_CATEGORY(Cat3) +LOG_DEFINE_CATEGORY(Cat4) class LogAPITest : public Test { @@ -74,6 +79,34 @@ protected: return TestPass; } + int testEnvLevels() + { + setenv("LIBCAMERA_LOG_LEVELS", + "Cat0:0,Cat0:9999,Cat1:INFO,Cat1:INVALID,Cat2:2,Cat2:-1," + "Cat3:ERROR,Cat3:{[]},Cat4:4,Cat4:rubbish", + true); + logSetTarget(libcamera::LoggingTargetNone); + + const std::pair expected[] = { + { _LOG_CATEGORY(Cat0)(), libcamera::LogDebug }, + { _LOG_CATEGORY(Cat1)(), libcamera::LogInfo }, + { _LOG_CATEGORY(Cat2)(), libcamera::LogWarning }, + { _LOG_CATEGORY(Cat3)(), libcamera::LogError }, + { _LOG_CATEGORY(Cat4)(), libcamera::LogFatal }, + }; + bool ok = true; + + for (const auto &[c, s] : expected) { + if (c.severity() != s) { + ok = false; + cerr << "Severity of " << c.name() << " (" << c.severity() << ") " + << "does not equal " << s << endl; + } + } + + return ok ? TestPass : TestFail; + } + int testFile() { int fd = open("/tmp", O_TMPFILE | O_RDWR, S_IRUSR | S_IWUSR); @@ -135,7 +168,11 @@ protected: int run() override { - int ret = testFile(); + int ret = testEnvLevels(); + if (ret != TestPass) + return TestFail; + + ret = testFile(); if (ret != TestPass) return TestFail; diff --git a/test/log/meson.build b/test/log/meson.build index 2298ff84e..f413c3898 100644 --- a/test/log/meson.build +++ b/test/log/meson.build @@ -11,5 +11,6 @@ foreach test : log_test link_with : test_libraries, include_directories : test_includes_internal) - test(test['name'], exe, suite : 'log') + test(test['name'], exe, suite : 'log', + should_fail : test.get('should_fail', false)) endforeach